refactor: migrate from urllib to httpx2

with urllib there was a decoding error for python.org

1 files changed, 5 insertions, 4 deletions

diff --git a/web_utils.py b/web_utils.py
index 91561b5..6fecb47 100644
--- a/web_utils.py
+++ b/web_utils.py
@@ -1,4 +1,4 @@
-import urllib.request
+import httpx2
 import subprocess
 import re
 from typing import Dict, Union
@@ -50,9 +50,10 @@ def fetch_and_markdownify(url: str) -> FetchResult:
         return _cache[url]
 
     try:
-        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
-        with urllib.request.urlopen(req, timeout=10) as response:
-            html = response.read().decode("utf-8")
+        with httpx2.Client(headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0) as client:
+            response = client.get(url)
+            response.raise_for_status()
+            html = response.text
             markdown = html_to_markdown(html)
             res = FetchSuccess(content=markdown)
             _cache[url] = res