diff options
| -rw-r--r-- | web_utils.py | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/web_utils.py b/web_utils.py index 91561b5..6fecb47 100644 --- a/web_utils.py +++ b/web_utils.py @@ -1,4 +1,4 @@ -import urllib.request +import httpx2 import subprocess import re from typing import Dict, Union @@ -50,9 +50,10 @@ def fetch_and_markdownify(url: str) -> FetchResult: return _cache[url] try: - req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) - with urllib.request.urlopen(req, timeout=10) as response: - html = response.read().decode("utf-8") + with httpx2.Client(headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0) as client: + response = client.get(url) + response.raise_for_status() + html = response.text markdown = html_to_markdown(html) res = FetchSuccess(content=markdown) _cache[url] = res |