From e42ae589c623dd1a5ae379899c94f93ed05bc848 Mon Sep 17 00:00:00 2001 From: Joel Kronqvist Date: Mon, 12 Oct 2026 19:31:08 +0300 Subject: refactor: migrate from urllib to httpx2 with urllib there was a decoding error for python.org --- web_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/web_utils.py b/web_utils.py index 91561b5..6fecb47 100644 --- a/web_utils.py +++ b/web_utils.py @@ -1,4 +1,4 @@ -import urllib.request +import httpx2 import subprocess import re from typing import Dict, Union @@ -50,9 +50,10 @@ def fetch_and_markdownify(url: str) -> FetchResult: return _cache[url] try: - req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) - with urllib.request.urlopen(req, timeout=10) as response: - html = response.read().decode("utf-8") + with httpx2.Client(headers={"User-Agent": "Mozilla/5.0"}, timeout=10.0) as client: + response = client.get(url) + response.raise_for_status() + html = response.text markdown = html_to_markdown(html) res = FetchSuccess(content=markdown) _cache[url] = res -- cgit v1.3.1