Increase HTTPX timeout to 30 seconds from 5 seconds

Improve generate_filepath rendering of paths with user-home ~
Only establish signal handlers when entering via entrypoint

src/hibp_downloader/__init__.py

@@ -13,6 +13,7 @@
 
 LOGGER_NAME = "hibp-downloader"
 PWNEDPASSWORDS_API_URL = "https://api.pwnedpasswords.com"
+HTTPX_TIMEOUT_SECONDS = 30
 LOCAL_CACHE_TTL_DEFAULT = 12 * 3600
 MULTIPROCESSING_PROCESSES_DEFAULT = int(cpu_count() if cpu_count() else 4)  # type: ignore[arg-type]
 MULTIPROCESSING_PREFIXES_CHUNK_SIZE = 10

src/hibp_downloader/lib/filedata.py

@@ -24,7 +24,7 @@ def generate_filepath(
 ) -> Path:
     return Path(
         os.path.join(
-            base_path,
+            os.path.expanduser(base_path),
             hash_type.lower(),
             prefix[0:2].lower(),
             prefix[2:4].lower(),
@@ -42,12 +42,13 @@ async def append_stringfile(filepath: Path, content: str) -> None:
 
 
 async def save_bytesfile(filepath: Path, content: bytes, timestamp: Union[datetime, None] = None) -> None:
-    await aiofiles.os.makedirs(os.path.dirname(filepath), exist_ok=True)
-    async with aiofiles.open(filepath, mode="wb") as f:
+    full_path = os.path.realpath(os.path.expanduser(filepath))
+    await aiofiles.os.makedirs(os.path.dirname(full_path), exist_ok=True)
+    async with aiofiles.open(full_path, mode="wb") as f:
         await f.write(content)
 
     if timestamp:
-        os.utime(filepath, times=(timestamp.timestamp(), timestamp.timestamp()))
+        os.utime(full_path, times=(timestamp.timestamp(), timestamp.timestamp()))
 
 
 async def save_datafile(

src/hibp_downloader/lib/http.py

@@ -1,11 +1,15 @@
+import random
+
 import httpx
 
-from hibp_downloader import LOGGER_NAME, __title__, __version__
+from hibp_downloader import HTTPX_TIMEOUT_SECONDS, LOGGER_NAME, __title__, __version__
 from hibp_downloader.exceptions import HibpDownloaderException
 from hibp_downloader.lib.logger import logger_get
 
 logger = logger_get(LOGGER_NAME)
 
+__TESTING_RANDOM_ERROR_INJECT_RATE = 0  # manual development testing only
+
 
 async def httpx_debug_request(request):
     logger.debug(f"request: {request.method} {request.url}")
@@ -39,19 +43,22 @@ async def httpx_binary_response(url, etag=None, method="GET", encoding="gzip", d
     httpx_client = {
         "headers": headers,
         "http2": True,
-        "timeout": 5,
+        "timeout": HTTPX_TIMEOUT_SECONDS,
         "follow_redirects": False,
         "trust_env": False,
     }
 
     if event_hooks:
         httpx_client["event_hooks"] = event_hooks
 
+    if __TESTING_RANDOM_ERROR_INJECT_RATE and __TESTING_RANDOM_ERROR_INJECT_RATE > random.random():
+        url = url.replace("http", "broken")
+
     async with httpx.AsyncClient(**httpx_client) as client:
         request = client.build_request(method=method, url=url)
         try:
             response = await client.send(request=request, stream=True)
-        except (httpx.ConnectError, httpx.RemoteProtocolError):
+        except (httpx.ConnectError, httpx.RemoteProtocolError, httpx.HTTPError):
             raise HibpDownloaderException(f"Unable to establish connection {url}")
 
         response.binary = b"".join([part async for part in response.aiter_raw()])

src/hibp_downloader/main.py

@@ -2,7 +2,7 @@
 # Copyright [2022-2023] Threat Patrols Pty Ltd (https://www.threatpatrols.com)
 #
 
-from signal import SIGINT, signal
+from signal import SIGINT, SIGTERM, signal
 
 from . import LOGGER_NAME
 from .lib.app import invoke_app
@@ -12,12 +12,11 @@
 
 
 def entrypoint():
+    signal(SIGINT, signal_handler)
+    signal(SIGTERM, signal_handler)
     invoke_app()
 
 
-def sigint_handler(__signal_received, __frame):
+def signal_handler(__signal_received, __frame):
     print("SIGINT received, exiting.")
     exit()
-
-
-signal(SIGINT, sigint_handler)