Disable TLS certificate verification in crawler allows MITM/SSRF

.env.example

@@ -29,6 +29,7 @@ SECUSCAN_DOCKER_NETWORK=restricted
 SECUSCAN_SAFE_MODE_DEFAULT=true
 SECUSCAN_REQUIRE_CONSENT=true
 SECUSCAN_ALLOW_LOOPBACK_SCANS=true
+# SECUSCAN_TLS_VERIFY=true
 # SECUSCAN_ALLOWED_NETWORKS=127.0.0.1,192.168.*.*,10.*.*.*,172.16.*.*
 # SECUSCAN_CORS_ALLOWED_ORIGINS=http://127.0.0.1:5173,http://localhost:5173
 

backend/secuscan/config.py

@@ -63,6 +63,7 @@ class Settings(BaseSettings):
     # Security
     safe_mode_default: bool = True
     verify_ssl: bool = True
+    tls_verify: bool = True
     dns_resolution_timeout_seconds: float = 1.5
     dns_cache_ttl_seconds: int = 60
     dns_rebind_check: bool = True

backend/secuscan/crawler.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import logging
 from html.parser import HTMLParser
 import re
 from typing import Any, Dict, List
@@ -11,6 +12,8 @@
 
 from .config import settings
 
+logger = logging.getLogger(__name__)
+
 
 class _SurfaceParser(HTMLParser):
     def __init__(self) -> None:
@@ -83,38 +86,94 @@ async def crawl_target(
     max_redirects: int = 10,
     max_size: int = 5 * 1024 * 1024,
 ) -> Dict[str, Any]:
-    """Fetch a target and normalize discovered links/forms/scripts/API hints."""
+    """Fetch a target and normalize discovered links/forms/scripts/API hints.
+
+    Redirect Policy & Security Semantics:
+    - Enforces configurable TLS certificate verification (via ``SECUSCAN_TLS_VERIFY``).
+    - Only same-host HTTP/HTTPS redirects relative to the original seed URL host are followed.
+    - Cross-host redirects (different target domain/hostname) or non-HTTP/HTTPS schemes
+      (e.g., ``javascript:``, ``file:``) are blocked and logged to mitigate MITM/SSRF.
+    - Redirect loops or chains exceeding ``max_redirects`` raise ``httpx.TooManyRedirects``.
+    """
     headers = _build_headers(extra_headers)
+    tls_verify = getattr(settings, "tls_verify", True) and getattr(settings, "verify_ssl", True)
+
+    seed_parsed = urlparse(url)
+    seed_hostname = (seed_parsed.hostname or "").lower()
+
+    current_url = url
+    history: List[httpx.Response] = []
+    redirect_count = 0
+
     async with httpx.AsyncClient(
-        follow_redirects=True,
-        max_redirects=max_redirects,
+        follow_redirects=False,
         timeout=timeout,
         headers=headers,
         cookies=cookies or {},
-        verify=settings.verify_ssl,
+        verify=tls_verify,
     ) as client:
-        async with client.stream("GET", url) as response:
-            # Check Content-Length header if present
-            content_length = response.headers.get("content-length")
-            if content_length:
-                try:
-                    cl_val = int(content_length)
-                except ValueError:
-                    cl_val = 0
-                if cl_val > max_size:
-                    raise ValueError(f"Response size exceeds limit of {max_size} bytes")
-
-            # Read response in chunks to enforce size limit
-            body_chunks = []
-            bytes_read = 0
-            async for chunk in response.aiter_bytes():
-                bytes_read += len(chunk)
-                if bytes_read > max_size:
-                    raise ValueError(f"Response size exceeds limit of {max_size} bytes")
-                body_chunks.append(chunk)
-
-            body_bytes = b"".join(body_chunks)
-            body = body_bytes.decode("utf-8", errors="replace")
+        while True:
+            async with client.stream("GET", current_url) as response:
+                # Check Content-Length header if present
+                content_length = response.headers.get("content-length")
+                if content_length:
+                    try:
+                        cl_val = int(content_length)
+                    except ValueError:
+                        cl_val = 0
+                    if cl_val > max_size:
+                        raise ValueError(f"Response size exceeds limit of {max_size} bytes")
+
+                # Read response in chunks to enforce size limit
+                body_chunks = []
+                bytes_read = 0
+                async for chunk in response.aiter_bytes():
+                    bytes_read += len(chunk)
+                    if bytes_read > max_size:
+                        raise ValueError(f"Response size exceeds limit of {max_size} bytes")
+                    body_chunks.append(chunk)
+
+                body_bytes = b"".join(body_chunks)
+                body = body_bytes.decode("utf-8", errors="replace")
+
+            # Check if redirect and process same-host policy
+            if response.is_redirect and "location" in response.headers:
+                if redirect_count >= max_redirects:
+                    raise httpx.TooManyRedirects(
+                        f"Too many redirects (max_redirects={max_redirects})",
+                        request=httpx.Request("GET", current_url),
+                    )
+                location = response.headers["location"]
+                next_url = urljoin(current_url, location)
+                next_parsed = urlparse(next_url)
+                next_scheme = (next_parsed.scheme or "").lower()
+                next_hostname = (next_parsed.hostname or "").lower()
+
+                if next_scheme not in {"http", "https"} or not next_hostname:
+                    logger.warning(
+                        "Crawler redirect blocked due to unsupported/invalid Location: %s -> %s",
+                        current_url,
+                        next_url,
+                    )
+                    break
+
+                if seed_hostname and next_hostname != seed_hostname:
+                    logger.warning(
+                        "Crawler redirect blocked due to host mismatch: %s -> %s (seed host: %s, redirect host: %s)",
+                        current_url,
+                        next_url,
+                        seed_hostname,
+                        next_hostname,
+                    )
+                    break
+
+                history.append(response)
+                current_url = next_url
+                redirect_count += 1
+            else:
+                break
+
+    response.history = history
     parser = _SurfaceParser()
     parser.feed(body)