Port sanitizeRequestHeaders, fetchWithValidatedEndpoint and unawaited redirect cancel

docs/articles/secure-url-fetching.md

@@ -21,5 +21,16 @@ var response = await Download.FetchWithValidatedRedirectsAsync(
     trustedOrigin: "https://example.com");
 ```
 
-- **Headers.** Each hop receives its own copy of the headers. A redirect that crosses origin drops every header except `user-agent`, and the dropped headers do not return when a later hop comes back to the first origin.
+- **Headers.** Before the first request, the helper removes proxy, origin-spoofing, cloud-metadata, cookie, and hop-by-hop headers (`host`, `forwarded`, `via`, `x-forwarded-*`, `x-real-ip`, `metadata`, `metadata-flavor`, `cookie`, and similar). It keeps `authorization` and other credential headers, so the caller must make sure the first URL may receive them. Each hop receives its own copy of the headers. A redirect that crosses origin drops every header except `user-agent`, and the dropped headers do not return when a later hop comes back to the first origin.
 - **`trustedOrigin`.** A hop on this origin skips validation. Use it for an origin that you configure, such as a provider base URL that points to `localhost` or an internal host. Never build it from response data. A hop on any other origin is always validated, even when `trustedOrigin` is set.
+- **Cancelling redirect bodies.** The helper starts cancelling a redirect response's body and moves on. It does not wait for the cancel to finish.
+
+## Fetching one endpoint
+
+`Download.FetchWithValidatedEndpointAsync` validates one URL and requests it once, without following redirects. Use it for requests that carry credentials or a body. Pass a `DownloadRequest` with `Method`, `Headers`, and `Body`. By default a redirect is rejected (`redirect` is `error`). Pass `manual` only if you validate the `Location` target yourself before you request it. As with `FetchWithValidatedRedirectsAsync`, `trustedOrigin` skips validation for a developer-configured origin.
+
+```csharp
+var response = await Download.FetchWithValidatedEndpointAsync(
+    "https://auth.example.com/token",
+    new DownloadRequest(headers, cancellationToken, "manual") { Method = "POST", Body = formBytes });
+```

src/Vercel.AI/ProviderUtils/RequestHeaders.cs

@@ -0,0 +1,65 @@
+// Copyright 2023 Vercel, Inc.
+// Copyright 2026 Darío Kondratiuk
+// SPDX-License-Identifier: Apache-2.0
+
+using Vercel.AI.Util;
+
+namespace Vercel.AI.ProviderUtils;
+
+internal static class RequestHeaders
+{
+    /// <summary>
+    /// Returns a copy of <paramref name="input"/> without the headers that must not reach an untrusted URL: host and virtual-host
+    /// routing, proxy and origin spoofing, cloud metadata, cookies, and hop-by-hop transport headers (RFC 7230 section 6.1).
+    /// <c>Authorization</c> and other credential-bearing caller headers are kept on purpose, because trusted provider requests may need them.
+    /// Maps to <c>sanitizeRequestHeaders</c>.
+    /// </summary>
+    internal static HeaderCollection SanitizeRequestHeaders(HeaderCollection input)
+    {
+        var headers = new HeaderCollection();
+        foreach (var header in input.Entries())
+        {
+            if (!IsBlockedRequestHeader(header.Key))
+            {
+                headers.Add(header.Key, header.Value);
+            }
+        }
+
+        return headers;
+    }
+
+    private static bool IsBlockedRequestHeader(string name)
+    {
+        switch (name.ToLowerInvariant())
+        {
+            // Hop-by-hop / transport
+            case "connection":
+            case "keep-alive":
+            case "te":
+            case "trailer":
+            case "transfer-encoding":
+            case "upgrade":
+            // Host / virtual-host routing
+            case "host":
+            // Proxy / origin spoofing
+            case "forwarded":
+            case "proxy-authorization":
+            case "via":
+            case "x-forwarded-for":
+            case "x-forwarded-host":
+            case "x-forwarded-proto":
+            case "x-real-ip":
+            // Cloud metadata (GCP, AWS IMDSv1/v2, Azure, Alibaba, DigitalOcean)
+            case "metadata":
+            case "metadata-flavor":
+            case "x-aws-ec2-metadata-token":
+            case "x-metadata-token":
+            // Session / cookie
+            case "cookie":
+            case "set-cookie":
+                return true;
+            default:
+                return false;
+        }
+    }
+}

src/Vercel.AI/Util/Download.cs

@@ -87,9 +87,15 @@ public DownloadRequest(HeaderCollection? headers, CancellationToken abortSignal,
     /// <summary>Cancellation signal for the attempt.</summary>
     public CancellationToken AbortSignal { get; }
 
-    /// <summary><c>manual</c> or <c>follow</c>.</summary>
+    /// <summary><c>manual</c>, <c>follow</c>, or <c>error</c>.</summary>
     public string Redirect { get; }
 
+    /// <summary>HTTP method. Defaults to <c>GET</c>.</summary>
+    public string Method { get; set; } = "GET";
+
+    /// <summary>Request body, or <c>null</c> for none.</summary>
+    public byte[]? Body { get; set; }
+
     /// <summary>Largest body the fetch may read. The default fetch stops reading past it.</summary>
     public long MaxBytes { get; set; } = Download.DefaultMaxBytes;
 }
@@ -321,9 +327,41 @@ public static Task<DownloadResponse> FetchWithValidatedRedirectsAsync(
         return FetchWithValidatedRedirectsAsync(url, DefaultMaxBytes, headers, abortSignal, maxRedirects, fetch, trustedOrigin);
     }
 
+    /// <summary>
+    /// Fetches one validated URL without following redirects. A redirect is rejected by default (<c>error</c>). A caller that passes
+    /// <c>manual</c> must validate the <c>Location</c> target before it requests another URL.
+    /// </summary>
+    /// <param name="url">URL to request.</param>
+    /// <param name="request">Method, headers, body, and cancellation signal. Its <see cref="DownloadRequest.Redirect"/> is replaced by <paramref name="redirect"/>.</param>
+    /// <param name="fetch">Fetch used for the request. Defaults to <see cref="Fetch"/>, then the built-in fetch.</param>
+    /// <param name="trustedOrigin">A developer-configured origin whose requests skip validation. Never derive it from response data.</param>
+    /// <param name="redirect"><c>error</c> or <c>manual</c>.</param>
+    /// <exception cref="DownloadError">The URL is unsafe.</exception>
+    public static async Task<DownloadResponse> FetchWithValidatedEndpointAsync(
+        string url,
+        DownloadRequest? request = null,
+        DownloadFetch? fetch = null,
+        string? trustedOrigin = null,
+        string redirect = "error")
+    {
+        var isTrusted = trustedOrigin != null && ProviderValues.IsSameOrigin(url, trustedOrigin);
+        if (!isTrusted)
+        {
+            ValidateDownloadUrl(url);
+        }
+
+        var init = new DownloadRequest(request == null ? null : new HeaderCollection(request.Headers.Entries()), request?.AbortSignal ?? default, redirect)
+        {
+            Method = request?.Method ?? "GET",
+            Body = request?.Body,
+            MaxBytes = request?.MaxBytes ?? DefaultMaxBytes,
+        };
+        return await (fetch ?? Fetch ?? DefaultFetchAsync)(url, init).ConfigureAwait(false);
+    }
+
     private static async Task<DownloadResponse> FetchWithValidatedRedirectsAsync(string url, long maxBytes, HeaderCollection? headers, CancellationToken abortSignal, int maxRedirects, DownloadFetch? fetch, string? trustedOrigin)
     {
-        var currentHeaders = headers;
+        var currentHeaders = headers == null ? null : RequestHeaders.SanitizeRequestHeaders(headers);
 
         // Snapshot per hop: an injected fetch may defer, and hops between credential drops would otherwise share one instance.
         DownloadRequest PerHopInit(string redirect)
@@ -355,7 +393,9 @@ DownloadRequest PerHopInit(string redirect)
             var location = response.Headers.Get("location");
             if (IsRedirectStatus(response.Status) && !string.IsNullOrEmpty(location))
             {
-                response.Cancel();
+                // Start releasing the redirect response's connection before the next hop. Do not wait for it: a cancel that stays
+                // pending must not hold up the redirect.
+                _ = Task.Run(response.Cancel);
                 var next = new Uri(new Uri(currentUrl), location);
                 if (currentHeaders != null && !ProviderValues.IsSameOrigin(next.AbsoluteUri, currentUrl))
                 {
@@ -438,16 +478,28 @@ private static async Task<DownloadResponse> DefaultFetchCoreAsync(string url, Do
     {
         using (var handler = new HttpClientHandler { AllowAutoRedirect = false })
         using (var client = new HttpClient(handler))
-        using (var message = new HttpRequestMessage(HttpMethod.Get, url))
+        using (var message = new HttpRequestMessage(new HttpMethod(request.Method), url))
         {
-            var userAgent = request.Headers.Get("user-agent");
-            if (!string.IsNullOrEmpty(userAgent))
+            if (request.Body != null)
             {
-                message.Headers.TryAddWithoutValidation("User-Agent", userAgent);
+                message.Content = new ByteArrayContent(request.Body);
+            }
+
+            foreach (var header in request.Headers.Entries())
+            {
+                if (!message.Headers.TryAddWithoutValidation(header.Key, header.Value))
+                {
+                    message.Content?.Headers.TryAddWithoutValidation(header.Key, header.Value);
+                }
             }
 
             using (var response = await client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, request.AbortSignal).ConfigureAwait(false))
             {
+                if (string.Equals(request.Redirect, "error", StringComparison.Ordinal) && IsRedirectStatus((int)response.StatusCode))
+                {
+                    throw new DownloadError(url, "Redirect from " + url + " was blocked");
+                }
+
                 var headers = new HeaderCollection();
                 foreach (var header in response.Headers)
                 {

tests/Vercel.AI.Tests/Upstream/ProviderUtils/FetchWithValidatedRedirects/FetchWithValidatedEndpointTests.cs

@@ -0,0 +1,71 @@
+// Copyright 2023 Vercel, Inc.
+// Copyright 2026 Darío Kondratiuk
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Text;
+using Vercel.AI.Util;
+
+namespace Vercel.AI.Tests.Upstream.ProviderUtils.FetchWithValidatedRedirects;
+
+/// <summary>Port of <c>fetch-with-validated-redirects.test.ts</c> &gt; <c>fetchWithValidatedEndpoint</c>.</summary>
+public sealed class FetchWithValidatedEndpointTests
+{
+    private const string Prefix = "packages/provider-utils/src/fetch-with-validated-redirects.test.ts::fetchWithValidatedEndpoint::";
+
+    [Fact]
+    [UpstreamTest(Prefix + "validates the URL before issuing the request", Coverage = UpstreamCoverage.Covered)]
+    public async Task Validates_the_URL_before_issuing_the_request()
+    {
+        var calls = 0;
+        DownloadFetch fetch = delegate
+        {
+            calls++;
+            return Task.FromResult(OkResponse());
+        };
+
+        await Assert.ThrowsAsync<DownloadError>(delegate
+        {
+            return Download.FetchWithValidatedEndpointAsync("http://169.254.169.254/token", fetch: fetch);
+        });
+        Assert.Equal(0, calls);
+    }
+
+    [Fact]
+    [UpstreamTest(Prefix + "uses redirect: error for credential-bearing requests", Coverage = UpstreamCoverage.Covered)]
+    public async Task Uses_redirect_error_for_credential_bearing_requests()
+    {
+        string? calledUrl = null;
+        DownloadRequest? calledRequest = null;
+        var calls = 0;
+        DownloadFetch fetch = delegate (string url, DownloadRequest request)
+        {
+            calls++;
+            calledUrl = url;
+            calledRequest = request;
+            return Task.FromResult(OkResponse());
+        };
+        var body = Encoding.UTF8.GetBytes("code=secret-code");
+        var headers = new HeaderCollection();
+        headers.Add("authorization", "Basic secret");
+
+        await Download.FetchWithValidatedEndpointAsync(
+            "https://auth.example.com/token",
+            new DownloadRequest(headers, default, "manual") { Method = "POST", Body = body },
+            fetch);
+
+        Assert.Equal(1, calls);
+        Assert.Equal("https://auth.example.com/token", calledUrl);
+        Assert.NotNull(calledRequest);
+        Assert.Equal("POST", calledRequest!.Method);
+        Assert.Equal(
+            new[] { new KeyValuePair<string, string>("authorization", "Basic secret") },
+            calledRequest.Headers.Entries());
+        Assert.Same(body, calledRequest.Body);
+        Assert.Equal("error", calledRequest.Redirect);
+    }
+
+    private static DownloadResponse OkResponse()
+    {
+        return DownloadResponse.Bytes(null, "image/png");
+    }
+}

tests/Vercel.AI.Tests/Upstream/ProviderUtils/FetchWithValidatedRedirects/FetchWithValidatedRedirectsTests.cs

@@ -96,19 +96,74 @@ await Assert.ThrowsAsync<DownloadError>(delegate
     public async Task Starts_cancelling_redirect_response_bodies_to_prevent_socket_leaks()
     {
         var cancelled = 0;
+        var bothCancelled = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+        Action onCancel = delegate
+        {
+            if (Interlocked.Increment(ref cancelled) == 2)
+            {
+                bothCancelled.SetResult(true);
+            }
+        };
         var fetch = new FetchMock(
-            RedirectResponse("https://cdn.example.com/file", onCancel: delegate { cancelled++; }),
-            RedirectResponse("http://169.254.169.254/", onCancel: delegate { cancelled++; }));
+            RedirectResponse("https://cdn.example.com/file", onCancel: onCancel),
+            RedirectResponse("http://169.254.169.254/", onCancel: onCancel));
 
         await Assert.ThrowsAsync<DownloadError>(delegate
         {
             return Download.FetchWithValidatedRedirectsAsync("https://example.com/file", fetch: fetch.Fetch);
         });
 
         // Both the followed hop and the hop rejected by the SSRF guard must have their bodies cancelled.
+        // Cancellation is started without being awaited, so wait for it to run.
+        await bothCancelled.Task.WaitAsync(TimeSpan.FromSeconds(10));
         Assert.Equal(2, cancelled);
     }
 
+    [Theory]
+    [InlineData(301)]
+    [InlineData(302)]
+    [InlineData(303)]
+    [InlineData(307)]
+    [InlineData(308)]
+    [UpstreamTest(Prefix + "does not wait for cancellation before following a %d redirect", Coverage = UpstreamCoverage.Covered)]
+    public async Task Does_not_wait_for_cancellation_before_following_a_redirect(int status)
+    {
+        var cancelEntered = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+        var finishCancellation = new ManualResetEventSlim(false);
+        var secondRequest = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+        var redirect = RedirectResponse(
+            "https://cdn.example.com/file",
+            status,
+            delegate
+            {
+                cancelEntered.SetResult(true);
+                finishCancellation.Wait();
+            });
+        var fetch = new FetchMock(redirect, OkResponse());
+        fetch.OnCall = delegate (int count)
+        {
+            if (count == 2)
+            {
+                secondRequest.SetResult(true);
+            }
+        };
+
+        var download = Download.FetchWithValidatedRedirectsAsync("https://example.com/file", fetch: fetch.Fetch);
+
+        await cancelEntered.Task.WaitAsync(TimeSpan.FromSeconds(10));
+
+        try
+        {
+            await secondRequest.Task.WaitAsync(TimeSpan.FromSeconds(10));
+            Assert.Equal(2, fetch.Calls.Count);
+        }
+        finally
+        {
+            finishCancellation.Set();
+            await download;
+        }
+    }
+
     [Fact]
     [UpstreamTest(Prefix + "resolves relative redirect targets against the current URL", Coverage = UpstreamCoverage.Covered)]
     public async Task Resolves_relative_redirect_targets_against_the_current_URL()
@@ -279,6 +334,24 @@ public async Task Uses_the_injected_fetch_instead_of_the_global_one()
         }
     }
 
+    [Fact]
+    [UpstreamTest(Prefix + "strips blocked request headers before sending", Coverage = UpstreamCoverage.Covered)]
+    public async Task Strips_blocked_request_headers_before_sending()
+    {
+        var fetch = new FetchMock(OkResponse());
+
+        await Download.FetchWithValidatedRedirectsAsync(
+            "https://example.com/file",
+            Headers(("authorization", "Bearer secret"), ("metadata-flavor", "Google"), ("x-forwarded-for", "10.0.0.1"), ("cookie", "session=abc")),
+            fetch: fetch.Fetch);
+
+        var sent = fetch.Calls[0].Request.Headers;
+        Assert.Null(sent.Get("metadata-flavor"));
+        Assert.Null(sent.Get("x-forwarded-for"));
+        Assert.Null(sent.Get("cookie"));
+        Assert.Equal("Bearer secret", sent.Get("authorization"));
+    }
+
     [Fact]
     [UpstreamTest(Prefix + "drops all caller headers except user-agent on a cross-origin redirect but keeps them same-origin", Coverage = UpstreamCoverage.Covered)]
     public async Task Drops_all_caller_headers_except_user_agent_on_a_cross_origin_redirect_but_keeps_them_same_origin()
@@ -378,9 +451,13 @@ public FetchMock(Func<DownloadResponse> factory)
 
         public List<FetchCall> Calls { get; } = new List<FetchCall>();
 
+        /// <summary>Called with the call count after each request is recorded.</summary>
+        public Action<int>? OnCall { get; set; }
+
         public Task<DownloadResponse> Fetch(string url, DownloadRequest request)
         {
             Calls.Add(new FetchCall(url, request));
+            OnCall?.Invoke(Calls.Count);
             return Task.FromResult(_factory != null ? _factory() : _responses.Dequeue());
         }
     }