diff --git a/README.md b/README.md index de73df6..3128268 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,35 @@ c.archive('archive-collection', request, r[0]['location']) c.revoke('all') ``` +### Extra HTTP headers + +The Python client can attach additional HTTP headers to requests. This is intended only for non-secret, safe metadata/debug headers; do not use it for credentials, cookies, tokens, or other sensitive values. + +Extra headers can be supplied directly to the constructor: + +```python +from polytope.api import Client + +c = Client(extra_headers={"Polytope-Mock-Roles": "beta:viewer"}) +``` + +They can also be stored in the client configuration file: + +```yaml +extra_headers: + Polytope-Mock-Roles: beta:viewer +``` + +or provided with the `POLYTOPE_EXTRA_HEADERS` environment variable. The environment variable value must be a JSON object string mapping header names to values: + +```bash +export POLYTOPE_EXTRA_HEADERS='{"Polytope-Mock-Roles":"beta:viewer"}' +``` + +Administrators can use this with server-side mocking/debug features, for example to test role-dependent behaviour with `Polytope-Mock-Roles: beta:viewer`. + +Unsafe or request-controlled headers are rejected case-insensitively. This includes authentication headers, cookies, hop-by-hop/protocol headers, content/range/checksum headers, and proxy/attribution IP headers. Blocked examples include `Cookie`, `Set-Cookie`, `X-Forwarded-For`, `X-Real-IP`, `Forwarded`, and `X-Proxy-Protocol-Addr`. +   ## 4. CLI example diff --git a/docs/source/client/python_api.rst b/docs/source/client/python_api.rst index 82e2379..436ebcd 100644 --- a/docs/source/client/python_api.rst +++ b/docs/source/client/python_api.rst @@ -8,3 +8,33 @@ A Python API for user-friendly interaction with a Polytope server is available a Once the Python module is installed, it can either be used directly in Python scripts and sessions (i.e. ``from polytope import api``), or be used via the command-line tool that is installed together with the module (i.e. ``which polytope``). More details on installation and usage can be found in the ``readme.md`` file in the source of `polytope-client `_, as well as in the documentation of the Python module (e.g. ``help(api)``) and in the documentation of the command-line tool (e.g. ``polytope --help``). + +Extra HTTP headers +------------------ + +The Python client can attach additional HTTP headers to each request. Extra headers are intended only for non-secret, safe metadata/debug headers; do not use them for credentials, cookies, tokens, or other sensitive values. + +Constructor usage: + +.. code-block:: python + + from polytope.api import Client + + c = Client(extra_headers={"Polytope-Mock-Roles": "beta:viewer"}) + +Configuration file usage: + +.. code-block:: yaml + + extra_headers: + Polytope-Mock-Roles: beta:viewer + +Environment variable usage. The value of ``POLYTOPE_EXTRA_HEADERS`` must be a JSON object string mapping header names to values: + +.. code-block:: bash + + export POLYTOPE_EXTRA_HEADERS='{"Polytope-Mock-Roles":"beta:viewer"}' + +Administrators can use this with server-side mocking/debug features, for example to test role-dependent behaviour with ``Polytope-Mock-Roles: beta:viewer``. + +Unsafe or request-controlled headers are rejected case-insensitively. This includes authentication headers, cookies, hop-by-hop/protocol headers, content/range/checksum headers, and proxy/attribution IP headers. Blocked examples include ``Cookie``, ``Set-Cookie``, ``X-Forwarded-For``, ``X-Real-IP``, ``Forwarded``, and ``X-Proxy-Protocol-Addr``. diff --git a/polytope/api/Admin.py b/polytope/api/Admin.py index 500f417..8543ed9 100644 --- a/polytope/api/Admin.py +++ b/polytope/api/Admin.py @@ -46,7 +46,9 @@ def describe_user(self): situation = "trying to describe a user" url = self.config.get_url("users") - headers = {"Content-Type": "application/json", "Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers( + {"Content-Type": "application/json", "Authorization": ", ".join(self.auth.get_auth_headers())} + ) method = "get" expected_responses = [requests.codes.ok] response, response_messages = helpers.try_request( @@ -87,6 +89,7 @@ def ping(self): expected=expected_responses, logger=self._logger, url=url, + headers=self.config.request_headers(), skip_tls=self.config.get()["skip_tls"], ) message = "The Polytope server is operating and accessible." diff --git a/polytope/api/Auth.py b/polytope/api/Auth.py index 1277da2..15eaa07 100644 --- a/polytope/api/Auth.py +++ b/polytope/api/Auth.py @@ -111,7 +111,9 @@ def login(self, username=None, password=None, persist=True, key_type="bearer"): url = self.config.get_url("auth") encode_str = "%s:%s" % (username, password) - headers = {"Authorization": "Basic %s" % base64.b64encode(bytes(encode_str, "utf-8")).decode("utf-8")} + headers = self.config.request_headers( + {"Authorization": "Basic %s" % base64.b64encode(bytes(encode_str, "utf-8")).decode("utf-8")} + ) data = {} method = "post" expected_responses = [requests.codes.ok] diff --git a/polytope/api/Client.py b/polytope/api/Client.py index a840d91..4bf1cb7 100644 --- a/polytope/api/Client.py +++ b/polytope/api/Client.py @@ -57,6 +57,7 @@ def __init__( # https insecure=None, skip_tls=None, + extra_headers=None, # other cli=False, ): @@ -101,6 +102,8 @@ def __init__( :type insecure: bool :param skip_tls: Skip TLS certificate verification. :type skip_tls: bool + :param extra_headers: Additional safe HTTP headers to send with requests. + :type extra_headers: dict :param cli: Whether the Client is being created from a CLI or not (configured automatically). This will determine whether some messages are printed or not by the client. @@ -129,6 +132,7 @@ def __init__( password, insecure=insecure, skip_tls=skip_tls, + extra_headers=extra_headers, logger=self._logger, cli=self._cli, ) diff --git a/polytope/api/CollectionVisitor.py b/polytope/api/CollectionVisitor.py index 96b7fdb..5481b25 100644 --- a/polytope/api/CollectionVisitor.py +++ b/polytope/api/CollectionVisitor.py @@ -44,7 +44,7 @@ def list(self): self._logger.info("Fetching collections...") url = self.config.get_url("collections") - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) method = "get" expected_responses = [requests.codes.ok] response, _ = helpers.try_request( diff --git a/polytope/api/Config.py b/polytope/api/Config.py index f4d1491..9079322 100644 --- a/polytope/api/Config.py +++ b/polytope/api/Config.py @@ -48,6 +48,7 @@ def __init__( skip_tls=None, logger=None, cli=False, + extra_headers=None, ): # hard-coded defaults are not specified in the __init__ header # so that session configuration specified in the headers is not @@ -82,6 +83,7 @@ def __init__( "password", "insecure", "skip_tls", + "extra_headers", ] # Reading session configuration @@ -108,6 +110,7 @@ def __init__( config["password"] = None config["insecure"] = False config["skip_tls"] = False + config["extra_headers"] = {} self.default_config = config # Reading system-wide file configuration @@ -133,6 +136,8 @@ def fun(x): for var in self.file_config_items: val = os.environ.get(fun(var)) if val: + if var == "extra_headers": + val = helpers.parse_extra_headers_json(val) env_var_config[var] = val self.env_var_config = env_var_config @@ -237,8 +242,20 @@ def get(self): if isinstance(config[item], str): config[item] = config[item].lower() in ["true", "1"] + config["extra_headers"] = helpers.normalize_extra_headers(config.get("extra_headers", {})) + return config + def request_headers(self, base=None): + headers = {} if base is None else dict(base) + base_names = {str(name).lower() for name in headers} + extra_headers = helpers.normalize_extra_headers(self.get().get("extra_headers", {})) + for name, value in extra_headers.items(): + if name.lower() in base_names: + raise ValueError("Extra header duplicates base request header: " + name) + headers[name] = value + return headers + def update_loggers(self): config_dict = self.get() @@ -427,6 +444,11 @@ def set(self, key, value, persist=False): print_value = value if key == "password" and value: print_value = "**hidden**" + if key == "extra_headers": + if isinstance(value, str): + value = helpers.parse_extra_headers_json(value) + value = helpers.normalize_extra_headers(value) + print_value = "**hidden**" if value else {} self.session_config[key] = value self.update_loggers() diff --git a/polytope/api/RequestManager.py b/polytope/api/RequestManager.py index cb9f5f5..f5bc0fc 100644 --- a/polytope/api/RequestManager.py +++ b/polytope/api/RequestManager.py @@ -63,7 +63,7 @@ def list(self, collection_id=None): self._logger.info("Fetching requests...") url = self.config.get_url("requests", collection_id=collection_id) - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) method = "get" expected_responses = [requests.codes.ok] response, _ = helpers.try_request( @@ -105,7 +105,7 @@ def describe(self, request_id): self._logger.info("Fetching request...") url = self.config.get_url("requests") - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) method = "get" expected_responses = [requests.codes.ok] response, response_messages = helpers.try_request( @@ -146,7 +146,7 @@ def revoke(self, request_id): :returns: None """ - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) response, messages = helpers.try_request( method="delete", @@ -345,7 +345,7 @@ def retrieve( self._logger.info(message) url = self.config.get_url("requests", collection_id=collection) - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) method = "post" expected_responses = [requests.codes.ok, requests.codes.accepted, requests.codes.no_content] # also requests.codes.other, implicitly handled by requests @@ -666,7 +666,7 @@ def download( request_id = url.split("/")[-1] else: url = self.config.get_url("requests", request_id=request_id) - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) method = "get" expected_responses = [requests.codes.ok, requests.codes.accepted] # requests will handle automatically requests.codes.other @@ -846,7 +846,7 @@ def archive( self._logger.info(message) url = self.config.get_url("upload", collection_id=collection) - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) method = "post" expected_responses = [requests.codes.accepted, requests.codes.other] # requests.codes.other is handled explicitly here (allow_redirects = False) @@ -935,11 +935,13 @@ def upload(self, request_id, input_file=None, asynchronous=False, max_attempts=N data_checksum = hashlib.md5(data).hexdigest() method = "post" - headers = { - "Content-Type": "application/x-grib", - "Authorization": ", ".join(self.auth.get_auth_headers()), - "X-Checksum": data_checksum, - } + headers = self.config.request_headers( + { + "Content-Type": "application/x-grib", + "Authorization": ", ".join(self.auth.get_auth_headers()), + "X-Checksum": data_checksum, + } + ) start = time.time() data_uploaded = False @@ -985,7 +987,7 @@ def upload(self, request_id, input_file=None, asynchronous=False, max_attempts=N situation = "waiting for the server to list the uploaded data" self._logger.info("Waiting for the server to list the uploaded data...") - headers = {"Authorization": ", ".join(self.auth.get_auth_headers())} + headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())}) method = "get" data_ready = False diff --git a/polytope/api/config_schema.json b/polytope/api/config_schema.json index cf9db4b..ad6e8e0 100644 --- a/polytope/api/config_schema.json +++ b/polytope/api/config_schema.json @@ -15,6 +15,12 @@ "log_level": {"type": "string"}, "user_key": {"type": "string"}, "user_email": {"type": "string"}, - "password": {"type": "string"} + "password": {"type": "string"}, + "extra_headers": { + "type": "object", + "additionalProperties": { + "type": ["string", "number", "integer", "boolean", "null"] + } + } } } diff --git a/polytope/api/helpers.py b/polytope/api/helpers.py index 11aa130..3c70fb5 100644 --- a/polytope/api/helpers.py +++ b/polytope/api/helpers.py @@ -19,6 +19,7 @@ import json import logging import os +import re import socket import time from inspect import signature @@ -236,6 +237,88 @@ def decorated(self, *args, **kwargs): return decorated +# Header-related helper functions +### + +# RFC 9110 token syntax for field names. +_HEADER_NAME_RE = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") + +UNSAFE_EXTRA_HEADERS = frozenset( + name.lower() + for name in [ + "Authorization", + "Proxy-Authorization", + "Cookie", + "Set-Cookie", + "Host", + "Content-Length", + "Transfer-Encoding", + "Connection", + "Keep-Alive", + "TE", + "Trailer", + "Upgrade", + "Content-Type", + "Content-Encoding", + "Accept-Encoding", + "Range", + "X-Checksum", + "X-Forwarded-For", + "X-Real-IP", + "Forwarded", + "X-Proxy-Protocol-Addr", + ] +) + + +def parse_extra_headers_json(value): + def reject_duplicate_pairs(pairs): + seen = set() + result = {} + for key, item in pairs: + key_lower = str(key).lower() + if key_lower in seen: + raise ValueError("Duplicate extra header name: " + str(key)) + seen.add(key_lower) + result[key] = item + return result + + try: + return json.loads(value, object_pairs_hook=reject_duplicate_pairs) + except ValueError: + raise + except Exception as e: + raise ValueError("extra_headers must be a JSON object string") from e + + +def normalize_extra_headers(extra_headers, reject_unsafe=True): + if extra_headers in (None, ""): + return {} + if isinstance(extra_headers, str): + extra_headers = parse_extra_headers_json(extra_headers) + if not isinstance(extra_headers, dict): + raise ValueError("extra_headers must be a mapping of header names to values") + + normalized = {} + seen = set() + for name, value in extra_headers.items(): + if not isinstance(name, str): + raise ValueError("Extra header names must be strings") + if not _HEADER_NAME_RE.match(name): + raise ValueError("Invalid extra header name: " + name) + name_lower = name.lower() + if name_lower in seen: + raise ValueError("Duplicate extra header name: " + name) + seen.add(name_lower) + if reject_unsafe and name_lower in UNSAFE_EXTRA_HEADERS: + raise ValueError("Unsafe extra header is not allowed: " + name) + value = str(value) + if "\r" in value or "\n" in value: + raise ValueError("Extra header values must not contain CR or LF characters") + normalized[name] = value + return normalized + + # Logging-related helper functions ### @@ -274,12 +357,8 @@ def format(self, record): if isinstance(val, typ): result[name] = val else: - raise TypeError( - "Extra information with key {} is \ - expected to be of type {}".format( - name, typ - ) - ) + raise TypeError("Extra information with key {} is \ + expected to be of type {}".format(name, typ)) return json.dumps(result, indent=self.indent, sort_keys=True) @@ -415,6 +494,8 @@ def convert_back(dictionary): ) elif k == "key_path": dictionary[k] = os.path.expanduser(v) + elif k == "extra_headers": + dictionary[k] = normalize_extra_headers(v) def validate_config(dictionary): diff --git a/tests/unit/test_extra_headers.py b/tests/unit/test_extra_headers.py new file mode 100644 index 0000000..24bd028 --- /dev/null +++ b/tests/unit/test_extra_headers.py @@ -0,0 +1,189 @@ +import json + +import pytest + +from polytope.api import Client, helpers +from polytope.api.Config import Config + +SAFE_HEADER = "Polytope-Mock-Roles" +SAFE_VALUE = "beta:viewer" + + +def test_extra_headers_from_constructor_config_file_and_env(tmp_path, monkeypatch): + client = Client(config_path=tmp_path / "ctor", extra_headers={SAFE_HEADER: SAFE_VALUE}) + assert client.config.get()["extra_headers"] == {SAFE_HEADER: SAFE_VALUE} + + config_dir = tmp_path / "file" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("extra_headers:\n Polytope-Mock-Roles: beta:viewer\n") + config = Config(config_path=config_dir) + assert config.get()["extra_headers"] == {SAFE_HEADER: SAFE_VALUE} + + monkeypatch.setenv("POLYTOPE_EXTRA_HEADERS", json.dumps({SAFE_HEADER: SAFE_VALUE})) + config = Config(config_path=tmp_path / "env") + assert config.get()["extra_headers"] == {SAFE_HEADER: SAFE_VALUE} + + config = Config(config_path=tmp_path / "set") + config.set("extra_headers", json.dumps({SAFE_HEADER: SAFE_VALUE})) + assert config.get()["extra_headers"] == {SAFE_HEADER: SAFE_VALUE} + + config = Config(config_path=tmp_path / "set-dict") + config.set("extra_headers", {SAFE_HEADER: SAFE_VALUE}) + assert config.get()["extra_headers"] == {SAFE_HEADER: SAFE_VALUE} + + +def test_extra_header_validation_rejects_invalid_names_duplicates_and_crlf(tmp_path, monkeypatch): + with pytest.raises(ValueError): + Client(config_path=tmp_path / "bad-name", extra_headers={"Bad Header": "value"}) + + with pytest.raises(ValueError): + Client(config_path=tmp_path / "bad-value", extra_headers={"X-Debug": "line\nbreak"}) + + monkeypatch.setenv("POLYTOPE_EXTRA_HEADERS", '{"X-Debug":"one","x-debug":"two"}') + with pytest.raises(ValueError): + Config(config_path=tmp_path / "dup-env") + monkeypatch.delenv("POLYTOPE_EXTRA_HEADERS") + + config_dir = tmp_path / "dup-file" + config_dir.mkdir() + (config_dir / "config.yaml").write_text("extra_headers:\n X-Debug: one\n x-debug: two\n") + with pytest.raises(ValueError): + Config(config_path=config_dir) + + config = Config(config_path=tmp_path / "bad-set-dict") + with pytest.raises(ValueError): + config.set("extra_headers", {"Authorization": "Bearer secret"}) + + +def test_unsafe_extra_headers_blocked_case_insensitively(tmp_path): + blocked = [ + "Authorization", + "Proxy-Authorization", + "Cookie", + "Set-Cookie", + "Host", + "Content-Length", + "Transfer-Encoding", + "Connection", + "Keep-Alive", + "TE", + "Trailer", + "Upgrade", + "Content-Type", + "Content-Encoding", + "Accept-Encoding", + "Range", + "X-Checksum", + "X-Forwarded-For", + "X-Real-IP", + "Forwarded", + "X-Proxy-Protocol-Addr", + ] + for name in blocked: + variant = name.swapcase() + with pytest.raises(ValueError): + Client(config_path=tmp_path / (name.lower().replace("-", "_") or "header"), extra_headers={variant: "x"}) + + +def test_request_headers_preserves_base_headers_and_rejects_base_duplicates(tmp_path): + config = Config(config_path=tmp_path, extra_headers={SAFE_HEADER: SAFE_VALUE}) + headers = config.request_headers({"Authorization": "Bearer token", "Range": "bytes=10-"}) + assert headers["Authorization"] == "Bearer token" + assert headers["Range"] == "bytes=10-" + assert headers[SAFE_HEADER] == SAFE_VALUE + + config = Config(config_path=tmp_path / "dup", extra_headers={"x-custom": "extra"}) + with pytest.raises(ValueError): + config.request_headers({"X-Custom": "base"}) + + +def test_extra_headers_propagate_to_request_sites(monkeypatch, tmp_path): + captured = [] + + class Response: + def __init__(self, status_code=200, headers=None, body=None, url="https://example.test/result"): + self.status_code = status_code + self.headers = headers or {} + self._body = body or {"message": []} + self.url = url + self.content = b"" + + def json(self): + return self._body + + def close(self): + pass + + def fake_try_request(*args, **kwargs): + captured.append( + ( + kwargs.get("situation"), + kwargs.get("headers", {}), + kwargs.get("url"), + kwargs.get("method") or (args[0] if args else None), + ) + ) + situation = kwargs.get("situation") + if situation == "trying to list collections": + return Response(body={"message": ["ecmwf-mars"]}), {} + if situation in ("trying to list requests", "trying to describe a request"): + return Response(body={"message": [{"id": "req-1", "status": "queued", "verb": "retrieve"}]}), {} + if situation == "trying to submit a retrieval request": + return Response(status_code=202, headers={"Location": "https://example.test/requests/req-2"}), { + "message": "ok" + } + if situation == "trying to download data": + return Response(status_code=200, headers={"Content-Length": "0", "Content-Type": "application/x-grib"}), {} + if situation == "trying to revoke a request": + return Response(), {"message": "revoked"} + if situation == "trying to submit an archive request": + return Response(status_code=303, headers={"location": "https://example.test/uploads/up-1"}), { + "message": "archive" + } + if situation == "trying to upload data": + return Response(status_code=202, headers={"Location": "https://example.test/uploads/up-1"}), {} + raise AssertionError("unexpected situation " + str(situation)) + + monkeypatch.setattr(helpers, "try_request", fake_try_request) + + client = Client( + config_path=tmp_path, + address="http://example.test", + insecure=True, + user_key="token", + extra_headers={SAFE_HEADER: SAFE_VALUE}, + ) + + client.list_collections() + client.list_requests() + client.retrieve("ecmwf-mars", {"param": "t"}, asynchronous=True) + client.download("req-1", pointer=True) + client.revoke("req-1") + + input_file = tmp_path / "payload.grib" + input_file.write_bytes(b"data") + client.upload("https://example.test/uploads/up-1", input_file=str(input_file), asynchronous=True) + + monkeypatch.setattr(client.request_manager.coll_visitor, "list", lambda: ["ecmwf-mars"]) + monkeypatch.setattr(client.request_manager, "upload", lambda *args, **kwargs: "uploaded") + metadata_file = tmp_path / "metadata.yaml" + metadata_file.write_text("class: od\n") + client.archive("ecmwf-mars", str(metadata_file), "http://example.test/data.grib", inline_metadata=False) + + assert captured + for _, headers, _, _ in captured: + assert headers[SAFE_HEADER] == SAFE_VALUE + assert headers.get("Authorization") == "Bearer token" or "Authorization" not in headers + assert headers.get("Content-Type") != SAFE_VALUE + assert headers.get("Range") != SAFE_VALUE + assert headers.get("X-Checksum") != SAFE_VALUE + for forbidden in ["Cookie", "Set-Cookie", "X-Forwarded-For", "X-Real-IP", "Forwarded", "X-Proxy-Protocol-Addr"]: + assert forbidden not in headers + + situations = [item[0] for item in captured] + assert "trying to list collections" in situations + assert "trying to submit a retrieval request" in situations + assert "trying to download data" in situations + assert "trying to revoke a request" in situations + assert "trying to submit an archive request" in situations + assert "trying to upload data" in situations