feat: extra_headers support in Client constructor and config

README.md

@@ -103,6 +103,35 @@ c.archive('archive-collection', request, r[0]['location'])
 c.revoke('all')
 ```
 
+### Extra HTTP headers
+
+The Python client can attach additional HTTP headers to requests. This is intended only for non-secret, safe metadata/debug headers; do not use it for credentials, cookies, tokens, or other sensitive values.
+
+Extra headers can be supplied directly to the constructor:
+
+```python
+from polytope.api import Client
+
+c = Client(extra_headers={"Polytope-Mock-Roles": "beta:viewer"})
+```
+
+They can also be stored in the client configuration file:
+
+```yaml
+extra_headers:
+  Polytope-Mock-Roles: beta:viewer
+```
+
+or provided with the `POLYTOPE_EXTRA_HEADERS` environment variable. The environment variable value must be a JSON object string mapping header names to values:
+
+```bash
+export POLYTOPE_EXTRA_HEADERS='{"Polytope-Mock-Roles":"beta:viewer"}'
+```
+
+Administrators can use this with server-side mocking/debug features, for example to test role-dependent behaviour with `Polytope-Mock-Roles: beta:viewer`.
+
+Unsafe or request-controlled headers are rejected case-insensitively. This includes authentication headers, cookies, hop-by-hop/protocol headers, content/range/checksum headers, and proxy/attribution IP headers. Blocked examples include `Cookie`, `Set-Cookie`, `X-Forwarded-For`, `X-Real-IP`, `Forwarded`, and `X-Proxy-Protocol-Addr`.
+
  
 ## 4. CLI example
 

docs/source/client/python_api.rst

@@ -8,3 +8,33 @@ A Python API for user-friendly interaction with a Polytope server is available a
 Once the Python module is installed, it can either be used directly in Python scripts and sessions (i.e. ``from polytope import api``), or be used via the command-line tool that is installed together with the module (i.e. ``which polytope``).
 
 More details on installation and usage can be found in the ``readme.md`` file in the source of `polytope-client <https://github.com/ecmwf-projects/polytope-client>`_, as well as in the documentation of the Python module (e.g. ``help(api)``) and in the documentation of the command-line tool (e.g. ``polytope --help``).
+
+Extra HTTP headers
+------------------
+
+The Python client can attach additional HTTP headers to each request. Extra headers are intended only for non-secret, safe metadata/debug headers; do not use them for credentials, cookies, tokens, or other sensitive values.
+
+Constructor usage:
+
+.. code-block:: python
+
+   from polytope.api import Client
+
+   c = Client(extra_headers={"Polytope-Mock-Roles": "beta:viewer"})
+
+Configuration file usage:
+
+.. code-block:: yaml
+
+   extra_headers:
+     Polytope-Mock-Roles: beta:viewer
+
+Environment variable usage. The value of ``POLYTOPE_EXTRA_HEADERS`` must be a JSON object string mapping header names to values:
+
+.. code-block:: bash
+
+   export POLYTOPE_EXTRA_HEADERS='{"Polytope-Mock-Roles":"beta:viewer"}'
+
+Administrators can use this with server-side mocking/debug features, for example to test role-dependent behaviour with ``Polytope-Mock-Roles: beta:viewer``.
+
+Unsafe or request-controlled headers are rejected case-insensitively. This includes authentication headers, cookies, hop-by-hop/protocol headers, content/range/checksum headers, and proxy/attribution IP headers. Blocked examples include ``Cookie``, ``Set-Cookie``, ``X-Forwarded-For``, ``X-Real-IP``, ``Forwarded``, and ``X-Proxy-Protocol-Addr``.

polytope/api/Admin.py

@@ -46,7 +46,9 @@ def describe_user(self):
         situation = "trying to describe a user"
 
         url = self.config.get_url("users")
-        headers = {"Content-Type": "application/json", "Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers(
+            {"Content-Type": "application/json", "Authorization": ", ".join(self.auth.get_auth_headers())}
+        )
         method = "get"
         expected_responses = [requests.codes.ok]
         response, response_messages = helpers.try_request(
@@ -87,6 +89,7 @@ def ping(self):
             expected=expected_responses,
             logger=self._logger,
             url=url,
+            headers=self.config.request_headers(),
             skip_tls=self.config.get()["skip_tls"],
         )
         message = "The Polytope server is operating and accessible."

polytope/api/Auth.py

@@ -111,7 +111,9 @@ def login(self, username=None, password=None, persist=True, key_type="bearer"):
 
         url = self.config.get_url("auth")
         encode_str = "%s:%s" % (username, password)
-        headers = {"Authorization": "Basic %s" % base64.b64encode(bytes(encode_str, "utf-8")).decode("utf-8")}
+        headers = self.config.request_headers(
+            {"Authorization": "Basic %s" % base64.b64encode(bytes(encode_str, "utf-8")).decode("utf-8")}
+        )
         data = {}
         method = "post"
         expected_responses = [requests.codes.ok]

polytope/api/Client.py

@@ -57,6 +57,7 @@ def __init__(
         # https
         insecure=None,
         skip_tls=None,
+        extra_headers=None,
         # other
         cli=False,
     ):
@@ -101,6 +102,8 @@ def __init__(
         :type insecure: bool
         :param skip_tls: Skip TLS certificate verification.
         :type skip_tls: bool
+        :param extra_headers: Additional safe HTTP headers to send with requests.
+        :type extra_headers: dict
         :param cli: Whether the Client is being created from a CLI or not
         (configured automatically). This will determine whether some messages
         are printed or not by the client.
@@ -129,6 +132,7 @@ def __init__(
             password,
             insecure=insecure,
             skip_tls=skip_tls,
+            extra_headers=extra_headers,
             logger=self._logger,
             cli=self._cli,
         )

polytope/api/CollectionVisitor.py

@@ -44,7 +44,7 @@ def list(self):
 
         self._logger.info("Fetching collections...")
         url = self.config.get_url("collections")
-        headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
         method = "get"
         expected_responses = [requests.codes.ok]
         response, _ = helpers.try_request(

polytope/api/Config.py

@@ -48,6 +48,7 @@ def __init__(
         skip_tls=None,
         logger=None,
         cli=False,
+        extra_headers=None,
     ):
         # hard-coded defaults are not specified in the __init__ header
         # so that session configuration specified in the headers is not
@@ -82,6 +83,7 @@ def __init__(
             "password",
             "insecure",
             "skip_tls",
+            "extra_headers",
         ]
 
         # Reading session configuration
@@ -108,6 +110,7 @@ def __init__(
         config["password"] = None
         config["insecure"] = False
         config["skip_tls"] = False
+        config["extra_headers"] = {}
         self.default_config = config
 
         # Reading system-wide file configuration
@@ -133,6 +136,8 @@ def fun(x):
         for var in self.file_config_items:
             val = os.environ.get(fun(var))
             if val:
+                if var == "extra_headers":
+                    val = helpers.parse_extra_headers_json(val)
                 env_var_config[var] = val
         self.env_var_config = env_var_config
 
@@ -237,8 +242,20 @@ def get(self):
             if isinstance(config[item], str):
                 config[item] = config[item].lower() in ["true", "1"]
 
+        config["extra_headers"] = helpers.normalize_extra_headers(config.get("extra_headers", {}))
+
         return config
 
+    def request_headers(self, base=None):
+        headers = {} if base is None else dict(base)
+        base_names = {str(name).lower() for name in headers}
+        extra_headers = helpers.normalize_extra_headers(self.get().get("extra_headers", {}))
+        for name, value in extra_headers.items():
+            if name.lower() in base_names:
+                raise ValueError("Extra header duplicates base request header: " + name)
+            headers[name] = value
+        return headers
+
     def update_loggers(self):
         config_dict = self.get()
 
@@ -427,6 +444,11 @@ def set(self, key, value, persist=False):
         print_value = value
         if key == "password" and value:
             print_value = "**hidden**"
+        if key == "extra_headers":
+            if isinstance(value, str):
+                value = helpers.parse_extra_headers_json(value)
+            value = helpers.normalize_extra_headers(value)
+            print_value = "**hidden**" if value else {}
 
         self.session_config[key] = value
         self.update_loggers()

polytope/api/RequestManager.py

@@ -63,7 +63,7 @@ def list(self, collection_id=None):
 
         self._logger.info("Fetching requests...")
         url = self.config.get_url("requests", collection_id=collection_id)
-        headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
         method = "get"
         expected_responses = [requests.codes.ok]
         response, _ = helpers.try_request(
@@ -105,7 +105,7 @@ def describe(self, request_id):
 
         self._logger.info("Fetching request...")
         url = self.config.get_url("requests")
-        headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
         method = "get"
         expected_responses = [requests.codes.ok]
         response, response_messages = helpers.try_request(
@@ -146,7 +146,7 @@ def revoke(self, request_id):
         :returns: None
         """
 
-        headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
 
         response, messages = helpers.try_request(
             method="delete",
@@ -345,7 +345,7 @@ def retrieve(
             self._logger.info(message)
 
             url = self.config.get_url("requests", collection_id=collection)
-            headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+            headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
             method = "post"
             expected_responses = [requests.codes.ok, requests.codes.accepted, requests.codes.no_content]
             # also requests.codes.other, implicitly handled by requests
@@ -666,7 +666,7 @@ def download(
             request_id = url.split("/")[-1]
         else:
             url = self.config.get_url("requests", request_id=request_id)
-        headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
         method = "get"
         expected_responses = [requests.codes.ok, requests.codes.accepted]
         # requests will handle automatically requests.codes.other
@@ -846,7 +846,7 @@ def archive(
         self._logger.info(message)
 
         url = self.config.get_url("upload", collection_id=collection)
-        headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
         method = "post"
         expected_responses = [requests.codes.accepted, requests.codes.other]
         # requests.codes.other is handled explicitly here (allow_redirects = False)
@@ -935,11 +935,13 @@ def upload(self, request_id, input_file=None, asynchronous=False, max_attempts=N
             data_checksum = hashlib.md5(data).hexdigest()
 
             method = "post"
-            headers = {
-                "Content-Type": "application/x-grib",
-                "Authorization": ", ".join(self.auth.get_auth_headers()),
-                "X-Checksum": data_checksum,
-            }
+            headers = self.config.request_headers(
+                {
+                    "Content-Type": "application/x-grib",
+                    "Authorization": ", ".join(self.auth.get_auth_headers()),
+                    "X-Checksum": data_checksum,
+                }
+            )
 
             start = time.time()
             data_uploaded = False
@@ -985,7 +987,7 @@ def upload(self, request_id, input_file=None, asynchronous=False, max_attempts=N
 
         situation = "waiting for the server to list the uploaded data"
         self._logger.info("Waiting for the server to list the uploaded data...")
-        headers = {"Authorization": ", ".join(self.auth.get_auth_headers())}
+        headers = self.config.request_headers({"Authorization": ", ".join(self.auth.get_auth_headers())})
         method = "get"
 
         data_ready = False

polytope/api/config_schema.json

@@ -15,6 +15,12 @@
         "log_level": {"type": "string"},
         "user_key": {"type": "string"},
         "user_email": {"type": "string"},
-        "password": {"type": "string"}
+        "password": {"type": "string"},
+        "extra_headers": {
+            "type": "object",
+            "additionalProperties": {
+                "type": ["string", "number", "integer", "boolean", "null"]
+            }
+        }
     }
 }