Merge pull request #6 from erkiaas/wildduck-operator-retry-api

wildduck-operator: retry on Wildduck API connection errors instead of crashing

Author: erkiaas

samples/wildduck-operator/wildduck_operator.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
-from operatorlib import ClaimMixin, Operator, ReconcileError
+from operatorlib import ClaimMixin, MonitoringMixin, Operator, ReconcileError
 import aiohttp
+import asyncio
 import os
 
 
-class WildduckOperator(ClaimMixin, Operator):
+class WildduckOperator(ClaimMixin, MonitoringMixin, Operator):
     OPERATOR = "wildduck-operator"
     GROUP = "codemowers.cloud"
     VERSION = "v1beta1"
@@ -23,6 +24,21 @@ def get_claim_plural(cls):
 
     @classmethod
     async def reconcile_claim(cls, api_client, co, body):
+        # Wildduck API being temporarily unreachable (pod restart, node
+        # reboot) is expected; retry instead of letting the connection
+        # error propagate and kill the whole operator process
+        backoff = 5
+        while True:
+            try:
+                return await cls.reconcile_wildduck_user(api_client, co, body)
+            except (aiohttp.ClientConnectionError, asyncio.TimeoutError) as e:
+                print("Wildduck API unreachable, retrying in %d seconds: %s" % (
+                    backoff, e))
+                await asyncio.sleep(backoff)
+                backoff = min(backoff * 2, 300)
+
+    @classmethod
+    async def reconcile_wildduck_user(cls, api_client, co, body):
         username = body["metadata"]["name"]
 
         # TODO: Cleaner env var handling