diff --git a/README.md b/README.md index c670416..bb29db1 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ The bot token and the chat id live in SSM Parameter Store and are read once per - **At-least-once delivery**: send, then record. If `notify` crashes between the two steps, the score goes out twice rather than never. - **Idempotence**: the DynamoDB check absorbs the duplicates caused by retries. -- **Dead-letter queue**: an event that still fails after Lambda's retries lands in an SQS queue instead of disappearing. -- **Alerting**: CloudWatch alarms (a message in the DLQ, no result on a match day) notify me on Telegram through SNS. +- **Dead-letter queue**: an event that still fails after Lambda's retries, or that a schedule or the bus cannot deliver, lands in an SQS queue instead of disappearing. +- **Alerting**: CloudWatch alarms (a message in the DLQ, a match never found on the page, a planner that stopped running) email me through SNS, a channel that keeps working when Telegram does not. - **Traceability**: structured JSON logs carrying the match id, so one match can be followed across all Lambdas. - **Least privilege**: one IAM role per Lambda, limited to what it actually uses. @@ -53,13 +53,14 @@ TypeScript · AWS Lambda (Node.js, arm64) · EventBridge (Scheduler and event bu ## Status -Work in progress: the diagram shows the target architecture, built one step at a time. +Work in progress: the diagram shows the target architecture, built one step at a time; its dashed grey boxes are not built yet. - Done: TypeScript project, required pull request checks (lint, types, unit tests, commitlint, Gitleaks, CodeQL), Dependabot updates, an empty CDK app synthesized on each pull request with cdk-nag rules, GitHub OIDC roles, a `cdk diff` of each pull request and a `cdk deploy` on each merge to `main`. - Done: the Treize Mondial adapter, the event bus and the `fetch-results` Lambda, which publishes `MatchFinished` once a match is finished. - Done: the `notify` Lambda, subscribed to `MatchFinished`: it checks DynamoDB, sends the score to Telegram, then records it. - Done: the `planner` Lambda, run every day at 10:00 (Paris time): it schedules one `fetch-results` check per match, about 2 hours after kick-off and never at night. When the score is not published yet, `fetch-results` checks again later, with growing delays, up to 7 checks. -- Next: the DLQ, the alarms, the smoke test and the `prod` alias rollback. +- Done: the dead-letter queue, fed by every Lambda and every schedule or rule that invokes one, and three CloudWatch alarms that email me through SNS. +- Next: the smoke test and the `prod` alias rollback. Progress is tracked in the [v1 milestone](https://github.com/TomSoulage/final-whistle/milestone/1). diff --git a/docs/architecture.excalidraw b/docs/architecture.excalidraw index b1870e4..3aadc25 100644 --- a/docs/architecture.excalidraw +++ b/docs/architecture.excalidraw @@ -94,7 +94,7 @@ "width": 190, "height": 90, "angle": 0, - "strokeColor": "#1e1e1e", + "strokeColor": "#868e96", "backgroundColor": "#f8f9fa", "fillStyle": "solid", "strokeWidth": 1, @@ -109,11 +109,11 @@ "type": 3 }, "seed": 970302523, - "version": 2, - "versionNonce": 1575296583, + "version": 3, + "versionNonce": 910000007, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, "index": "a2", @@ -861,15 +861,15 @@ "frameId": null, "roundness": null, "seed": 873550681, - "version": 2, - "versionNonce": 1351827847, + "version": 3, + "versionNonce": 910000002, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, - "text": "every morning", - "originalText": "every morning", + "text": "every day, 10:00 Paris", + "originalText": "every day, 10:00 Paris", "fontSize": 13, "fontFamily": 2, "textAlign": "center", @@ -1168,7 +1168,7 @@ "x": 576, "y": 316, "width": 188, - "height": 32.5, + "height": 48.75, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", @@ -1183,15 +1183,15 @@ "frameId": null, "roundness": null, "seed": 1170241, - "version": 2, - "versionNonce": 1326301673, + "version": 3, + "versionNonce": 910000005, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, - "text": "MatchFinished\n{ competition, teams, score }", - "originalText": "MatchFinished\n{ competition, teams, score }", + "text": "MatchFinished\n{ matchId, competition,\ndate, teams, score }", + "originalText": "MatchFinished\n{ matchId, competition,\ndate, teams, score }", "fontSize": 13, "fontFamily": 2, "textAlign": "center", @@ -1689,15 +1689,15 @@ "frameId": null, "roundness": null, "seed": 1464750149, - "version": 150, - "versionNonce": 978301577, + "version": 151, + "versionNonce": 910000003, "isDeleted": false, "boundElements": [], - "updated": 1793452996300, + "updated": 1793544430244, "link": null, "locked": false, - "text": "bot token", - "originalText": "bot token", + "text": "bot token + chat id", + "originalText": "bot token + chat id", "fontSize": 13, "fontFamily": 2, "textAlign": "center", @@ -1835,7 +1835,7 @@ "x": 856, "y": 646, "width": 178, - "height": 16.25, + "height": 32.5, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", @@ -1850,15 +1850,15 @@ "frameId": null, "roundness": null, "seed": 250199007, - "version": 2, - "versionNonce": 774801641, + "version": 3, + "versionNonce": 900000001, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793541377872, "link": null, "locked": false, - "text": "failed deliveries", - "originalText": "failed deliveries", + "text": "failures of every Lambda,\nschedule and rule", + "originalText": "failures of every Lambda,\nschedule and rule", "fontSize": 13, "fontFamily": 2, "textAlign": "center", @@ -2016,14 +2016,14 @@ "frameId": null, "roundness": null, "seed": 2142981915, - "version": 132, - "versionNonce": 1777883847, + "version": 134, + "versionNonce": 900000002, "isDeleted": false, "boundElements": [], - "updated": 1793439766953, + "updated": 1793546918845, "locked": false, - "text": "JSON logs · matchId\nin every line\nalarms: DLQ,\nno results on match day", - "originalText": "JSON logs · matchId\nin every line\nalarms: DLQ,\nno results on match day", + "text": "JSON logs · matchId\nin every line\nalarms: DLQ, match not\nfound, planner not run", + "originalText": "JSON logs · matchId\nin every line\nalarms: DLQ, match not\nfound, planner not run", "fontSize": 13, "fontFamily": 2, "textAlign": "center", @@ -2034,7 +2034,7 @@ "index": "ap", "created": null, "hasTextLink": false, - "rawText": "JSON logs · matchId\nin every line\nalarms: DLQ,\nno results on match day", + "rawText": "JSON logs · matchId\nin every line\nalarms: DLQ, match not\nfound, planner not run", "labelPosition": null, "baseFontSize": null }, @@ -2714,9 +2714,9 @@ { "type": "text", "id": "3a775505", - "x": 512, + "x": 540, "y": 388, - "width": 107.52000000000001, + "width": 150, "height": 30, "angle": 0, "strokeColor": "#c92a2a", @@ -2730,15 +2730,15 @@ "frameId": null, "roundness": null, "seed": 1756216313, - "version": 2, - "versionNonce": 1522360679, + "version": 4, + "versionNonce": 910000001, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, - "text": "reschedules +1h\nif score missing", - "originalText": "reschedules +1h\nif score missing", + "text": "reschedules with growing\ndelays, up to 7 checks", + "originalText": "reschedules with growing\ndelays, up to 7 checks", "fontSize": 12, "fontFamily": 2, "textAlign": "left", @@ -3399,8 +3399,8 @@ "id": "9d42f670", "x": 1278.290450555534, "y": 639.6868370300052, - "width": 256.7095494444659, - "height": 321.1934675634957, + "width": 182, + "height": 0, "angle": 0, "strokeColor": "#c92a2a", "backgroundColor": "transparent", @@ -3413,11 +3413,11 @@ "frameId": null, "roundness": null, "seed": 159214593, - "version": 436, - "versionNonce": 788744457, + "version": 437, + "versionNonce": 900000003, "isDeleted": false, "boundElements": [], - "updated": 1793439789106, + "updated": 1793541377872, "link": null, "locked": false, "points": [ @@ -3426,12 +3426,8 @@ 0 ], [ - 255.12144226093915, - 1.5066305334904655 - ], - [ - 256.7095494444659, - -319.6868370300052 + 182, + 0 ] ], "startArrowhead": null, @@ -3454,9 +3450,9 @@ { "type": "text", "id": "365fdcd6", - "x": 1436.9543865526439, - "y": 615.6019422718024, - "width": 36.400000000000006, + "x": 1300, + "y": 618, + "width": 110, "height": 16.25, "angle": 0, "strokeColor": "#c92a2a", @@ -3470,15 +3466,15 @@ "frameId": null, "roundness": null, "seed": 1970015974, - "version": 49, - "versionNonce": 641348105, + "version": 50, + "versionNonce": 900000004, "isDeleted": false, "boundElements": [], - "updated": 1793439778774, + "updated": 1793541377872, "link": null, "locked": false, - "text": "alert", - "originalText": "alert", + "text": "alert (SNS)", + "originalText": "alert (SNS)", "fontSize": 13, "fontFamily": 2, "textAlign": "left", @@ -4181,14 +4177,14 @@ "frameId": null, "roundness": null, "seed": 973005002, - "version": 66, - "versionNonce": 455172775, + "version": 68, + "versionNonce": 910000006, "isDeleted": false, "boundElements": [], - "updated": 1793452555353, + "updated": 1793546918845, "locked": false, - "text": "✓ lint · types · unit tests (parsers)\n✓ cdk synth + cdk-nag (infra security rules)\n✓ cdk diff posted on the PR (AWS changes)\n✓ CodeQL (code scanning)\n✓ Gitleaks (secret scanning)", - "originalText": "✓ lint · types · unit tests (parsers)\n✓ cdk synth + cdk-nag (infra security rules)\n✓ cdk diff posted on the PR (AWS changes)\n✓ CodeQL (code scanning)\n✓ Gitleaks (secret scanning)", + "text": "✓ lint · types · unit tests\n✓ cdk synth + cdk-nag (infra security rules)\n✓ cdk diff in the job summary (AWS changes)\n✓ commitlint (PR title + commits)\n✓ CodeQL · Gitleaks (code + secret scanning)", + "originalText": "✓ lint · types · unit tests\n✓ cdk synth + cdk-nag (infra security rules)\n✓ cdk diff in the job summary (AWS changes)\n✓ commitlint (PR title + commits)\n✓ CodeQL · Gitleaks (code + secret scanning)", "fontSize": 13, "fontFamily": 2, "textAlign": "left", @@ -4199,7 +4195,7 @@ "index": "b0g", "created": null, "hasTextLink": false, - "rawText": "✓ lint · types · unit tests (parsers)\n✓ cdk synth + cdk-nag (infra security rules)\n✓ cdk diff posted on the PR (AWS changes)\n✓ CodeQL (code scanning)\n✓ Gitleaks (secret scanning)", + "rawText": "✓ lint · types · unit tests\n✓ cdk synth + cdk-nag (infra security rules)\n✓ cdk diff in the job summary (AWS changes)\n✓ commitlint (PR title + commits)\n✓ CodeQL · Gitleaks (code + secret scanning)", "labelPosition": null, "baseFontSize": null }, @@ -5466,7 +5462,7 @@ "x": 596, "y": 1196, "width": 188, - "height": 32.5, + "height": 48.75, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", @@ -5481,15 +5477,15 @@ "frameId": null, "roundness": null, "seed": 1914968990, - "version": 2, - "versionNonce": 2130947945, + "version": 3, + "versionNonce": 910000015, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, - "text": "infra changes\n+ new Lambda versions", - "originalText": "infra changes\n+ new Lambda versions", + "text": "infra changes\n(new Lambda versions:\nplanned)", + "originalText": "infra changes\n(new Lambda versions:\nplanned)", "fontSize": 13, "fontFamily": 2, "textAlign": "center", @@ -5512,11 +5508,11 @@ "width": 200, "height": 170, "angle": 0, - "strokeColor": "#1e1e1e", + "strokeColor": "#868e96", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1, - "strokeStyle": "solid", + "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [ @@ -5527,11 +5523,11 @@ "type": 3 }, "seed": 1430007344, - "version": 2, - "versionNonce": 668997543, + "version": 3, + "versionNonce": 910000008, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, "index": "b1C", @@ -5673,11 +5669,11 @@ "width": 260, "height": 75, "angle": 0, - "strokeColor": "#2b8a3e", + "strokeColor": "#868e96", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1, - "strokeStyle": "solid", + "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [ @@ -5688,11 +5684,11 @@ "type": 3 }, "seed": 471538974, - "version": 4, - "versionNonce": 164506121, + "version": 5, + "versionNonce": 910000009, "isDeleted": false, "boundElements": [], - "updated": 1793439041526, + "updated": 1793544430244, "link": null, "locked": false, "index": "b1G", @@ -5794,11 +5790,11 @@ "width": 260, "height": 95, "angle": 0, - "strokeColor": "#c92a2a", + "strokeColor": "#868e96", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1, - "strokeStyle": "solid", + "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [ @@ -5809,11 +5805,11 @@ "type": 3 }, "seed": 1923399979, - "version": 2, - "versionNonce": 439366377, + "version": 3, + "versionNonce": 910000010, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, "index": "b1J", @@ -5884,15 +5880,15 @@ "frameId": null, "roundness": null, "seed": 2105307110, - "version": 2, - "versionNonce": 1901850057, + "version": 3, + "versionNonce": 910000016, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, - "text": "prod alias stays on the\nprevious version + Telegram alert", - "originalText": "prod alias stays on the\nprevious version + Telegram alert", + "text": "prod alias stays on the\nprevious version + email alert", + "originalText": "prod alias stays on the\nprevious version + email alert", "fontSize": 13, "fontFamily": 2, "textAlign": "center", @@ -6050,18 +6046,18 @@ "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, - "strokeStyle": "solid", + "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1343752580, - "version": 2, - "versionNonce": 1559220105, + "version": 3, + "versionNonce": 910000011, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, "points": [ @@ -6095,18 +6091,18 @@ "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, - "strokeStyle": "solid", + "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1443425336, - "version": 2, - "versionNonce": 106793351, + "version": 3, + "versionNonce": 910000012, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, "points": [ @@ -6190,18 +6186,18 @@ "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, - "strokeStyle": "solid", + "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1341696988, - "version": 2, - "versionNonce": 287359143, + "version": 3, + "versionNonce": 910000013, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, "points": [ @@ -6285,18 +6281,18 @@ "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, - "strokeStyle": "solid", + "strokeStyle": "dashed", "roughness": 0, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 597266123, - "version": 2, - "versionNonce": 1219498951, + "version": 3, + "versionNonce": 910000014, "isDeleted": false, "boundElements": [], - "updated": 1793438840569, + "updated": 1793544430244, "link": null, "locked": false, "points": [ @@ -6444,11 +6440,11 @@ "width": 94.77341199882187, "height": 91.30962976345916, "angle": 0, - "strokeColor": "#adb5bd", + "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "dotted", + "strokeWidth": 1.5, + "strokeStyle": "solid", "roughness": 0, "opacity": 100, "groupIds": [], @@ -6456,11 +6452,11 @@ "index": "b1c", "roundness": null, "seed": 1925914217, - "version": 425, - "versionNonce": 1471493609, + "version": 427, + "versionNonce": 930000001, "isDeleted": false, "boundElements": null, - "updated": 1793452996300, + "updated": 1793546455973, "created": 1793452894331, "link": null, "locked": false, @@ -6510,7 +6506,7 @@ "width": 98.22978764857275, "height": 32.5, "angle": 0, - "strokeColor": "#adb5bd", + "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, @@ -6521,14 +6517,14 @@ "frameId": null, "roundness": null, "seed": 1158499431, - "version": 359, - "versionNonce": 1120716807, + "version": 362, + "versionNonce": 920000002, "isDeleted": false, "boundElements": [], - "updated": 1793452998050, + "updated": 1793546918845, "locked": false, - "text": "token, read once\nat cold start", - "originalText": "token, read once at cold start", + "text": "read once\nper instance", + "originalText": "read once\nper instance", "fontSize": 13, "fontFamily": 2, "textAlign": "left", @@ -6539,7 +6535,7 @@ "index": "b1e", "created": 1793452963179, "hasTextLink": false, - "rawText": "token, read once at cold start", + "rawText": "read once\nper instance", "labelPosition": null, "baseFontSize": null }, @@ -6640,6 +6636,362 @@ "rawText": "3 - remember (if sent)", "labelPosition": null, "baseFontSize": null + }, + { + "angle": 0, + "backgroundColor": "#ffffff", + "fillStyle": "solid", + "roughness": 0, + "opacity": 100, + "frameId": null, + "isDeleted": false, + "boundElements": [], + "updated": 1793541377872, + "link": null, + "locked": false, + "version": 1, + "created": null, + "hasTextLink": false, + "type": "rectangle", + "id": "email-box-v2", + "index": "b1i", + "x": 1460, + "y": 590, + "width": 150, + "height": 100, + "strokeColor": "#1e1e1e", + "strokeWidth": 1, + "strokeStyle": "solid", + "groupIds": [ + "email0v2" + ], + "roundness": { + "type": 3 + }, + "seed": 900000005, + "versionNonce": 900000006 + }, + { + "angle": 0, + "backgroundColor": "transparent", + "fillStyle": "solid", + "roughness": 0, + "opacity": 100, + "frameId": null, + "isDeleted": false, + "boundElements": [], + "updated": 1793541377872, + "link": null, + "locked": false, + "version": 1, + "created": null, + "hasTextLink": false, + "type": "text", + "id": "email-title-v2", + "index": "b1j", + "x": 1466, + "y": 612, + "width": 138, + "height": 20, + "strokeColor": "#1e1e1e", + "strokeWidth": 1, + "strokeStyle": "solid", + "groupIds": [ + "email0v2" + ], + "roundness": null, + "seed": 900000007, + "versionNonce": 900000008, + "text": "Email", + "originalText": "Email", + "fontSize": 16, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": false, + "rawText": "", + "labelPosition": null, + "baseFontSize": null + }, + { + "angle": 0, + "backgroundColor": "transparent", + "fillStyle": "solid", + "roughness": 0, + "opacity": 100, + "frameId": null, + "isDeleted": false, + "boundElements": [], + "updated": 1793541377872, + "link": null, + "locked": false, + "version": 1, + "created": null, + "hasTextLink": false, + "type": "text", + "id": "email-sub-v2", + "index": "b1k", + "x": 1466, + "y": 636, + "width": 138, + "height": 32.5, + "strokeColor": "#495057", + "strokeWidth": 1, + "strokeStyle": "solid", + "groupIds": [ + "email0v2" + ], + "roundness": null, + "seed": 900000009, + "versionNonce": 900000010, + "text": "alerts, independent\nof Telegram", + "originalText": "alerts, independent\nof Telegram", + "fontSize": 13, + "fontFamily": 2, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": false, + "rawText": "", + "labelPosition": null, + "baseFontSize": null + }, + { + "angle": 0, + "backgroundColor": "transparent", + "fillStyle": "solid", + "roughness": 0, + "opacity": 100, + "frameId": null, + "isDeleted": false, + "boundElements": [], + "updated": 1793541377872, + "link": null, + "locked": false, + "version": 1, + "created": null, + "hasTextLink": false, + "type": "arrow", + "id": "planner-dlq-v2", + "index": "b1l", + "x": 760, + "y": 640, + "width": 90, + "height": 0, + "strokeColor": "#c92a2a", + "strokeWidth": 1.5, + "strokeStyle": "dashed", + "groupIds": [], + "roundness": null, + "seed": 900000011, + "versionNonce": 900000012, + "points": [ + [ + 0, + 0 + ], + [ + 90, + 0 + ] + ], + "startArrowhead": null, + "endArrowhead": "arrow", + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "moveMidPointsWithElement": false + }, + { + "angle": 0, + "backgroundColor": "transparent", + "fillStyle": "solid", + "roughness": 0, + "opacity": 100, + "frameId": null, + "isDeleted": false, + "boundElements": [], + "updated": 1793541377872, + "link": null, + "locked": false, + "version": 1, + "created": null, + "hasTextLink": false, + "type": "text", + "id": "planner-dlq-label-v2", + "index": "b1m", + "x": 785, + "y": 620, + "width": 40, + "height": 16.25, + "strokeColor": "#c92a2a", + "strokeWidth": 1, + "strokeStyle": "solid", + "groupIds": [], + "roundness": null, + "seed": 900000013, + "versionNonce": 900000014, + "text": "fails", + "originalText": "fails", + "fontSize": 13, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": false, + "rawText": "", + "labelPosition": null, + "baseFontSize": null + }, + { + "type": "text", + "id": "legend-planned-v2", + "index": "b1n", + "x": 1260, + "y": 64, + "width": 340, + "height": 16.25, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "roundness": null, + "seed": 910000017, + "version": 1, + "versionNonce": 910000018, + "isDeleted": false, + "boundElements": [], + "updated": 1793544430244, + "link": null, + "locked": false, + "text": "dashed grey box = planned, not built yet", + "originalText": "dashed grey box = planned, not built yet", + "fontSize": 13, + "fontFamily": 2, + "textAlign": "right", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": false, + "created": null, + "hasTextLink": false, + "rawText": "", + "labelPosition": null, + "baseFontSize": null + }, + { + "angle": 0, + "backgroundColor": "transparent", + "fillStyle": "solid", + "roughness": 0, + "opacity": 100, + "frameId": null, + "isDeleted": false, + "boundElements": [], + "updated": 1793546308915, + "link": null, + "locked": false, + "version": 2, + "created": null, + "hasTextLink": false, + "type": "arrow", + "id": "fetch-dlq-v2", + "index": "b1o", + "x": 500, + "y": 368, + "width": 350, + "height": 232, + "strokeColor": "#c92a2a", + "strokeWidth": 1.5, + "strokeStyle": "dashed", + "groupIds": [], + "roundness": null, + "seed": 920000003, + "versionNonce": 920000004, + "points": [ + [ + 0, + 0 + ], + [ + 25, + 0 + ], + [ + 25, + 77 + ], + [ + 320, + 77 + ], + [ + 320, + 232 + ], + [ + 350, + 232 + ] + ], + "startArrowhead": null, + "endArrowhead": "arrow", + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "moveMidPointsWithElement": false + }, + { + "angle": 0, + "backgroundColor": "transparent", + "fillStyle": "solid", + "roughness": 0, + "opacity": 100, + "frameId": null, + "isDeleted": false, + "boundElements": [], + "updated": 1793546308915, + "link": null, + "locked": false, + "version": 1, + "created": null, + "hasTextLink": false, + "type": "text", + "id": "fetch-dlq-label-v2", + "index": "b1p", + "x": 690, + "y": 449, + "width": 40, + "height": 16.25, + "strokeColor": "#c92a2a", + "strokeWidth": 1, + "strokeStyle": "solid", + "groupIds": [], + "roundness": null, + "seed": 920000005, + "versionNonce": 920000006, + "text": "fails", + "originalText": "fails", + "fontSize": 13, + "fontFamily": 2, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "lineHeight": 1.25, + "autoResize": false, + "rawText": "", + "labelPosition": null, + "baseFontSize": null } ], "appState": { @@ -6803,4 +7155,4 @@ "lastRetrieved": 1793438932524 } } -} \ No newline at end of file +} diff --git a/docs/architecture.png b/docs/architecture.png index 9945905..6a06208 100644 Binary files a/docs/architecture.png and b/docs/architecture.png differ diff --git a/docs/decisions.md b/docs/decisions.md index c373b84..b463516 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -31,7 +31,7 @@ Log of the project's structural choices. Each decision has a stable identifier ( - **D6. An event bus between detection and notification.** `fetch-results` publishes a `MatchFinished` event instead of calling `notify` directly. The producer does not know its consumers: a new one (history, website rebuild) is a new rule on the bus, with no change to the producer (fan-out). Each consumer handles its own failures. Cost: one more hop, and custom events are billed ($1 per million, a fraction of a cent here). - **D7. At-least-once delivery: send, then record.** `notify` checks DynamoDB, sends to Telegram, then records the score. A crash between sending and recording means a duplicate message, never a lost score. A rare duplicate is accepted. - **D8. Idempotence key**: the page date and both teams, e.g. `2026-10-30|Angleterre|Papouasie-Nouvelle-Guinée`. Team names are kept exactly as published. -- **D9. Failed events go to a DLQ.** After Lambda's retries, a failing event lands in an SQS dead-letter queue, watched by an alarm, instead of being lost silently. +- **D9. Failed events go to a DLQ.** After Lambda's retries, a failing event lands in an SQS dead-letter queue, watched by an alarm, instead of being lost silently. _Detailed in D37._ - **D28. The `MatchFinished` contract.** `fetch-results` takes the match check `{ date, homeTeam, awayTeam, attempt }` (the key of D8 and the attempt number of D35, from 1) as input and validates it first: a malformed check fails at once instead of being looked up and rescheduled. A finished match is published on the `final-whistle` bus as `MatchFinished` (source `final-whistle.fetch-results`); its detail is the normalised result plus its match id, so consumers never rebuild the key. The bus is created with its producer and each consumer adds its own rule (D6). Source, type and detail are defined once in `src/events`, for the producer and the rules. PutEvents reports a rejected entry without throwing, so a rejection is turned into an error: the invocation fails and is retried instead of losing the score. - **D30. `notify`: check, send, remember.** `notify` is subscribed to `MatchFinished` by its own rule and validates the event detail first, against the contract in `src/events` (D28). It then follows D7: - **Check**: a strongly consistent read of the match id in the `SentMatches` table, so a score recorded a moment ago is always seen. @@ -40,6 +40,16 @@ Log of the project's structural choices. Each decision has a stable identifier ( - **Table**: provisioned at 1 read and 1 write unit (D10), kept if the stack is deleted and protected against deletion. Point-in-time recovery is not enabled (cdk-nag `DDB3` acknowledged on the table): it is billed (D17), and losing the table costs at worst one score sent twice, never a lost score. - **Telegram parameters**: `/final-whistle/telegram/bot-token` (SecureString) and `/final-whistle/telegram/chat-id` (String), created by hand before the first deployment, since CloudFormation cannot create a SecureString. They are read together in one `GetParameters` call on the first send of each Lambda instance and kept in memory; SSM reports a missing parameter without throwing, so it is turned into an error naming it. They are not resolved at deployment, because CloudFormation would read them with the CDK deploy role, which may only read the bootstrap version. The role may only call `ssm:GetParameters` on these two parameters, with no KMS permission: the AWS-managed key lets SSM decrypt for the account's roles. - **No token in the logs**: the bot token is part of the Telegram URL, so a failed request is reported by the match, the error name and its system code only (e.g. `TypeError ECONNRESET`); the original error, which may quote the URL, is never kept. A rejection keeps the HTTP status and Telegram's `description`. +- **D37. One failure queue for every asynchronous step** (details D9). Nothing in the pipeline waits for an answer, so a failure is never seen by a caller: each step that can drop an event sends it to the same SQS queue instead. + - **Lambdas**: `planner`, `fetch-results` and `notify` send an invocation that still fails after Lambda's 2 retries to the queue as an on-failure destination, rather than a plain dead-letter queue: the message holds the original event, the error and the function, so one queue serves the three Lambdas and a failure is understood without searching the logs. The destination is set on `$LATEST`, the version invoked today; once the schedules and the rule invoke the `prod` alias (D15), it has to move to the alias. + - **Launchers**: the morning schedule, the match check schedules and the `MatchFinished` rule send what they cannot deliver to their Lambda (a permission removed or a function renamed by a broken deployment) to the same queue, after their own retries (up to 24 h). Each one may only send to that queue: Scheduler through its invoke role, EventBridge through the queue policy, limited to the rule. Such a message holds the original event, with the reason in its attributes. + - **Queue**: 14 days of retention, the longest SQS allows, encryption by SQS (free) and HTTPS only. It has no dead-letter queue of its own (cdk-nag `SQS3` acknowledged): it is the last stop, watched by an alarm (D39). A failure is replayed by hand. +- **D38. Alerts by email through SNS.** The alarms publish on the `final-whistle-alerts` SNS topic, which sends an email. Telegram was the first choice, but an alert must not depend on what it watches: when Telegram or the bot token fails, `notify` fails, and an alert sent through Telegram would fail with it. Email needs no code (no Lambda to relay the alert) and stays free (1,000 emails a month). The subscription is created by hand once the topic exists, so the address never enters the public repository or the `cdk diff` of a pull request. The topic accepts HTTPS only and is not encrypted at rest (cdk-nag `SNS2` acknowledged): CloudWatch alarms cannot publish on a topic encrypted with the AWS-managed key, a customer-managed key is billed (D17), and the topic only carries alarm states. +- **D39. Alarms on failures of the system, not on missing scores.** A postponed match is not a problem; a pipeline that silently stops working is. Three alarms, each notifying the topic when it goes to ALARM: + - **Message sent to the failure queue**: any message arriving in the queue of D37, which covers every error the Lambdas rethrow (D29) and every event a launcher could not deliver. The alarm counts arrivals (`NumberOfMessagesSent` over 5 minutes), not the messages waiting: a message left in the queue would keep an alarm on its level in ALARM for up to 14 days and hide every later failure. Each new failure raises a new alert, and the alarm returns to OK on its own. + - **Match not found**: a match given up while still not found on the page (`match given up` with `status: "not-found"`, counted by a log metric filter). `planner` found it that morning, so `fetch-results` failing to find it for 7 attempts points to a lookup problem (a team name edited on the page, a page moved). A match given up as `not-finished` raises nothing. Assumption, not observed yet (no saved page has a postponed or cancelled match): a postponed match stays on the page without a score, so it ends as `not-finished`. If Treize Mondial removes its row instead, this alarm fires on a postponement, and it is revisited then. + - **Planner not run**: no `planner` invocation for 26 hours (hourly invocation counts, an hour without data counting as none). The planner runs every day, even without a match, every 24 h (25 h when the clocks go back), so silence means its schedule no longer fires, which leaves no error to catch. + - **Cost**: 3 of the 10 free alarms and 1 of the 10 free custom metrics. ### Data and secrets diff --git a/infra/failure-alarms.ts b/infra/failure-alarms.ts new file mode 100644 index 0000000..0efe5df --- /dev/null +++ b/infra/failure-alarms.ts @@ -0,0 +1,141 @@ +import { Duration, Stack, Validations } from 'aws-cdk-lib'; +import { Alarm, ComparisonOperator, type IMetric, Stats, TreatMissingData } from 'aws-cdk-lib/aws-cloudwatch'; +import { SnsAction } from 'aws-cdk-lib/aws-cloudwatch-actions'; +import { ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import type { IFunction } from 'aws-cdk-lib/aws-lambda'; +import { FilterPattern, type ILogGroup, MetricFilter } from 'aws-cdk-lib/aws-logs'; +import { Topic } from 'aws-cdk-lib/aws-sns'; +import type { IQueue } from 'aws-cdk-lib/aws-sqs'; +import { Construct } from 'constructs'; +import { MATCH_GIVEN_UP_MESSAGE, type MatchStatus } from '../src/fetch-results/check-match.js'; + +/** The email subscription is added by hand once the topic exists, so the address never enters the public repo (D38). */ +export const ALERT_TOPIC_NAME = 'final-whistle-alerts'; +const ALARM_NAME_PREFIX = 'final-whistle-'; +const METRIC_NAMESPACE = 'FinalWhistle'; +const MATCH_NOT_FOUND_METRIC = 'MatchGivenUpNotFound'; +/** SQS publishes its queue metrics every few minutes. */ +const FAILURE_QUEUE_PERIOD_MINUTES = 5; +const FAILURE_QUEUE_PERIOD = Duration.minutes(FAILURE_QUEUE_PERIOD_MINUTES); +const MATCH_NOT_FOUND_PERIOD = Duration.hours(1); +const PLANNER_RUN_PERIOD = Duration.hours(1); +/** The planner runs every 24 h (25 h when the clocks go back): 26 empty hours mean it did not run. */ +const PLANNER_SILENT_HOURS = 26; +/** A match still not found when given up; `not-finished` is a postponed or abandoned match, not a failure. */ +const NOT_FOUND_STATUS: MatchStatus = 'not-found'; + +export interface FailureAlarmsProps { + readonly failureQueue: IQueue; + /** The log group of fetch-results, where a match given up is logged. */ + readonly matchCheckLogGroup: ILogGroup; + readonly planner: IFunction; +} + +interface AlarmDefinition { + readonly id: string; + readonly name: string; + readonly description: string; + readonly metric: IMetric; + readonly comparisonOperator: ComparisonOperator; + readonly evaluationPeriods: number; + readonly treatMissingData: TreatMissingData; +} + +/** + * The alarms of the pipeline (D39): they watch for a failure of the system, never for a missing score, + * and notify the alert topic (D38). + */ +export class FailureAlarms extends Construct { + private readonly topic: Topic; + + constructor(scope: Construct, id: string, props: FailureAlarmsProps) { + super(scope, id); + this.topic = createAlertTopic(this); + this.addFailureQueueAlarm(props.failureQueue); + this.addMatchNotFoundAlarm(props.matchCheckLogGroup); + this.addPlannerNotRunAlarm(props.planner); + } + + /** + * Counts arrivals rather than the messages waiting: each new failure raises a new alert, even while + * an earlier one is still in the queue, and the alarm returns to OK on its own. + */ + private addFailureQueueAlarm(failureQueue: IQueue): void { + this.addAlarm({ + id: 'FailureQueueAlarm', + name: 'failure-queue-message-sent', + description: 'A failed invocation or an undelivered event was sent to the failure queue.', + metric: failureQueue.metricNumberOfMessagesSent({ period: FAILURE_QUEUE_PERIOD, statistic: Stats.SUM }), + comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + evaluationPeriods: 1, + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + } + + /** A match the planner found but fetch-results never found again on the page: most likely a lookup problem (D39). */ + private addMatchNotFoundAlarm(logGroup: ILogGroup): void { + const filter = new MetricFilter(this, 'MatchNotFoundFilter', { + logGroup, + filterPattern: FilterPattern.all( + FilterPattern.stringValue('$.message', '=', MATCH_GIVEN_UP_MESSAGE), + FilterPattern.stringValue('$.status', '=', NOT_FOUND_STATUS), + ), + metricNamespace: METRIC_NAMESPACE, + metricName: MATCH_NOT_FOUND_METRIC, + metricValue: '1', + }); + this.addAlarm({ + id: 'MatchNotFoundAlarm', + name: 'match-not-found', + description: 'A match was given up because it was never found on the source page.', + metric: filter.metric({ period: MATCH_NOT_FOUND_PERIOD, statistic: Stats.SUM }), + comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + evaluationPeriods: 1, + treatMissingData: TreatMissingData.NOT_BREACHING, + }); + } + + /** No run at all leaves no error to catch: an hour without invocations counts as breaching. */ + private addPlannerNotRunAlarm(planner: IFunction): void { + this.addAlarm({ + id: 'PlannerNotRunAlarm', + name: 'planner-not-run', + description: `The planner has not run for ${String(PLANNER_SILENT_HOURS)} hours.`, + metric: planner.metricInvocations({ period: PLANNER_RUN_PERIOD, statistic: Stats.SUM }), + comparisonOperator: ComparisonOperator.LESS_THAN_THRESHOLD, + evaluationPeriods: PLANNER_SILENT_HOURS, + treatMissingData: TreatMissingData.BREACHING, + }); + } + + /** Every alarm fires at a threshold of 1 and notifies the topic when it goes to ALARM. */ + private addAlarm(definition: AlarmDefinition): void { + const alarm = new Alarm(this, definition.id, { + alarmName: `${ALARM_NAME_PREFIX}${definition.name}`, + alarmDescription: definition.description, + metric: definition.metric, + threshold: 1, + comparisonOperator: definition.comparisonOperator, + evaluationPeriods: definition.evaluationPeriods, + treatMissingData: definition.treatMissingData, + }); + alarm.addAlarmAction(new SnsAction(this.topic)); + } +} + +/** HTTPS only; CloudWatch alarms of this account may publish on it. */ +function createAlertTopic(scope: Construct): Topic { + const topic = new Topic(scope, 'AlertTopic', { topicName: ALERT_TOPIC_NAME, enforceSSL: true }); + topic.grantPublish( + new ServicePrincipal('cloudwatch.amazonaws.com', { + conditions: { StringEquals: { 'aws:SourceAccount': Stack.of(scope).account } }, + }), + ); + Validations.of(topic).acknowledge({ + id: 'AwsSolutions::AwsSolutions-SNS2', + reason: + 'CloudWatch alarms cannot publish on a topic encrypted with the AWS-managed SNS key, and a customer-managed KMS key ' + + 'is billed (D17). The topic only carries alarm states, no secret and no score.', + }); + return topic; +} diff --git a/infra/failure-queue.ts b/infra/failure-queue.ts new file mode 100644 index 0000000..af9e5d1 --- /dev/null +++ b/infra/failure-queue.ts @@ -0,0 +1,23 @@ +import { Duration, Validations } from 'aws-cdk-lib'; +import { Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs'; +import type { Construct } from 'constructs'; + +/** The longest SQS allows: a failure can wait a few days before it is looked at. */ +const RETENTION_DAYS = 14; + +/** + * The dead-letter queue of the pipeline (D9, D37): what a Lambda still fails after its retries, and what + * a schedule or the bus rule cannot deliver to its Lambda, lands here instead of being dropped. + */ +export function createFailureQueue(scope: Construct): Queue { + const queue = new Queue(scope, 'FailureQueue', { + retentionPeriod: Duration.days(RETENTION_DAYS), + encryption: QueueEncryption.SQS_MANAGED, + enforceSSL: true, + }); + Validations.of(queue).acknowledge({ + id: 'AwsSolutions::AwsSolutions-SQS3', + reason: 'This queue is the dead-letter queue of the pipeline; it is watched by an alarm, not drained by a consumer.', + }); + return queue; +} diff --git a/infra/fetch-results-function.ts b/infra/fetch-results-function.ts index ca1e06d..791d45e 100644 --- a/infra/fetch-results-function.ts +++ b/infra/fetch-results-function.ts @@ -1,5 +1,6 @@ import { fileURLToPath } from 'node:url'; import type { EventBus } from 'aws-cdk-lib/aws-events'; +import type { IQueue } from 'aws-cdk-lib/aws-sqs'; import type { Construct } from 'constructs'; import { EVENT_BUS_NAME_VARIABLE } from '../src/fetch-results/environment.js'; import { PipelineFunction } from './pipeline-function.js'; @@ -13,6 +14,7 @@ const MEMORY_SIZE_MB = 256; export interface FetchResultsFunctionProps { readonly eventBus: EventBus; + readonly failureQueue: IQueue; } /** The fetch-results Lambda: on top of its logs, it may only publish on the bus (and schedule its next attempt, see the stack). */ @@ -24,6 +26,7 @@ export class FetchResultsFunction extends PipelineFunction { timeoutSeconds: TIMEOUT_SECONDS, memorySizeMb: MEMORY_SIZE_MB, environment: { [EVENT_BUS_NAME_VARIABLE]: props.eventBus.eventBusName }, + failureQueue: props.failureQueue, }); props.eventBus.grantPutEventsTo(this.lambdaFunction); } diff --git a/infra/final-whistle-stack.ts b/infra/final-whistle-stack.ts index 02841a2..d680c7f 100644 --- a/infra/final-whistle-stack.ts +++ b/infra/final-whistle-stack.ts @@ -1,6 +1,8 @@ import { Stack, type StackProps } from 'aws-cdk-lib'; import { EventBus } from 'aws-cdk-lib/aws-events'; import type { Construct } from 'constructs'; +import { FailureAlarms } from './failure-alarms.js'; +import { createFailureQueue } from './failure-queue.js'; import { FetchResultsFunction } from './fetch-results-function.js'; import { MatchCheckSchedules } from './match-check-schedules.js'; import { NotifyFunction } from './notify-function.js'; @@ -13,13 +15,22 @@ export const EVENT_BUS_NAME = 'final-whistle'; export class FinalWhistleStack extends Stack { constructor(scope: Construct, id: string, props?: StackProps) { super(scope, id, props); + const failureQueue = createFailureQueue(this); // Created with its producer; consumers subscribe with their own rules (D6). const eventBus = new EventBus(this, 'EventBus', { eventBusName: EVENT_BUS_NAME }); - const fetchResults = new FetchResultsFunction(this, 'FetchResults', { eventBus }); - const matchChecks = new MatchCheckSchedules(this, 'MatchChecks', { checkFunction: fetchResults.lambdaFunction }); + const fetchResults = new FetchResultsFunction(this, 'FetchResults', { eventBus, failureQueue }); + const matchChecks = new MatchCheckSchedules(this, 'MatchChecks', { + checkFunction: fetchResults.lambdaFunction, + failureQueue, + }); // A check that finds no score schedules the next attempt itself (D35). matchChecks.allowScheduleChecks(fetchResults); - new PlannerFunction(this, 'Planner', { matchChecks }); - new NotifyFunction(this, 'Notify', { eventBus, sentMatches: createSentMatchesTable(this) }); + const planner = new PlannerFunction(this, 'Planner', { matchChecks, failureQueue }); + new NotifyFunction(this, 'Notify', { eventBus, sentMatches: createSentMatchesTable(this), failureQueue }); + new FailureAlarms(this, 'Alarms', { + failureQueue, + matchCheckLogGroup: fetchResults.logGroup, + planner: planner.lambdaFunction, + }); } } diff --git a/infra/match-check-schedules.ts b/infra/match-check-schedules.ts index dc7dcbd..03059a3 100644 --- a/infra/match-check-schedules.ts +++ b/infra/match-check-schedules.ts @@ -2,8 +2,13 @@ import { ArnFormat, Stack, Validations } from 'aws-cdk-lib'; import { PolicyStatement, type Role } from 'aws-cdk-lib/aws-iam'; import type { IFunction } from 'aws-cdk-lib/aws-lambda'; import { ScheduleGroup } from 'aws-cdk-lib/aws-scheduler'; +import type { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; -import { MATCH_CHECK_GROUP_VARIABLE, MATCH_CHECK_ROLE_VARIABLE } from '../src/scheduling/environment.js'; +import { + MATCH_CHECK_DEAD_LETTER_QUEUE_VARIABLE, + MATCH_CHECK_GROUP_VARIABLE, + MATCH_CHECK_ROLE_VARIABLE, +} from '../src/scheduling/environment.js'; import type { PipelineFunction } from './pipeline-function.js'; import { createSchedulerInvokeRole, SCHEDULER_SERVICE } from './scheduler-invoke-role.js'; @@ -12,16 +17,20 @@ const MATCH_CHECK_GROUP_NAME = 'final-whistle-match-checks'; export interface MatchCheckSchedulesProps { /** The fetch-results function each match check invokes. */ readonly checkFunction: IFunction; + /** Where Scheduler sends a match check it cannot invoke fetch-results with (D37). */ + readonly failureQueue: IQueue; } /** - * The one-time match checks (D4): their schedule group, and the role Scheduler invokes fetch-results with. + * The one-time match checks (D4): their schedule group, the role Scheduler invokes fetch-results with, + * and the dead-letter queue it sends a check to when that invocation fails. * Lambdas allowed to schedule checks may only create schedules in this group, with this role. */ export class MatchCheckSchedules extends Construct { /** The fetch-results function each match check invokes. */ readonly checkFunctionArn: string; private readonly invokeRole: Role; + private readonly failureQueueArn: string; constructor(scope: Construct, id: string, props: MatchCheckSchedulesProps) { super(scope, id); @@ -29,22 +38,27 @@ export class MatchCheckSchedules extends Construct { this.invokeRole = createSchedulerInvokeRole( this, 'InvokeRole', - 'EventBridge Scheduler: invoke fetch-results for a match check', + 'EventBridge Scheduler: invoke fetch-results for a match check, or send the check to the failure queue', ); this.invokeRole.addToPolicy( new PolicyStatement({ actions: ['lambda:InvokeFunction'], resources: [props.checkFunction.functionArn] }), ); + this.invokeRole.addToPolicy( + new PolicyStatement({ actions: ['sqs:SendMessage'], resources: [props.failureQueue.queueArn] }), + ); this.checkFunctionArn = props.checkFunction.functionArn; + this.failureQueueArn = props.failureQueue.queueArn; } /** - * Lets a Lambda schedule match checks: the group and invoke role go into its environment, and it may + * Lets a Lambda schedule match checks: the group, invoke role and queue go into its environment, and it may * only create schedules in that group and pass that role to Scheduler. */ allowScheduleChecks(pipelineFunction: PipelineFunction): void { const { lambdaFunction } = pipelineFunction; lambdaFunction.addEnvironment(MATCH_CHECK_GROUP_VARIABLE, MATCH_CHECK_GROUP_NAME); lambdaFunction.addEnvironment(MATCH_CHECK_ROLE_VARIABLE, this.invokeRole.roleArn); + lambdaFunction.addEnvironment(MATCH_CHECK_DEAD_LETTER_QUEUE_VARIABLE, this.failureQueueArn); lambdaFunction.addToRolePolicy( new PolicyStatement({ actions: ['scheduler:CreateSchedule'], resources: [this.getSchedulesInGroupArn()] }), ); diff --git a/infra/notify-function.ts b/infra/notify-function.ts index efbd057..466fb4d 100644 --- a/infra/notify-function.ts +++ b/infra/notify-function.ts @@ -4,6 +4,7 @@ import type { Table } from 'aws-cdk-lib/aws-dynamodb'; import { type EventBus, Rule } from 'aws-cdk-lib/aws-events'; import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets'; import { PolicyStatement } from 'aws-cdk-lib/aws-iam'; +import type { IQueue } from 'aws-cdk-lib/aws-sqs'; import type { Construct } from 'constructs'; import { MATCH_FINISHED_DETAIL_TYPE, MATCH_FINISHED_SOURCE } from '../src/events/match-finished.js'; import { @@ -28,6 +29,7 @@ export const TELEGRAM_PARAMETER_NAMES = { export interface NotifyFunctionProps { readonly eventBus: EventBus; readonly sentMatches: Table; + readonly failureQueue: IQueue; } /** The notify Lambda, subscribed to MatchFinished by its own rule (D6), with access to its table and parameters only. */ @@ -43,13 +45,15 @@ export class NotifyFunction extends PipelineFunction { [BOT_TOKEN_PARAMETER_VARIABLE]: TELEGRAM_PARAMETER_NAMES.botToken, [CHAT_ID_PARAMETER_VARIABLE]: TELEGRAM_PARAMETER_NAMES.chatId, }, + failureQueue: props.failureQueue, }); this.allowSentMatchesAccess(props.sentMatches); this.allowTelegramParametersRead(); new Rule(this, 'MatchFinishedRule', { eventBus: props.eventBus, eventPattern: { source: [MATCH_FINISHED_SOURCE], detailType: [MATCH_FINISHED_DETAIL_TYPE] }, - targets: [new LambdaFunction(this.lambdaFunction)], + // An event the rule cannot deliver to notify goes to the failure queue (D37). + targets: [new LambdaFunction(this.lambdaFunction, { deadLetterQueue: props.failureQueue })], }); } diff --git a/infra/pipeline-function.ts b/infra/pipeline-function.ts index 22c83f8..d292e08 100644 --- a/infra/pipeline-function.ts +++ b/infra/pipeline-function.ts @@ -1,8 +1,10 @@ import { Duration } from 'aws-cdk-lib'; import { PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { Architecture, Runtime } from 'aws-cdk-lib/aws-lambda'; +import { SqsDestination } from 'aws-cdk-lib/aws-lambda-destinations'; import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs'; import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs'; +import type { IQueue } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; const LOG_RETENTION = RetentionDays.ONE_MONTH; @@ -15,18 +17,23 @@ export interface PipelineFunctionProps { readonly timeoutSeconds: number; readonly memorySizeMb: number; readonly environment: Readonly>; + /** Where an invocation that still fails after Lambda's retries is sent, with its error (D37). */ + readonly failureQueue: IQueue; } /** * A pipeline Lambda: Node.js 24 on arm64, its own log group with a retention, and its own role - * that can only write to that log group. Each Lambda adds the few permissions it needs (least privilege). + * that can only write to that log group and send its failures to the failure queue. Each Lambda adds the few + * permissions it needs (least privilege). */ export class PipelineFunction extends Construct { readonly lambdaFunction: NodejsFunction; + readonly logGroup: LogGroup; constructor(scope: Construct, id: string, props: PipelineFunctionProps) { super(scope, id); const logGroup = new LogGroup(this, 'LogGroup', { retention: LOG_RETENTION }); + this.logGroup = logGroup; const role = new Role(this, 'Role', { assumedBy: new ServicePrincipal('lambda.amazonaws.com'), description: props.roleDescription, @@ -47,6 +54,7 @@ export class PipelineFunction extends Construct { role, logGroup, environment: { ...props.environment }, + onFailure: new SqsDestination(props.failureQueue), }); } } diff --git a/infra/planner-function.ts b/infra/planner-function.ts index f21402e..738b569 100644 --- a/infra/planner-function.ts +++ b/infra/planner-function.ts @@ -3,6 +3,7 @@ import { Stack, TimeZone, Validations } from 'aws-cdk-lib'; import type { CfnFunction } from 'aws-cdk-lib/aws-lambda'; import { Schedule, ScheduleExpression } from 'aws-cdk-lib/aws-scheduler'; import { LambdaInvoke } from 'aws-cdk-lib/aws-scheduler-targets'; +import type { IQueue } from 'aws-cdk-lib/aws-sqs'; import type { Construct } from 'constructs'; import { MATCH_CHECK_FUNCTION_VARIABLE } from '../src/scheduling/environment.js'; import type { MatchCheckSchedules } from './match-check-schedules.js'; @@ -19,6 +20,7 @@ const PLANNER_HOUR = '10'; export interface PlannerFunctionProps { readonly matchChecks: MatchCheckSchedules; + readonly failureQueue: IQueue; } /** The planner Lambda, run every morning: it may only read the page and schedule match checks. */ @@ -30,18 +32,22 @@ export class PlannerFunction extends PipelineFunction { timeoutSeconds: TIMEOUT_SECONDS, memorySizeMb: MEMORY_SIZE_MB, environment: { [MATCH_CHECK_FUNCTION_VARIABLE]: props.matchChecks.checkFunctionArn }, + failureQueue: props.failureQueue, }); props.matchChecks.allowScheduleChecks(this); - this.addMorningSchedule(); + this.addMorningSchedule(props.failureQueue); } - /** In the default group, so the planner's rights on the match check group never reach its own trigger. */ - private addMorningSchedule(): void { + /** + * In the default group, so the planner's rights on the match check group never reach its own trigger. + * A run Scheduler cannot invoke goes to the failure queue (D37). + */ + private addMorningSchedule(failureQueue: IQueue): void { const role = createSchedulerInvokeRole(this, 'MorningScheduleRole', 'EventBridge Scheduler: run the planner'); new Schedule(this, 'MorningSchedule', { description: 'Run the planner every morning (Paris time)', schedule: ScheduleExpression.cron({ minute: '0', hour: PLANNER_HOUR, timeZone: TimeZone.EUROPE_PARIS }), - target: new LambdaInvoke(this.lambdaFunction, { role }), + target: new LambdaInvoke(this.lambdaFunction, { role, deadLetterQueue: failureQueue }), }); const functionLogicalId: unknown = Stack.of(this).resolve((this.lambdaFunction.node.defaultChild as CfnFunction).logicalId); Validations.of(role).acknowledge({ diff --git a/src/fetch-results/check-match.ts b/src/fetch-results/check-match.ts index dc3d3a8..0893aa5 100644 --- a/src/fetch-results/check-match.ts +++ b/src/fetch-results/check-match.ts @@ -6,6 +6,8 @@ import { planRetry } from '../scheduling/plan-check-time.js'; import { describeError, type LogFields, type Logger } from '../shared/logger.js'; const INPUT_NAME = 'fetch-results input'; +/** Logged once the attempts are used up; the match-not-found alarm counts it (D39). */ +export const MATCH_GIVEN_UP_MESSAGE = 'match given up'; export interface CheckMatchDependencies { readonly getMatchResult: (key: MatchKey) => Promise; @@ -51,7 +53,7 @@ async function reportLookup(check: MatchCheck, logger: Logger, dependencies: Che } const retry = await scheduleRetry(check, dependencies); if (retry === null) { - logger.warn('match given up', { attempt: check.attempt, status: lookup.status }); + logger.warn(MATCH_GIVEN_UP_MESSAGE, { attempt: check.attempt, status: lookup.status }); } else if (lookup.status === 'not-finished') { logger.info('match not finished yet', { attempt: check.attempt, ...retry }); } else { diff --git a/src/scheduling/environment.ts b/src/scheduling/environment.ts index 341abfd..ccdbeb5 100644 --- a/src/scheduling/environment.ts +++ b/src/scheduling/environment.ts @@ -5,8 +5,9 @@ import type { MatchCheckTarget } from './match-schedule.js'; export const MATCH_CHECK_GROUP_VARIABLE = 'MATCH_CHECK_SCHEDULE_GROUP'; export const MATCH_CHECK_FUNCTION_VARIABLE = 'MATCH_CHECK_FUNCTION_ARN'; export const MATCH_CHECK_ROLE_VARIABLE = 'MATCH_CHECK_ROLE_ARN'; +export const MATCH_CHECK_DEAD_LETTER_QUEUE_VARIABLE = 'MATCH_CHECK_DEAD_LETTER_QUEUE_ARN'; -/** The schedule group and invoke role of match checks; the function to invoke is given by each Lambda. */ +/** The schedule group, invoke role and dead-letter queue of match checks; the function to invoke is given by each Lambda. */ export type MatchCheckDestination = Omit; /** Read at cold start: a missing variable fails with its name. */ @@ -14,5 +15,6 @@ export function readMatchCheckDestination(): MatchCheckDestination { return { scheduleGroupName: getRequiredEnv(MATCH_CHECK_GROUP_VARIABLE), roleArn: getRequiredEnv(MATCH_CHECK_ROLE_VARIABLE), + deadLetterQueueArn: getRequiredEnv(MATCH_CHECK_DEAD_LETTER_QUEUE_VARIABLE), }; } diff --git a/src/scheduling/match-schedule.ts b/src/scheduling/match-schedule.ts index d8f2237..be7e773 100644 --- a/src/scheduling/match-schedule.ts +++ b/src/scheduling/match-schedule.ts @@ -14,11 +14,15 @@ const SCHEDULE_NAME_PREFIX = 'match-'; /** 128 bits of SHA-256: a collision between two matches is out of reach. */ const SCHEDULE_NAME_HASH_LENGTH = 32; -/** Where a match check goes: the schedule group, the fetch-results function and the role Scheduler invokes it with. */ +/** + * Where a match check goes: the schedule group, the fetch-results function, the role Scheduler invokes it with, + * and the queue Scheduler sends the check to when it cannot invoke the function (D37). + */ export interface MatchCheckTarget { readonly scheduleGroupName: string; readonly functionArn: string; readonly roleArn: string; + readonly deadLetterQueueArn: string; } export type SendCreateSchedule = (command: CreateScheduleCommand) => Promise; @@ -49,7 +53,12 @@ export function buildMatchCheckCommand( ScheduleExpressionTimezone: PARIS_TIME_ZONE, FlexibleTimeWindow: { Mode: FlexibleTimeWindowMode.OFF }, ActionAfterCompletion: ActionAfterCompletion.DELETE, - Target: { Arn: target.functionArn, RoleArn: target.roleArn, Input: JSON.stringify(check) }, + Target: { + Arn: target.functionArn, + RoleArn: target.roleArn, + Input: JSON.stringify(check), + DeadLetterConfig: { Arn: target.deadLetterQueueArn }, + }, }); } diff --git a/src/shared/read-once.ts b/src/shared/read-once.ts index d5666dc..f756601 100644 --- a/src/shared/read-once.ts +++ b/src/shared/read-once.ts @@ -1,6 +1,7 @@ /** * Runs `read` on the first call only and shares its result with every later call, e.g. a secret - * read at cold start. A failed read is forgotten, so the next call tries again. + * read on first use and kept for the life of the Lambda instance. A failed read is forgotten, so the + * next call tries again. */ export function readOnce(read: () => Promise): () => Promise { let pending: Promise | undefined; diff --git a/test/infra/final-whistle-stack.test.ts b/test/infra/final-whistle-stack.test.ts index e20eae8..577799e 100644 --- a/test/infra/final-whistle-stack.test.ts +++ b/test/infra/final-whistle-stack.test.ts @@ -40,6 +40,20 @@ const LOG_WRITE_STATEMENT = (constructId: string) => ({ Resource: { 'Fn::GetAtt': [getLogicalId('AWS::Logs::LogGroup', constructId), 'Arn'] }, }); +function getQueueArn(): unknown { + return { 'Fn::GetAtt': [getLogicalId('AWS::SQS::Queue'), 'Arn'] }; +} + +/** What Lambda needs to send a failed invocation to the failure queue, and nothing more. */ +const FAILURE_SEND_STATEMENT = () => ({ + Action: ['sqs:SendMessage', 'sqs:GetQueueAttributes', 'sqs:GetQueueUrl'], + Effect: 'Allow', + Resource: getQueueArn(), +}); + +/** What Scheduler needs to send an invocation it could not make to the failure queue. */ +const SCHEDULER_FAILURE_SEND_STATEMENT = () => ({ Action: 'sqs:SendMessage', Effect: 'Allow', Resource: getQueueArn() }); + /** Create schedules in the match check group only, and pass their role to Scheduler only. */ const SCHEDULE_CHECK_STATEMENTS = () => [ { @@ -99,6 +113,45 @@ describe('FinalWhistleStack', () => { }); }); +describe('failure queue', () => { + it('keeps failures for 14 days, encrypted by SQS, reachable over HTTPS only', () => { + template.resourceCountIs('AWS::SQS::Queue', 1); + template.hasResourceProperties('AWS::SQS::Queue', { MessageRetentionPeriod: 1_209_600, SqsManagedSseEnabled: true }); + template.hasResourceProperties('AWS::SQS::QueuePolicy', { + PolicyDocument: { + Statement: Match.arrayWith([ + Match.objectLike({ Effect: 'Deny', Condition: { Bool: { 'aws:SecureTransport': 'false' } } }), + ]), + }, + }); + }); + + it.each(['FetchResults', 'Notify', 'Planner'])('receives what %s still fails after its retries', (constructId) => { + template.hasResourceProperties('AWS::Lambda::EventInvokeConfig', { + FunctionName: { Ref: getLogicalId('AWS::Lambda::Function', constructId) }, + DestinationConfig: { OnFailure: { Destination: getQueueArn() } }, + }); + }); + + it('receives what the MatchFinished rule cannot deliver to notify, and only from that rule', () => { + template.hasResourceProperties('AWS::Events::Rule', { + Targets: [Match.objectLike({ DeadLetterConfig: { Arn: getQueueArn() } })], + }); + template.hasResourceProperties('AWS::SQS::QueuePolicy', { + PolicyDocument: { + Statement: Match.arrayWith([ + Match.objectLike({ + Action: 'sqs:SendMessage', + Effect: 'Allow', + Principal: { Service: 'events.amazonaws.com' }, + Condition: { ArnEquals: { 'aws:SourceArn': { 'Fn::GetAtt': [getLogicalId('AWS::Events::Rule'), 'Arn'] } } }, + }), + ]), + }, + }); + }); +}); + describe('fetch-results', () => { it('runs on Node.js 24, arm64, 256 MB, with the bus and where to schedule its next attempt in its environment', () => { expect(getProperties('AWS::Lambda::Function', 'FetchResults')).toMatchObject({ @@ -111,15 +164,17 @@ describe('fetch-results', () => { EVENT_BUS_NAME: { Ref: getLogicalId('AWS::Events::EventBus') }, MATCH_CHECK_SCHEDULE_GROUP: 'final-whistle-match-checks', MATCH_CHECK_ROLE_ARN: { 'Fn::GetAtt': [getLogicalId('AWS::IAM::Role', 'MatchChecks'), 'Arn'] }, + MATCH_CHECK_DEAD_LETTER_QUEUE_ARN: getQueueArn(), }, }, LoggingConfig: { LogGroup: { Ref: getLogicalId('AWS::Logs::LogGroup', 'FetchResults') } }, }); }); - it('may only write its logs, publish on its bus, and schedule its next attempt', () => { + it('may only write its logs, send its failures, publish on its bus, and schedule its next attempt', () => { expectPolicyStatements('FetchResults', [ LOG_WRITE_STATEMENT('FetchResults'), + FAILURE_SEND_STATEMENT(), { Action: 'events:PutEvents', Effect: 'Allow', @@ -148,9 +203,10 @@ describe('notify', () => { }); }); - it('may only write its logs, read and write one item of its table, and read the two Telegram parameters', () => { + it('may only write its logs, send its failures, read and write one item of its table, and read the two Telegram parameters', () => { expectPolicyStatements('Notify', [ LOG_WRITE_STATEMENT('Notify'), + FAILURE_SEND_STATEMENT(), { Action: ['dynamodb:GetItem', 'dynamodb:PutItem'], Effect: 'Allow', @@ -203,7 +259,7 @@ describe('match checks', () => { template.hasResourceProperties('AWS::Scheduler::ScheduleGroup', { Name: 'final-whistle-match-checks' }); }); - it('invoke fetch-results with a role that can do nothing else', () => { + it('invoke fetch-results with a role that may otherwise only send a check it could not deliver to the failure queue', () => { expect(getProperties('AWS::IAM::Role', 'MatchChecks')).toMatchObject({ AssumeRolePolicyDocument: SCHEDULER_TRUST }); expectPolicyStatements('MatchChecks', [ { @@ -211,6 +267,7 @@ describe('match checks', () => { Effect: 'Allow', Resource: { 'Fn::GetAtt': [getLogicalId('AWS::Lambda::Function', 'FetchResults'), 'Arn'] }, }, + SCHEDULER_FAILURE_SEND_STATEMENT(), ]); }); }); @@ -227,17 +284,22 @@ describe('planner', () => { MATCH_CHECK_SCHEDULE_GROUP: 'final-whistle-match-checks', MATCH_CHECK_FUNCTION_ARN: { 'Fn::GetAtt': [getLogicalId('AWS::Lambda::Function', 'FetchResults'), 'Arn'] }, MATCH_CHECK_ROLE_ARN: { 'Fn::GetAtt': [getLogicalId('AWS::IAM::Role', 'MatchChecks'), 'Arn'] }, + MATCH_CHECK_DEAD_LETTER_QUEUE_ARN: getQueueArn(), }, }, LoggingConfig: { LogGroup: { Ref: getLogicalId('AWS::Logs::LogGroup', 'Planner') } }, }); }); - it('may only write its logs, create schedules in the match check group, and pass their role to Scheduler', () => { - expectPolicyStatements('PlannerRoleDefaultPolicy', [LOG_WRITE_STATEMENT('Planner'), ...SCHEDULE_CHECK_STATEMENTS()]); + it('may only write its logs, send its failures, create schedules in the match check group, and pass their role to Scheduler', () => { + expectPolicyStatements('PlannerRoleDefaultPolicy', [ + LOG_WRITE_STATEMENT('Planner'), + FAILURE_SEND_STATEMENT(), + ...SCHEDULE_CHECK_STATEMENTS(), + ]); }); - it('runs every day at 10:00 Paris time, from a schedule outside the match check group', () => { + it('runs every day at 10:00 Paris time, from a schedule outside the match check group, with the failure queue as its DLQ', () => { template.resourceCountIs('AWS::Scheduler::Schedule', 1); template.hasResourceProperties('AWS::Scheduler::Schedule', { ScheduleExpression: 'cron(0 10 * * ? *)', @@ -246,17 +308,96 @@ describe('planner', () => { Target: Match.objectLike({ Arn: { 'Fn::GetAtt': [getLogicalId('AWS::Lambda::Function', 'Planner'), 'Arn'] }, RoleArn: { 'Fn::GetAtt': [getLogicalId('AWS::IAM::Role', 'PlannerMorningScheduleRole'), 'Arn'] }, + DeadLetterConfig: { Arn: getQueueArn() }, }), }); }); - it('is run by a role that may only invoke the planner', () => { + it('is run by a role that may only invoke the planner, or send a run it could not make to the failure queue', () => { expect(getProperties('AWS::IAM::Role', 'PlannerMorningScheduleRole')).toMatchObject({ AssumeRolePolicyDocument: SCHEDULER_TRUST, }); const plannerArn = { 'Fn::GetAtt': [getLogicalId('AWS::Lambda::Function', 'Planner'), 'Arn'] }; expectPolicyStatements('PlannerMorningScheduleRole', [ { Action: 'lambda:InvokeFunction', Effect: 'Allow', Resource: [plannerArn, { 'Fn::Join': ['', [plannerArn, ':*']] }] }, + SCHEDULER_FAILURE_SEND_STATEMENT(), ]); }); }); + +describe('alarms', () => { + const ALERT_ACTIONS = () => [{ Ref: getLogicalId('AWS::SNS::Topic') }]; + + it('notify a named, HTTPS-only topic that CloudWatch alarms of this account may publish on', () => { + template.resourceCountIs('AWS::SNS::Topic', 1); + template.hasResourceProperties('AWS::SNS::Topic', { TopicName: 'final-whistle-alerts' }); + template.resourceCountIs('AWS::SNS::Subscription', 0); + template.hasResourceProperties('AWS::SNS::TopicPolicy', { + PolicyDocument: { + Statement: [ + Match.objectLike({ Effect: 'Deny', Condition: { Bool: { 'aws:SecureTransport': 'false' } } }), + Match.objectLike({ + Action: 'sns:Publish', + Effect: 'Allow', + Principal: { Service: 'cloudwatch.amazonaws.com' }, + Condition: { StringEquals: { 'aws:SourceAccount': { Ref: 'AWS::AccountId' } } }, + }), + ], + }, + }); + }); + + it('are three, each notifying the topic only when it goes to ALARM', () => { + template.resourceCountIs('AWS::CloudWatch::Alarm', 3); + template.allResourcesProperties('AWS::CloudWatch::Alarm', { + AlarmActions: ALERT_ACTIONS(), + OKActions: Match.absent(), + Threshold: 1, + }); + }); + + it('fire on each message sent to the failure queue, even while an earlier one is still waiting', () => { + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + AlarmName: 'final-whistle-failure-queue-message-sent', + Namespace: 'AWS/SQS', + MetricName: 'NumberOfMessagesSent', + Dimensions: [{ Name: 'QueueName', Value: { 'Fn::GetAtt': [getLogicalId('AWS::SQS::Queue'), 'QueueName'] } }], + Statistic: 'Sum', + Period: 300, + EvaluationPeriods: 1, + ComparisonOperator: 'GreaterThanOrEqualToThreshold', + TreatMissingData: 'notBreaching', + }); + }); + + it('count in the fetch-results logs only the matches given up as not found, not the postponed ones', () => { + template.resourceCountIs('AWS::Logs::MetricFilter', 1); + template.hasResourceProperties('AWS::Logs::MetricFilter', { + LogGroupName: { Ref: getLogicalId('AWS::Logs::LogGroup', 'FetchResults') }, + FilterPattern: '{ ($.message = "match given up") && ($.status = "not-found") }', + MetricTransformations: [{ MetricNamespace: 'FinalWhistle', MetricName: 'MatchGivenUpNotFound', MetricValue: '1' }], + }); + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + AlarmName: 'final-whistle-match-not-found', + Namespace: 'FinalWhistle', + MetricName: 'MatchGivenUpNotFound', + Statistic: 'Sum', + ComparisonOperator: 'GreaterThanOrEqualToThreshold', + TreatMissingData: 'notBreaching', + }); + }); + + it('fire when the planner has had no invocation for 26 hours, an hour without data counting as none', () => { + template.hasResourceProperties('AWS::CloudWatch::Alarm', { + AlarmName: 'final-whistle-planner-not-run', + Namespace: 'AWS/Lambda', + MetricName: 'Invocations', + Dimensions: [{ Name: 'FunctionName', Value: { Ref: getLogicalId('AWS::Lambda::Function', 'Planner') } }], + Statistic: 'Sum', + Period: 3600, + EvaluationPeriods: 26, + ComparisonOperator: 'LessThanThreshold', + TreatMissingData: 'breaching', + }); + }); +}); diff --git a/test/scheduling/environment.test.ts b/test/scheduling/environment.test.ts index 9892ea4..5f54e3c 100644 --- a/test/scheduling/environment.test.ts +++ b/test/scheduling/environment.test.ts @@ -6,20 +6,23 @@ afterEach(() => { }); describe('readMatchCheckDestination', () => { - it('reads the schedule group and the invoke role', () => { + it('reads the schedule group, the invoke role and the dead-letter queue', () => { vi.stubEnv('MATCH_CHECK_SCHEDULE_GROUP', 'final-whistle-match-checks'); vi.stubEnv('MATCH_CHECK_ROLE_ARN', 'arn:aws:iam::123456789012:role/match-checks'); + vi.stubEnv('MATCH_CHECK_DEAD_LETTER_QUEUE_ARN', 'arn:aws:sqs:eu-west-3:123456789012:failures'); expect(readMatchCheckDestination()).toEqual({ scheduleGroupName: 'final-whistle-match-checks', roleArn: 'arn:aws:iam::123456789012:role/match-checks', + deadLetterQueueArn: 'arn:aws:sqs:eu-west-3:123456789012:failures', }); }); it('fails with the name of a missing variable', () => { vi.stubEnv('MATCH_CHECK_SCHEDULE_GROUP', 'final-whistle-match-checks'); - vi.stubEnv('MATCH_CHECK_ROLE_ARN', ''); + vi.stubEnv('MATCH_CHECK_ROLE_ARN', 'arn:aws:iam::123456789012:role/match-checks'); + vi.stubEnv('MATCH_CHECK_DEAD_LETTER_QUEUE_ARN', ''); - expect(() => readMatchCheckDestination()).toThrow('MATCH_CHECK_ROLE_ARN'); + expect(() => readMatchCheckDestination()).toThrow('MATCH_CHECK_DEAD_LETTER_QUEUE_ARN'); }); }); diff --git a/test/scheduling/match-schedule.test.ts b/test/scheduling/match-schedule.test.ts index 6689413..ad60eb4 100644 --- a/test/scheduling/match-schedule.test.ts +++ b/test/scheduling/match-schedule.test.ts @@ -12,6 +12,7 @@ const TARGET = { scheduleGroupName: 'final-whistle-match-checks', functionArn: 'arn:aws:lambda:eu-west-3:123456789012:function:fetch-results', roleArn: 'arn:aws:iam::123456789012:role/match-checks', + deadLetterQueueArn: 'arn:aws:sqs:eu-west-3:123456789012:failures', }; const CHECK_AT = parseParisWallClock('2026-11-01', '17:00'); @@ -33,7 +34,7 @@ describe('buildScheduleName', () => { }); describe('buildMatchCheckCommand', () => { - it('invokes fetch-results once with the match check, in Paris time, then deletes itself', () => { + it('invokes fetch-results once with the match check, in Paris time, then deletes itself; an invocation it cannot make goes to the dead-letter queue', () => { expect(buildMatchCheckCommand(TARGET, CHECK, CHECK_AT).input).toEqual({ Name: buildScheduleName(CHECK), GroupName: TARGET.scheduleGroupName, @@ -42,7 +43,12 @@ describe('buildMatchCheckCommand', () => { ScheduleExpressionTimezone: 'Europe/Paris', FlexibleTimeWindow: { Mode: 'OFF' }, ActionAfterCompletion: 'DELETE', - Target: { Arn: TARGET.functionArn, RoleArn: TARGET.roleArn, Input: JSON.stringify(CHECK) }, + Target: { + Arn: TARGET.functionArn, + RoleArn: TARGET.roleArn, + Input: JSON.stringify(CHECK), + DeadLetterConfig: { Arn: TARGET.deadLetterQueueArn }, + }, }); }); });