Ask rtibblesbot to validate a contributor-opened issue

.github/workflows/automation-group-issue-comment.yml

@@ -0,0 +1,43 @@
+# This file is generated by scripts/generate-automation.js from automation-registry.yml.
+# Do not edit it by hand - edit the registry and run:
+#   node scripts/generate-automation.js
+#
+# Reusable workflow called from automation.yml: the automations triggered by issue_comment.
+name: "Automation: Issue Comment"
+on:
+  workflow_call:
+    secrets:
+      LE_BOT_APP_ID:
+        description: "Forwarded to: issue-validation-done, contributor-issue-comment"
+        required: true
+      LE_BOT_PRIVATE_KEY:
+        description: "Forwarded to: issue-validation-done, contributor-issue-comment"
+        required: true
+      SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL:
+        description: "Forwarded to: contributor-issue-comment"
+        required: false
+      SLACK_WEBHOOK_URL:
+        description: "Forwarded to: issue-validation-done, contributor-issue-comment"
+        required: false
+jobs:
+  issue-validation-done:
+    name: Issue Validation Done
+    if: ${{ github.event_name == 'issue_comment' && github.event.action == 'created' }}
+    uses: learningequality/.github/.github/workflows/issue-validation-done.yml@main
+    permissions:
+      contents: read
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  contributor-issue-comment:
+    name: Contributor Issue Comment
+    if: ${{ github.event_name == 'issue_comment' && github.event.action == 'created' }}
+    uses: learningequality/.github/.github/workflows/contributor-issue-comment.yml@main
+    permissions:
+      contents: read
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+      SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL: ${{ secrets.SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL }}

.github/workflows/automation-group-issues.yml

@@ -8,10 +8,10 @@ on:
   workflow_call:
     secrets:
       LE_BOT_APP_ID:
-        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label"
+        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation"
         required: true
       LE_BOT_PRIVATE_KEY:
-        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label"
+        description: "Forwarded to: issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation"
         required: true
 jobs:
   issue-open:
@@ -50,3 +50,12 @@ jobs:
     secrets:
       LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
       LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+  issue-validation:
+    name: Issue Validation
+    if: ${{ github.event_name == 'issues' && github.event.action == 'opened' }}
+    uses: learningequality/.github/.github/workflows/issue-validation.yml@main
+    permissions:
+      contents: read
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}

.github/workflows/automation.yml

@@ -18,16 +18,16 @@ on:
         description: Unused - accepted until every consumer stops forwarding it
         required: false
       LE_BOT_APP_ID:
-        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, contributor-issue-comment, unassign-inactive-issues"
+        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation, issue-validation-done, contributor-issue-comment, unassign-inactive-issues"
         required: true
       LE_BOT_PRIVATE_KEY:
-        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, contributor-issue-comment, unassign-inactive-issues"
+        description: "Forwarded to: review-requested, pull-request-label, dependabot-reviewer, contributor-pr-reply, pull-request-template, issue-open, issue-label-header, good-first-issue-comment, community-contribution-label, issue-validation, issue-validation-done, contributor-issue-comment, unassign-inactive-issues"
         required: true
       SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL:
         description: "Forwarded to: contributor-pr-reply, contributor-issue-comment, unassign-inactive-issues"
         required: false
       SLACK_WEBHOOK_URL:
-        description: "Forwarded to: contributor-issue-comment"
+        description: "Forwarded to: issue-validation-done, contributor-issue-comment"
         required: false
 jobs:
   pull-request-target:
@@ -42,24 +42,24 @@ jobs:
       SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL: ${{ secrets.SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL }}
   issues:
     name: Issues
-    if: ${{ (github.event_name == 'issues' && contains(fromJSON('["opened","reopened"]'), github.event.action)) || (github.event_name == 'issues' && contains(fromJSON('["labeled","unlabeled"]'), github.event.action) && github.event.label.name == 'help wanted') || (github.event_name == 'issues' && github.event.action == 'labeled' && (github.event.label.name == 'good first issue' || github.event.label.name == 'help wanted')) || (github.event_name == 'issues' && contains(fromJSON('["assigned","unassigned"]'), github.event.action)) }}
+    if: ${{ (github.event_name == 'issues' && contains(fromJSON('["opened","reopened"]'), github.event.action)) || (github.event_name == 'issues' && contains(fromJSON('["labeled","unlabeled"]'), github.event.action) && github.event.label.name == 'help wanted') || (github.event_name == 'issues' && github.event.action == 'labeled' && (github.event.label.name == 'good first issue' || github.event.label.name == 'help wanted')) || (github.event_name == 'issues' && contains(fromJSON('["assigned","unassigned"]'), github.event.action)) || (github.event_name == 'issues' && github.event.action == 'opened') }}
     uses: learningequality/.github/.github/workflows/automation-group-issues.yml@main
     permissions:
       contents: read
     secrets:
       LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
       LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
-  contributor-issue-comment:
-    name: Contributor Issue Comment
-    if: ${{ github.event_name == 'issue_comment' && github.event.action == 'created' }}
-    uses: learningequality/.github/.github/workflows/contributor-issue-comment.yml@main
+  issue-comment:
+    name: Issue Comment
+    if: ${{ (github.event_name == 'issue_comment' && github.event.action == 'created') || (github.event_name == 'issue_comment' && github.event.action == 'created') }}
+    uses: learningequality/.github/.github/workflows/automation-group-issue-comment.yml@main
     permissions:
       contents: read
     secrets:
       LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
       LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
       SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL: ${{ secrets.SLACK_COMMUNITY_NOTIFICATIONS_WEBHOOK_URL }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
   resolve-bot-pr-threads:
     name: Resolve Bot PR Threads
     if: ${{ github.event_name == 'pull_request_review_comment' }}

.github/workflows/issue-validation-done.yml

@@ -0,0 +1,57 @@
+name: Clear the issue review label when the validation lands
+on:
+  workflow_call:
+    secrets:
+      LE_BOT_APP_ID:
+        description: 'GitHub App ID for authentication'
+        required: true
+      LE_BOT_PRIVATE_KEY:
+        description: 'GitHub App Private Key for authentication'
+        required: true
+      SLACK_WEBHOOK_URL:
+        description: 'Webhook URL for Slack #support-dev channel'
+        required: true
+jobs:
+  clear-review-label:
+    name: Clear review label
+    if: >
+      github.event.comment.user.login == 'rtibblesbot' && contains(github.event.comment.body, '<!-- ISSUE_VALIDATION -->')
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate App Token
+        id: generate-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.LE_BOT_APP_ID }}
+          private-key: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+          permission-issues: write
+      - name: Checkout .github repository
+        uses: actions/checkout@v7
+        with:
+          repository: learningequality/.github
+          ref: main
+      - name: Setup Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+      - name: Run script
+        id: script
+        uses: actions/github-script@v9
+        with:
+          github-token: ${{ steps.generate-token.outputs.token }}
+          script: |
+            const script = require('./scripts/issue-validation-done.js');
+            return await script({ github, context, core });
+      - name: Send Slack notification to support-dev channel
+        if: ${{ steps.script.outputs.support_dev_message }}
+        uses: slackapi/slack-github-action@v4.0.0
+        with:
+          webhook-type: incoming-webhook
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          payload: |
+            {
+              "text": "${{ steps.script.outputs.support_dev_message }}"
+            }

.github/workflows/issue-validation.yml

@@ -0,0 +1,54 @@
+name: Ask rtibblesbot to validate a contributor issue
+on:
+  workflow_call:
+    secrets:
+      LE_BOT_APP_ID:
+        description: 'GitHub App ID for authentication'
+        required: true
+      LE_BOT_PRIVATE_KEY:
+        description: 'GitHub App Private Key for authentication'
+        required: true
+permissions:
+  contents: read
+jobs:
+  check-if-contributor:
+    name: Check if author is contributor
+    uses: learningequality/.github/.github/workflows/is-contributor.yml@main
+    secrets:
+      LE_BOT_APP_ID: ${{ secrets.LE_BOT_APP_ID }}
+      LE_BOT_PRIVATE_KEY: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+    with:
+      username: ${{ github.event.issue.user.login }}
+      author_association: ${{ github.event.issue.author_association }}
+  request-validation:
+    name: Request validation
+    needs: [check-if-contributor]
+    if: ${{ needs.check-if-contributor.outputs.is_contributor == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate App Token
+        id: generate-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.LE_BOT_APP_ID }}
+          private-key: ${{ secrets.LE_BOT_PRIVATE_KEY }}
+          permission-issues: write
+      - name: Checkout .github repository
+        uses: actions/checkout@v7
+        with:
+          repository: learningequality/.github
+          ref: main
+      - name: Setup Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: '22'
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+      - name: Run script
+        id: script
+        uses: actions/github-script@v9
+        with:
+          github-token: ${{ steps.generate-token.outputs.token }}
+          script: |
+            const script = require('./scripts/issue-validation.js');
+            return await script({ github, context, core });

automation-registry.yml

@@ -128,6 +128,27 @@ automations:
     secrets:
       LE_BOT_APP_ID: required
       LE_BOT_PRIVATE_KEY: required
+  - name: issue-validation
+    enabled: true
+    leaf: issue-validation.yml
+    on:
+      issues:
+        types: [opened]
+    if: "github.event_name == 'issues' && github.event.action == 'opened'"
+    secrets:
+      LE_BOT_APP_ID: required
+      LE_BOT_PRIVATE_KEY: required
+  - name: issue-validation-done
+    enabled: true
+    leaf: issue-validation-done.yml
+    on:
+      issue_comment:
+        types: [created]
+    if: "github.event_name == 'issue_comment' && github.event.action == 'created'"
+    secrets:
+      LE_BOT_APP_ID: required
+      LE_BOT_PRIVATE_KEY: required
+      SLACK_WEBHOOK_URL: optional
   - name: contributor-issue-comment
     enabled: true
     leaf: contributor-issue-comment.yml

docs/community-automations.md

@@ -102,6 +102,62 @@ Four states fail, and the log names every one of them rather than stopping at th
 
 The failed check is the only report. Nothing is posted on the pull request.
 
+# `issue-validation` and `issue-validation-done`
+
+These two automations ask `rtibblesbot` to validate claims in contributor-submitted issues and track the review while it is in progress.
+
+## `issue-validation`
+
+Runs on `issues: [opened]`. It proceeds only if the repository is in `ISSUE_VALIDATION_REPOS`, the issue opener is a contributor, and the issue does not already have the `community-issue-review` label.
+
+It posts a comment mentioning `@rtibblesbot` and adds the label. If the label does not exist in the repository, the add-label call creates it.
+
+**The mention is essential.** The agent polls its GitHub notifications and has no inbound endpoint, so a mention is the only way to reach it. Adding a label does not generate a notification, and a handle inside backticks, as in `BOT_MESSAGE_RTIBBLESBOT_REVIEW`, does not trigger one either.
+
+## `issue-validation-done`
+
+Runs on `issue_comment: [created]`. It removes the `community-issue-review` label only when both conditions are met:
+
+- The comment contains the `<!-- ISSUE_VALIDATION -->` marker.
+- The comment author is `rtibblesbot`.
+
+Both checks are necessary. Anyone can include the marker in a comment, and `rtibblesbot` also comments on issues for other reasons.
+
+The leaf workflow tests both conditions in its job `if:`, as `resolve-bot-pr-threads` does, so an ordinary comment starts no runner. The script tests them again.
+
+This logic cannot live in `contributor-issue-comment`, because that automation deliberately ignores bot comments.
+
+It also posts one line to Slack `#support-dev` for every validation comment, so a maintainer knows to follow up. The line carries the verdict, a link to the comment, a link to the issue, the reporter's handle, and the follow-up the verdict calls for:
+
+```
+*[kolibri] <comment-url|Validation: duplicate of #15001> on issue: <issue-url|Title> reported by _octocat_* _(consider closing as duplicate)_
+```
+
+`ISSUE_VALIDATION_FOLLOW_UPS` holds the six verdicts and their follow-ups. `learningequality/coding-agent#1229` fixes the vocabulary from the agent's side.
+
+The agent's LLM session writes the verdict line, so its shape varies. `readVerdict` therefore mirrors the agent's own reader, which tolerates a missing bold, a colon on either side of it, a lowercase label, a list or quote prefix, backticks, and trailing punctuation. Keep the two readers in step. A stricter reader here reports a healthy run as broken.
+
+A comment outside all of that reads `Validation: no verdict read` and asks a maintainer to check the agent run. The agent posts no comment at all when it states no verdict, so this line means the rendering drifted, not that a verdict is missing.
+
+## Handling stalled reviews
+
+The weekly `unassign-inactive-issues` run also sweeps stalled validations through `scripts/issue-review-sweep.js`.
+
+If the agent never responds, for example, because the service is down or all available tokens have reached their limits, the label would otherwise remain indefinitely. The sweep removes it when the issue's `labeled` event is older than `ISSUE_REVIEW_MAX_DAYS` days.
+
+## Scope and configuration
+
+Neither automation closes issues or acts on the validation findings. The agent's comment recommends whether the issue should be closed; a maintainer must request that action in a reply.
+
+Configure the following constants in `scripts/constants.js`:
+
+- `ISSUE_VALIDATION_REPOS`: The repositories where validation runs, which are the community repositories plus the sandbox the automations are tested in.
+- `LABEL_COMMUNITY_ISSUE_REVIEW`: The label applied while validation is in progress.
+- `ISSUE_VALIDATION_MARKER`: The marker that signals validation completion, shared with `learningequality/coding-agent`.
+- `ISSUE_REVIEW_MAX_DAYS`: The maximum age of the label before the sweep clears it.
+- `BOT_MESSAGE_ISSUE_VALIDATION`: The comment text posted to request validation.
+- `ISSUE_VALIDATION_FOLLOW_UPS`: The six verdicts and the follow-up each one asks a maintainer for, used in the Slack line.
+
 # `holiday-message`
 
 Sends a holiday message to community pull requests and issue comments.