diff --git a/.github/workflows/eval-pr-comment.yml b/.github/workflows/eval-pr-comment.yml new file mode 100644 index 00000000..9f88e20a --- /dev/null +++ b/.github/workflows/eval-pr-comment.yml @@ -0,0 +1,48 @@ +name: Eval PR Comment + +# Posts a small comment on every new PR with a link to the evals-monitor app, +# where the author can start an eval for their PR. We post a link rather than +# triggering the job from CI because GitHub Actions runners cannot reach the +# dogfood.staging workspace (network perimeter). +on: + pull_request: + types: [opened, reopened] + branches: [main] + +permissions: + contents: read + pull-requests: write + +jobs: + post-eval-link: + name: Post eval link comment + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + steps: + - name: Post eval link comment + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const marker = ""; + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const url = `https://evals-monitor-6051921418418893.staging.aws.databricksapps.com/prs/appkit/${issue_number}`; + + // Idempotent: skip if we've already commented on this PR. + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number, + per_page: 100, + }); + if (comments.some((c) => c.body?.includes(marker))) return; + + const body = `${marker}\n> 🔬  **Run evals on this PR**  Â·  [**Go to Evals Monitor →**](${url})`; + + await github.rest.issues.createComment({ + owner, + repo, + issue_number, + body, + });