From 772e83d104074b7e6bae13495863131d50db5342 Mon Sep 17 00:00:00 2001 From: fallenbagel <98979876+fallenbagel@users.noreply.github.com> Date: Thu, 2 Apr 2026 13:59:51 +0800 Subject: [PATCH] chore: add PR validation workflow and update contributing guidelines (#2777) --- .github/workflows/pr-validation.yml | 283 ++++++++++++++++++++++++++++ .github/workflows/semantic-pr.yml | 28 --- CONTRIBUTING.md | 101 ++++++++-- bin/check-pr-template.mjs | 92 +++++++++ 4 files changed, 456 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/pr-validation.yml delete mode 100644 .github/workflows/semantic-pr.yml create mode 100644 bin/check-pr-template.mjs diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 00000000..1d3ace53 --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,283 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "PR Validation" + +on: + pull_request_target: + types: + - opened + - reopened + - edited + - synchronize + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + semantic-title: + name: Validate PR Title + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + checks: write + issues: write + steps: + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 + id: lint_pr_title + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + if: always() && steps.lint_pr_title.outputs.error_message != null + env: + ERROR_MESSAGE: ${{ steps.lint_pr_title.outputs.error_message }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const message = process.env.ERROR_MESSAGE; + const prNumber = context.payload.pull_request.number; + + const body = [ + `### PR Title Validation Failed\n`, + message, + `\n---\n`, + `PR titles must follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/).`, + `*This check will re-run when you update your PR title.*`, + ].join('\n'); + + const allComments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + } + ); + + const botComment = allComments.find( + c => c.user.type === 'Bot' && c.body && c.body.includes('### PR Title Validation Failed') + ); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); + } + + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + if: always() && steps.lint_pr_title.outputs.error_message == null + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const prNumber = context.payload.pull_request.number; + + const allComments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + } + ); + + const botComment = allComments.find( + c => c.user.type === 'Bot' && c.body && c.body.includes('### PR Title Validation Failed') + ); + + if (botComment) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + }); + } + + template-check: + name: Validate PR Template + if: github.event.action != 'synchronize' + runs-on: ubuntu-24.04 + permissions: + contents: read + issues: write + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version-file: 'package.json' + + - name: Skip bot PRs + id: bot-check + if: github.event.pull_request.user.type == 'Bot' + run: echo "skip=true" >> "$GITHUB_OUTPUT" + + - name: Write PR body to file + if: steps.bot-check.outputs.skip != 'true' + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: printf '%s' "$PR_BODY" > /tmp/pr-body.txt + + - name: Run template check + if: steps.bot-check.outputs.skip != 'true' + id: check + env: + AUTHOR_ASSOCIATION: ${{ github.event.pull_request.author_association }} + run: | + set +e + ISSUES=$(node bin/check-pr-template.mjs /tmp/pr-body.txt) + EXIT_CODE=$? + echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" + { + echo 'issues<> "$GITHUB_OUTPUT" + exit 0 + + - name: Label and comment on failure + if: steps.bot-check.outputs.skip != 'true' && steps.check.outputs.exit_code != '0' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + ISSUES_JSON: ${{ steps.check.outputs.issues }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issues = JSON.parse(process.env.ISSUES_JSON); + const author = process.env.PR_AUTHOR; + const prNumber = context.payload.pull_request.number; + const LABEL = 'blocked:template'; + + const issueList = issues.map(i => `- ${i}`).join('\n'); + + const commentBody = [ + `Hey @${author}, thanks for submitting this PR! However, it looks like the PR template hasn't been fully filled out.\n`, + `### Issues found:\n`, + issueList, + `\n---\n`, + `**Please update your PR description to follow the [PR template](https://github.com/${context.repo.owner}/${context.repo.repo}/blob/develop/.github/PULL_REQUEST_TEMPLATE.md).**`, + `Incomplete or missing PR descriptions may indicate insufficient review of the changes, and PRs that do not follow the template **may be closed without review**.`, + `See our [Contributing Guide](https://github.com/${context.repo.owner}/${context.repo.repo}/blob/develop/CONTRIBUTING.md) for more details.\n`, + `*This check will automatically re-run when you edit your PR description.*`, + ].join('\n'); + + const allComments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + } + ); + + const botComment = allComments.find( + c => c.user.type === 'Bot' && c.body && c.body.includes('### Issues found:') + ); + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: commentBody, + }); + } + + try { + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: [LABEL], + }); + } catch (e) { + try { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: LABEL, + color: 'B60205', + description: 'PR template not properly filled out', + }); + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: [LABEL], + }); + } catch (e2) { + console.log('Could not create/add label:', e2.message); + } + } + + core.setFailed('PR template is not properly filled out.'); + + - name: Remove label on success + if: steps.bot-check.outputs.skip != 'true' && steps.check.outputs.exit_code == '0' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const prNumber = context.payload.pull_request.number; + const LABEL = 'blocked:template'; + + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + name: LABEL, + }); + } catch (e) { + console.log('Could not remove label', e.message); + } + + const allComments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + } + ); + + const botComment = allComments.find( + c => c.user.type === 'Bot' && c.body && c.body.includes('### Issues found:') + ); + + if (botComment) { + await github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + }); + } \ No newline at end of file diff --git a/.github/workflows/semantic-pr.yml b/.github/workflows/semantic-pr.yml deleted file mode 100644 index 06546033..00000000 --- a/.github/workflows/semantic-pr.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Semantic PR" - -on: - pull_request_target: - types: - - opened - - reopened - - edited - - synchronize - -permissions: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - main: - name: Validate PR Title - runs-on: ubuntu-slim - permissions: - contents: read - pull-requests: read - checks: write - steps: - - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d1581c7..107ffb3e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,43 +10,104 @@ All help is welcome and greatly appreciated! If you would like to contribute to > This is an open-source project maintained by volunteers. > We do not have the resources to review pull requests that could have been avoided with proper human oversight. > While we have no issue with contributors using AI tools as an aid, it is your responsibility as a contributor to ensure that all submissions are carefully reviewed and meet our quality standards. -> Submissions that appear to be unreviewed AI output will be considered low-effort and may result in a ban. > -> If you are using **any kind of AI assistance** to contribute to Seerr, -> it must be disclosed in the pull request. +> **We expect AI-assisted development, not AI-driven development.** +> Use AI as a tool to help you write code. Do not let an AI agent +> autonomously generate an entire contribution and submit it on your behalf. +> We have been increasingly receiving low-effort, fully AI-generated PRs +> and will not tolerate them. Contributors who repeatedly submit unreviewed +> AI output may result in a ban. +> +> **Submissions that appear to be unreviewed AI output will be considered low-effort and may result in a ban.** Signs of unreviewed AI output include but are not limited to: +> +> - Blank or template-default PR descriptions +> - AI-generated PR descriptions that replace our template with their own structure (e.g., "Summary / What changed / Root cause / Test plan" instead of following the PR template; this is the default output format of tools like Claude Code and is an immediate indicator that the PR was not reviewed by a human) +> - Unchecked checklists or missing checklist entirely +> - Failing CI checks that would have been caught by running `pnpm build` +> - Code that does not match the described changes +> - Inability to answer questions about the submitted code +> +> **Read and follow the [Contributing Guide](CONTRIBUTING.md) before submitting.** +> If your AI tool generates its own PR description format, it is your +> responsibility to rewrite it to follow our template before submitting. +> An incomplete PR template tells maintainers that insufficient review has +> been performed on the submission, regardless of the actual code quality. +> We may close such PRs without review. +> +> If you are using **any kind of AI assistance** to contribute to Seerr, it must be disclosed in the pull request. + +### Disclosure Requirements If you are using any kind of AI assistance while contributing to Seerr, -**this must be disclosed in the pull request**, along with the extent to -which AI assistance was used (e.g. docs only vs. code generation). -If PR responses are being generated by an AI, disclose that as well. +**this must be disclosed in the pull request description**, along with +the extent to which AI assistance was used (e.g., docs only vs. code generation). +If PR responses (comments, review replies) are being generated by AI, +disclose that as well. + As a small exception, trivial tab-completion doesn't need to be disclosed, so long as it is limited to single keywords or short phrases. -An example disclosure: +Example disclosures: -> This PR was written primarily by Claude Code. - -Or a more detailed disclosure: - -> I consulted ChatGPT to understand the codebase but the solution +> **AI Disclosure:** This PR was written primarily by Claude Code. +> **AI Disclosure:** I consulted ChatGPT to understand the codebase but the solution > was fully authored manually by myself. +> **AI Disclosure:** None. -Failure to disclose this is first and foremost rude to the human operators -on the other end of the pull request, but it also makes it difficult to -determine how much scrutiny to apply to the contribution. +When using AI assistance, we expect contributors to: + +- **Understand the code** that is produced and be able to answer + questions about it. +- **Follow the contributing guide**. AI tools do not excuse you from + filling out the PR template, testing section, and checklist. +- **Run the build and tests** before submitting. + +Failure to disclose AI assistance is first and foremost disrespectful to the +human maintainers on the other end of the pull request, but it also makes it +difficult to determine how much scrutiny to apply to the contribution. In a perfect world, AI assistance would produce equal or higher quality -work than any human. That isn't the world we live in today, and in most cases -it's generating slop. I say this despite being a fan of and using them -successfully myself (with heavy supervision)! +work than any human. That is not the world we live in today, and in most cases +it is generating slop. When using AI assistance, we expect contributors to understand the code that is produced and be able to answer critical questions about it. It -isn't a maintainers job to review a PR so broken that it requires +is not a maintainer's job to review a PR so broken that it requires significant rework to be acceptable. Please be respectful to maintainers and disclose AI assistance. +### Expectations for AI-Assisted Contributions + +1. **PR descriptions and all comments must be your own words.** Do not paste + LLM output as your PR description, review response, or issue comment. + We want *your* understanding and explanation of the changes, not a + machine-generated summary. An exception is made for LLM-assisted + translation, however, note it explicitly if used. + +2. **Contributions must be concise and focused.** A PR that claims to fix one thing + but touches a bunch of unrelated code will be rejected. This is a + common side effect of broad AI prompts and makes review unnecessarily + difficult. + +3. **You must be able to handle review feedback yourself.** If you cannot discuss or + implement requested changes without round-tripping reviewer comments + through an AI, that tells us you don't understand the code you + submitted. We will close the PR. + +4. **Don't commit non-project files.** Editor configs, AI tool + directories, and other local tooling files do not belong in the + repository. Keep your commits clean. + +5. **Changes must be tested.** Build the project, run the tests, and + manually verify the functionality you modified. Don't just assume + CI will catch everything. + +6. **Final discretion lies with the reviewers.** If a PR cannot be + reasonably reviewed for any reason due to over-complexity, size, or poor + structure, it will be rejected. This applies equally to AI-assisted + and non-AI-assisted contributions. + ## Development ### Tools Required @@ -202,4 +263,4 @@ DB_TYPE="postgres" DB_USER=postgres DB_PASS=postgres pnpm migration:generate ser ## Attribution -This contribution guide was inspired by the [Next.js](https://github.com/vercel/next.js), [Radarr](https://github.com/Radarr/Radarr), and [Ghostty](https://github.com/ghostty-org/ghostty) contribution guides. +This contribution guide was inspired by the [Next.js](https://github.com/vercel/next.js), [Radarr](https://github.com/Radarr/Radarr), and [Ghostty](https://github.com/ghostty-org/ghostty) contribution guides. In addition, our AI policy was draws from [Jellyfin's LLM policies](https://jellyfin.org/docs/general/contributing/llm-policies/). diff --git a/bin/check-pr-template.mjs b/bin/check-pr-template.mjs new file mode 100644 index 00000000..ff403336 --- /dev/null +++ b/bin/check-pr-template.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node +/** + * Validate that a pull request body follows the PR template. + * + */ +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +const bodyFile = process.argv[2]; + +if (!bodyFile) { + console.error('body file path is required as an argument.'); + process.exit(2); +} + +const body = readFileSync(resolve(bodyFile), 'utf8'); +const issues = []; + +const MAINTAINER_ROLES = ['OWNER', 'MEMBER', 'COLLABORATOR']; +const isMaintainer = MAINTAINER_ROLES.includes( + process.env.AUTHOR_ASSOCIATION ?? '' +); + +const stripComments = (s) => { + let result = s; + let previous; + do { + previous = result; + result = result.replace(//g, ''); + } while (result !== previous); + return result; +}; + +const stripFixesPlaceholder = (s) => s.replace(/-\s*Fixes\s*`?#XXXX`?/gi, ''); + +const descriptionMatch = body.match(/## Description\s*\n([\s\S]*?)(?=\n## |$)/); +const descriptionContent = descriptionMatch + ? stripFixesPlaceholder(stripComments(descriptionMatch[1])).trim() + : ''; + +if (!descriptionContent) { + issues.push( + '**Description** section is empty or only contains placeholder text.' + ); +} + +const testingMatch = body.match( + /## How Has This Been Tested\?\s*\n([\s\S]*?)(?=\n## |$)/ +); +const testingContent = testingMatch + ? stripComments(testingMatch[1]).trim() + : ''; + +if (!testingContent) { + issues.push('**How Has This Been Tested?** section is empty.'); +} + +const checklistMatch = body.match(/## Checklist:\s*\n([\s\S]*?)$/); +const checklistContent = checklistMatch ? checklistMatch[1] : ''; + +const totalBoxes = (checklistContent.match(/- \[[ x]\]/g) || []).length; +const checkedBoxes = (checklistContent.match(/- \[x\]/gi) || []).length; + +if (totalBoxes === 0) { + issues.push('**Checklist** section is missing or has been removed.'); +} else if (checkedBoxes === 0) { + issues.push( + 'No items in the **checklist** have been checked. Please review and check all applicable items.' + ); +} + +if ( + !/- \[x\] I have read and followed the contribution/i.test(checklistContent) +) { + issues.push('The **contribution guidelines** checkbox has not been checked.'); +} + +if ( + !isMaintainer && + !/- \[x\] Disclosed any use of AI/i.test(checklistContent) +) { + issues.push('The **AI disclosure** checkbox has not been checked.'); +} + +if (/-\s*Fixes\s*`?#XXXX`?/i.test(body)) { + issues.push( + 'The `Fixes #XXXX` placeholder has not been updated. Please link the relevant issue or remove it.' + ); +} + +console.log(JSON.stringify(issues)); +process.exit(issues.length > 0 ? 1 : 0);