From 21875d70526c10ec94f9af1587689da43fbe3f27 Mon Sep 17 00:00:00 2001 From: Ludy Date: Tue, 24 Jun 2025 00:05:54 +0200 Subject: [PATCH] feat: add automated PR title review using GitHub Actions and AI (#3784) # Description of Changes - Added a new GitHub Actions workflow `.github/workflows/ai_pr_title_review.yml` to perform AI-powered PR title evaluations - Introduced configuration files: - `.github/config/repo_devs.json` to define trusted developers - `.github/config/system-prompt.txt` as the system prompt for the AI model - Workflow checks the PR actor against the `repo_devs.json` list and evaluates the PR title if the actor is a listed developer - Integrates GPT-4o via `actions/ai-inference` to analyze diffs and suggest improved PR titles in JSON - Posts a suggestion comment or praise, depending on AI rating, using `github-script` - Supports secure repo setup with hardened runners and tokenized GitHub App bot access --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/HowToAddNewLanguage.md) (if applicable) - [ ] I have performed a self-review of my own code - [ ] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#6-testing) for more details. --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/config/repo_devs.json | 12 ++ .github/config/system-prompt.txt | 13 ++ .github/workflows/ai_pr_title_review.yml | 228 +++++++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 .github/config/repo_devs.json create mode 100644 .github/config/system-prompt.txt create mode 100644 .github/workflows/ai_pr_title_review.yml diff --git a/.github/config/repo_devs.json b/.github/config/repo_devs.json new file mode 100644 index 000000000..a2aaac4f2 --- /dev/null +++ b/.github/config/repo_devs.json @@ -0,0 +1,12 @@ +{ + "repo_devs": [ + "frooodle", + "sf298", + "Ludy87", + "LaserKaspar", + "sbplat", + "reecebrowne", + "DarioGii", + "ConnorYoh" + ] +} diff --git a/.github/config/system-prompt.txt b/.github/config/system-prompt.txt new file mode 100644 index 000000000..f3842878f --- /dev/null +++ b/.github/config/system-prompt.txt @@ -0,0 +1,13 @@ +You are a professional software engineer specializing in reviewing pull request titles. + +Your job is to analyze a git diff and an existing PR title, then evaluate and improve the PR title. + +You must: +- Always return valid JSON +- Only return the JSON response (no Markdown, no formatting) +- Use one of these conventional commit types at the beginning of the title: build, chore, ci, docs, feat, fix, perf, refactor, revert, style, test +- Use lowercase only, no emojis, no trailing period +- Ensure the title is between 5 and 72 printable ASCII characters +- Never let spelling or grammar errors affect the rating +- If the PR title is rated 6 or higher and only contains spelling or grammar mistakes, correct it - do not rephrase it +- If the PR title is rated below 6, generate a new, better title based on the diff diff --git a/.github/workflows/ai_pr_title_review.yml b/.github/workflows/ai_pr_title_review.yml new file mode 100644 index 000000000..0447a9b62 --- /dev/null +++ b/.github/workflows/ai_pr_title_review.yml @@ -0,0 +1,228 @@ +name: AI - PR Title Review + +on: + pull_request: + types: [opened, edited] + branches: [main] + +permissions: # required for secure-repo hardening + contents: read + +jobs: + ai-title-review: + permissions: + contents: read + pull-requests: write + models: read + + runs-on: ubuntu-latest + + steps: + - name: Harden Runner + uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 + with: + egress-policy: audit + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Configure Git to suppress detached HEAD warning + run: git config --global advice.detachedHead false + + - name: Setup GitHub App Bot + if: github.actor != 'dependabot[bot]' + id: setup-bot + uses: ./.github/actions/setup-bot + continue-on-error: true + with: + app-id: ${{ secrets.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + + - name: Check if actor is repo developer + id: actor + run: | + if [[ "${{ github.actor }}" == *"[bot]" ]]; then + echo "PR opened by a bot ā€“ skipping AI title review." + echo "is_repo_dev=false" >> $GITHUB_OUTPUT + exit 0 + fi + if [ ! -f .github/config/repo_devs.json ]; then + echo "Error: .github/config/repo_devs.json not found" >&2 + exit 1 + fi + # Validate JSON and extract repo_devs + REPO_DEVS=$(jq -r '.repo_devs[]' .github/config/repo_devs.json 2>/dev/null || { echo "Error: Invalid JSON in repo_devs.json" >&2; exit 1; }) + # Convert developer list into Bash array + mapfile -t DEVS_ARRAY <<< "$REPO_DEVS" + if [[ " ${DEVS_ARRAY[*]} " == *" ${{ github.actor }} "* ]]; then + echo "is_repo_dev=true" >> $GITHUB_OUTPUT + else + echo "is_repo_dev=false" >> $GITHUB_OUTPUT + fi + + - name: Get PR diff + if: steps.actor.outputs.is_repo_dev == 'true' + id: get_diff + run: | + git fetch origin ${{ github.base_ref }} + git diff origin/${{ github.base_ref }}...HEAD | head -n 10000 | grep -vP '[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x{202E}\x{200B}]' > pr.diff + echo "diff<> $GITHUB_OUTPUT + cat pr.diff >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Check and sanitize PR title + if: steps.actor.outputs.is_repo_dev == 'true' + id: sanitize_pr_title + env: + PR_TITLE_RAW: ${{ github.event.pull_request.title }} + run: | + # Sanitize PR title: max 72 characters, only printable characters + PR_TITLE=$(echo "$PR_TITLE_RAW" | tr -d '\n\r' | head -c 72 | sed 's/[^[:print:]]//g') + if [[ ${#PR_TITLE} -lt 5 ]]; then + echo "PR title is too short. Must be at least 5 characters." >&2 + fi + echo "pr_title=$PR_TITLE" >> $GITHUB_OUTPUT + + - name: AI PR Title Analysis + if: steps.actor.outputs.is_repo_dev == 'true' + id: ai-title-analysis + uses: actions/ai-inference@d645f067d89ee1d5d736a5990e327e504d1c5a4a # v1.1.0 + with: + model: openai/gpt-4o + system-prompt-file: ".github/config/system-prompt.txt" + prompt: | + Based on the following input data: + + { + "diff": "${{ steps.get_diff.outputs.diff }}", + "pr_title": "${{ steps.sanitize_pr_title.outputs.pr_title }}" + } + + Respond ONLY with valid JSON in the format: + { + "improved_rating": <0-10>, + "improved_ai_title_rating": <0-10>, + "improved_title": "" + } + + - name: Validate and set SCRIPT_OUTPUT + if: steps.actor.outputs.is_repo_dev == 'true' + run: | + cat < ai_response.json + ${{ steps.ai-title-analysis.outputs.response }} + EOF + + # Validate JSON structure + jq -e ' + (keys | sort) == ["improved_ai_title_rating", "improved_rating", "improved_title"] and + (.improved_rating | type == "number" and . >= 0 and . <= 10) and + (.improved_ai_title_rating | type == "number" and . >= 0 and . <= 10) and + (.improved_title | type == "string") + ' ai_response.json + if [ $? -ne 0 ]; then + echo "Invalid AI response format" >&2 + cat ai_response.json >&2 + exit 1 + fi + # Parse JSON fields + IMPROVED_RATING=$(jq -r '.improved_rating' ai_response.json) + IMPROVED_TITLE=$(jq -r '.improved_title' ai_response.json) + # Limit comment length to 1000 characters + COMMENT=$(cat < /tmp/ai-title-comment.md + # Log input and output to the GitHub Step Summary + echo "### šŸ¤– AI PR Title Analysis" >> $GITHUB_STEP_SUMMARY + echo "### Input PR Title" >> $GITHUB_STEP_SUMMARY + echo '```bash' >> $GITHUB_STEP_SUMMARY + echo "${{ steps.sanitize_pr_title.outputs.pr_title }}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo '### AI Response (raw JSON)' >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + cat ai_response.json >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - name: Post comment on PR if needed + if: steps.actor.outputs.is_repo_dev == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + continue-on-error: true + with: + github-token: ${{ steps.setup-bot.outputs.token }} + script: | + const fs = require('fs'); + const body = fs.readFileSync('/tmp/ai-title-comment.md', 'utf8'); + const { GITHUB_REPOSITORY } = process.env; + const [owner, repo] = GITHUB_REPOSITORY.split('/'); + const issue_number = context.issue.number; + + const ratingMatch = body.match(/\*\*PR-Title Rating\*\*: (\d+)\/10/); + const rating = ratingMatch ? parseInt(ratingMatch[1], 10) : null; + + const expectedActor = "${{ steps.setup-bot.outputs.app-slug }}[bot]"; + const comments = await github.rest.issues.listComments({ owner, repo, issue_number }); + + const existing = comments.data.find(c => + c.user?.login === expectedActor && + c.body.includes("## šŸ¤– AI PR Title Suggestion") + ); + + if (rating === null) { + console.log("No rating found in AI response ā€“ skipping."); + return; + } + + if (rating <= 5) { + if (existing) { + await github.rest.issues.updateComment({ + owner, repo, + comment_id: existing.id, + body + }); + console.log("Updated existing suggestion comment."); + } else { + await github.rest.issues.createComment({ + owner, repo, issue_number, + body + }); + console.log("Created new suggestion comment."); + } + } else { + const praise = `## šŸ¤– AI PR Title Suggestion\n\nGreat job! The current PR title is clear and well-structured.\n\nāœ… No suggestions needed.\n\n---\n*Generated by GitHub Models AI*`; + + if (existing) { + await github.rest.issues.updateComment({ + owner, repo, + comment_id: existing.id, + body: praise + }); + console.log("Replaced suggestion with praise."); + } else { + console.log("Rating > 5 and no existing comment ā€“ skipping comment."); + } + } + + - name: is not repo dev + if: steps.actor.outputs.is_repo_dev != 'true' + run: | + exit 0 # Skip the AI title review for non-repo developers + + - name: Clean up + if: always() + run: | + rm -f pr.diff ai_response.json /tmp/ai-title-comment.md + echo "Cleaned up temporary files." + continue-on-error: true # Ensure cleanup runs even if previous steps fail