From cdb489636b03db0fbe6f0d2025d9d17893c248a1 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Tue, 6 Jun 2023 12:05:46 -0700 Subject: [PATCH 1/7] CI test --- .github/workflows/ci.yml | 99 ++++++++++++++++++++++++++++------- .gitmodules | 2 +- tests/Auto-GPT-test-cassettes | 2 +- 3 files changed, 81 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f96529932b3..dfb4dde61224 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,22 +148,38 @@ jobs: - name: Run pytest with coverage run: | - pytest -n auto --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \ - tests/unit tests/integration tests/challenges - python tests/challenges/utils/build_current_score.py + echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml + echo "test" >> tests/challenges/current_score.json env: CI: true PROXY: ${{ secrets.PROXY }} AGENT_MODE: ${{ vars.AGENT_MODE }} AGENT_TYPE: ${{ vars.AGENT_TYPE }} + - id: get_cassettes_diff + name: Identify differences between cassettes of current PR and target branch + if: ${{ startsWith(github.event_name, 'pull_request') }} + run: | + cd tests/Auto-GPT-test-cassettes + cassette_diff=$(git diff --name-only --diff-filter=M -- '**/*.yaml') + + if [ -n "$cassette_diff" ]; then + echo "updated=true" >> $GITHUB_OUTPUT + fi + + - name: Attempt to beat Challenges if cassettes changed + if: ${{ startsWith(github.event_name, 'pull_request') }} && "${{ steps.get_cassettes_diff.outputs.updated }}" == "true" + run: | + echo "test" >> tests/Auto-GPT-test-cassettes/test_write_file/test_write_file.yaml + echo "test" >> tests/challenges/current_score.json + - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 - id: setup_git_auth name: Set up git token authentication - # Cassettes may be pushed even when tests fail - if: success() || failure() + # Cassettes may be pushed even when tests fail. + if: always() && (startsWith(github.event_name, 'pull_request') || success()) run: | config_key="http.${{ github.server_url }}/.extraheader" base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0) @@ -177,15 +193,23 @@ jobs: echo "config_key=$config_key" >> $GITHUB_OUTPUT - - name: Push updated challenge scores - if: github.event_name == 'push' + - id: push_challenge_scores + name: Push updated challenge scores run: | score_file="tests/challenges/current_score.json" if ! git diff --quiet $score_file; then git add $score_file git commit -m "Update challenge scores" - git push origin HEAD:${{ github.ref_name }} + if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then + updated_score_branch="update-score-${{ github.event.pull_request.number }}" + git checkout -b $updated_score_branch + git remote add base_repo https://github.com/${{ github.event.pull_request.base.repo.owner.login }}/${{ github.event.pull_request.base.repo.name }}.git + git push -f base_repo $updated_score_branch + echo "updated_score_branch=$updated_score_branch" >> $GITHUB_OUTPUT + else + git push origin HEAD:${{ github.ref }} + fi else echo "The challenge scores didn't change." fi @@ -193,7 +217,7 @@ jobs: - id: push_cassettes name: Push updated cassettes # For pull requests, push updated cassettes even when tests fail - if: github.event_name == 'push' || success() || failure() + if: always() && (startsWith(github.event_name, 'pull_request') || success()) run: | if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then is_pull_request=true @@ -214,9 +238,7 @@ jobs: git commit -m "Update cassette submodule" git push origin HEAD:$cassette_branch fi - echo "updated=true" >> $GITHUB_OUTPUT else - echo "updated=false" >> $GITHUB_OUTPUT echo "No cassette changes to commit" fi @@ -225,22 +247,59 @@ jobs: run: | git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' + + echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token + + - id: create_pr + name: Create Pull Request if score changed, if not cleanup branch and PR + if: ${{ startsWith(github.event_name, 'pull_request')}} + run: | + base_repo_owner=${{ github.event.pull_request.base.repo.owner.login }} + base_repo_name=${{ github.event.pull_request.base.repo.name }} + updated_score_branch=${{steps.push_challenge_scores.outputs.updated_score_branch}} + pr_number=$(gh api repos/$base_repo_owner/$base_repo_name/pulls -q ".[] | select(.head.ref == \"$updated_score_branch\") | .number") + if [ -z "$updated_score_branch" ]; then + if [ -n "$pr_number" ]; then + gh api -X PATCH repos/$base_repo_owner/$base_repo_name/pulls/$pr_number -f state=closed + fi + # Delete branch + gh api -X DELETE repos/$base_repo_owner/$base_repo_name/git/refs/heads/$updated_score_branch || echo "Branch not found or already deleted." + echo "Score didn't change." + exit 0 + fi + if [ -z "$pr_number" ] || [ "$(gh api repos/$base_repo_owner/$base_repo_name/pulls/$pr_number --jq ".state")" = "closed" ]; then + pr_url=$(gh pr create --title "Update Score In Pull Request Number ${{ github.event.pull_request.number }}" \ + --head "${{ github.event.pull_request.base.repo.owner.login }}:$updated_score_branch" \ + --base "${{ github.event.pull_request.head.ref }}" \ + --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \ + --repo ${{ github.event.pull_request.head.repo.full_name }}) + else + pr_url=$(gh pr view $pr_number --json url --jq '.url') + fi + echo "pr_url=$pr_url" >> $GITHUB_OUTPUT - name: Apply "behaviour change" label and comment on PR if: ${{ startsWith(github.event_name, 'pull_request') }} run: | - PR_NUMBER=${{ github.event.pull_request.number }} - TOKEN=${{ secrets.PAT_REVIEW }} REPO=${{ github.repository }} - - if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then + COMMENT_BODY="" + SCORE_CHANGED_MESSAGE="The pipeline will fail until you merge the updated score: ${{ steps.create_pr.outputs.pr_url }}" + if [[ "${{ steps.get_cassettes_diff.outputs.updated }}" == "true" ]]; then echo "Adding label and comment..." curl -X POST \ - -H "Authorization: Bearer $TOKEN" \ + -H "Authorization: Bearer ${{ secrets.PAT_REVIEW }}" \ -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels \ + https://api.github.com/repos/$REPO/issues/${{ github.event.pull_request.number }}/labels \ -d '{"labels":["behaviour change"]}' - - echo $TOKEN | gh auth login --with-token - gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged." + COMMENT_BODY="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged. " + fi + if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then + COMMENT_BODY="$COMMENT_BODY$SCORE_CHANGED_MESSAGE" + fi + if [ ! -z "$COMMENT_BODY" ]; then + gh api repos/$REPO/issues/${{ github.event.pull_request.number }}/comments -X POST -F body="$COMMENT_BODY" + fi + if [[ -n "${{ steps.create_pr.outputs.pr_url }}" ]]; then + echo "$SCORE_CHANGED_MESSAGE" + exit 1 fi diff --git a/.gitmodules b/.gitmodules index 55d09f842c8f..eb14002855be 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "tests/Auto-GPT-test-cassettes"] path = tests/Auto-GPT-test-cassettes url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes - branch = master \ No newline at end of file + branch = ci-test-2 diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes index 38ecb0145aa3..b5d1e20afdd1 160000 --- a/tests/Auto-GPT-test-cassettes +++ b/tests/Auto-GPT-test-cassettes @@ -1 +1 @@ -Subproject commit 38ecb0145aa3a88e2eb5f04a556146614a2882e1 +Subproject commit b5d1e20afdd18de8097b8d9bf89746bc0ac7fc07 From 17b3d3ab14b06955c89617372a330313be4e9cfd Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Tue, 6 Jun 2023 12:10:23 -0700 Subject: [PATCH 2/7] CI test --- tests/Auto-GPT-test-cassettes | 2 +- tests/challenges/basic_abilities/test_write_file.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes index b5d1e20afdd1..ca0dbdd27fde 160000 --- a/tests/Auto-GPT-test-cassettes +++ b/tests/Auto-GPT-test-cassettes @@ -1 +1 @@ -Subproject commit b5d1e20afdd18de8097b8d9bf89746bc0ac7fc07 +Subproject commit ca0dbdd27fdea959d661af5a9c27cd4ecb3b505d diff --git a/tests/challenges/basic_abilities/test_write_file.py b/tests/challenges/basic_abilities/test_write_file.py index 033f76e6bf33..fc0d6b240ac2 100644 --- a/tests/challenges/basic_abilities/test_write_file.py +++ b/tests/challenges/basic_abilities/test_write_file.py @@ -21,6 +21,7 @@ def test_write_file( config: Config, level_to_run: int, ) -> None: + # dummy file_path = str(writer_agent.workspace.get_path("hello_world.txt")) run_interaction_loop(monkeypatch, writer_agent, CYCLE_COUNT) From c559b44d905ecf5119d39b377aeb0582c0fbfe86 Mon Sep 17 00:00:00 2001 From: Auto-GPT-Bot Date: Tue, 6 Jun 2023 19:11:41 +0000 Subject: [PATCH 3/7] Update challenge scores --- tests/challenges/current_score.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/challenges/current_score.json b/tests/challenges/current_score.json index bd91afcb195a..036e7c67b159 100644 --- a/tests/challenges/current_score.json +++ b/tests/challenges/current_score.json @@ -45,4 +45,5 @@ "max_level_beaten": 1 } } -} \ No newline at end of file +}test +test From cf07dc66a4c3f52d60c2d9abb9446530ae5127a5 Mon Sep 17 00:00:00 2001 From: Auto-GPT-Bot Date: Tue, 6 Jun 2023 19:11:43 +0000 Subject: [PATCH 4/7] Update cassette submodule --- tests/Auto-GPT-test-cassettes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes index ca0dbdd27fde..469f9f7bc59a 160000 --- a/tests/Auto-GPT-test-cassettes +++ b/tests/Auto-GPT-test-cassettes @@ -1 +1 @@ -Subproject commit ca0dbdd27fdea959d661af5a9c27cd4ecb3b505d +Subproject commit 469f9f7bc59a6470168ff0c7013cd773427d0c19 From 3f9d88e642ddaa3b5be83f16d9e3ca30db41bc75 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Tue, 6 Jun 2023 12:12:36 -0700 Subject: [PATCH 5/7] CI test From badf243998f8d0a2df7dee7bd4dd785cb895c4b1 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Tue, 6 Jun 2023 12:16:12 -0700 Subject: [PATCH 6/7] CI test From 0db74fb29fd954f91217960de16f92308102c1a5 Mon Sep 17 00:00:00 2001 From: Auto-GPT-Bot Date: Tue, 6 Jun 2023 19:17:24 +0000 Subject: [PATCH 7/7] Update challenge scores --- tests/challenges/current_score.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/challenges/current_score.json b/tests/challenges/current_score.json index 036e7c67b159..fb3a18c1415d 100644 --- a/tests/challenges/current_score.json +++ b/tests/challenges/current_score.json @@ -47,3 +47,5 @@ } }test test +test +test