From 3cc948de5ce78de50e93c510c9921c52a1a54c02 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 31 May 2023 20:26:37 -0700 Subject: [PATCH 1/7] ci test --- .github/workflows/ci.yml | 115 +++++++++++++++++- .gitmodules | 2 +- tests/Auto-GPT-test-cassettes | 2 +- tests/integration/agent_factory.py | 4 +- .../basic_abilities/test_browse_website.py | 3 +- .../basic_abilities/test_write_file.py | 3 +- .../integration/challenges/current_score.json | 6 +- .../test_information_retrieval_challenge_a.py | 3 +- .../test_information_retrieval_challenge_b.py | 3 +- .../test_kubernetes_template_challenge_a.py | 2 + .../memory/test_memory_challenge_a.py | 3 +- .../memory/test_memory_challenge_b.py | 3 +- .../memory/test_memory_challenge_c.py | 3 +- .../memory/test_json_file_memory.py | 4 +- 14 files changed, 137 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5219b9826bd6..15582c6a94a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ concurrency: group: ${{ format('ci-{0}', github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }} cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} +env: + PR_NUM: ${{ github.event.pull_request.number }} + + jobs: lint: runs-on: ubuntu-latest @@ -73,6 +77,9 @@ jobs: python-version: ["3.10"] steps: + - name: Get the current date and time + run: echo "CURRENT_DATE_TIME=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_ENV + - name: Check out repository uses: actions/checkout@v3 with: @@ -82,6 +89,75 @@ jobs: submodules: true token: ${{ secrets.PAT_REVIEW }} + - name: Update Score + run: | + echo "update current score" >> tests/integration/challenges/current_score.json + + - name: Commit and push + run: | + new_branch="update-score-${{ env.PR_NUM }}-${{ env.CURRENT_DATE_TIME }}" + echo "NEW_BRANCH=$new_branch" >> $GITHUB_ENV + + git config --global user.email "action@github.com" + git config --global user.name "GitHub Action" + git add -A + git commit -m "Update score" + git checkout -b $new_branch + git push origin $new_branch +# - name: Install GitHub CLI +# run: | +# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 +# sudo apt-add-repository https://cli.github.com/packages +# sudo apt-get update +# sudo apt-get install gh + + - name: Authenticate with GitHub CLI + run: echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token + + - name: Set base repository + run: gh repo clone ${BASE_REPOSITORY} + + - name: Create Pull Request + run: | + echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token + gh pr create --title "Update Score In Pull Request Number ${{ env.PR_NUM }} ${{ env.CURRENT_DATE_TIME }}" \ + --head "${{ github.repository_owner }}:${{ env.NEW_BRANCH }}" \ + --base "${{ github.event.pull_request.user.login }}:${{ github.event.pull_request.head.ref }}" \ + --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \ + --repo ${{ github.repository }} + + - name: Make some changes + run: | + echo "This is a change" > change.txt + + - name: Commit and push changes + run: | + git config --global user.name "Auto-GPT-Bot" + git config --global user.email "github-bot@agpt.co" + git checkout -b source-branch + git add . + git commit -m "Add change.txt" + git push origin source-branch + + - name: Create Pull Request + env: + GITHUB_TOKEN: ${{ secrets.PAT }} # Make sure to replace this with your own PAT. + run: | + PR_BODY='{ + "title": "pull-request-title", + "head": "source-branch", + "base": "target-branch" + }' + + PR_RESPONSE=$(curl \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + https://api.github.com/repos/${{ github.event.pull_request.head.repo.full_name }}/pulls \ + -d "${PR_BODY}") + + echo "Pull Request Response -> ${PR_RESPONSE}" + - name: Use cassettes previously sent if: ${{ github.event_name == 'pull_request_target' }} run: | @@ -119,7 +195,7 @@ jobs: - name: Run pytest tests with coverage run: | - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term + pytest tests/integration/challenges/basic_abilities/test_write_file.py python tests/integration/challenges/utils/build_current_score.py env: CI: true @@ -127,9 +203,6 @@ jobs: AGENT_MODE: ${{ vars.AGENT_MODE }} AGENT_TYPE: ${{ vars.AGENT_TYPE }} - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - - name: Update cassette submodule to push target if push event if: ${{ github.event_name == 'push' }} run: | @@ -193,6 +266,40 @@ jobs: echo "DIFF_EXISTS=false" >> $GITHUB_ENV fi + - name: Run pytest tests with coverage + if: ${{ env.DIFF_EXISTS == 'true' }} + run: | + pytest -n auto tests/integration/challenges --beat-challenges --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term + python tests/integration/challenges/utils/build_current_score.py + env: + CI: true + PROXY: ${{ secrets.PROXY }} + AGENT_MODE: ${{ vars.AGENT_MODE }} + AGENT_TYPE: ${{ vars.AGENT_TYPE }} + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + + - name: Beat challenges - Update cassette submodule to submodule branch if PR event + if: ${{ env.DIFF_EXISTS == 'true' }} + run: | + new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" + + cd tests/Auto-GPT-test-cassettes + git config --global user.name "Auto-GPT-Bot" + git config --global user.email "github-bot@agpt.co" + git add . + + # Check if there are any changes + if ! git diff-index --quiet HEAD; then + git commit -m "Auto-update cassettes after merging PR #$pr_number (Beat challenges)" + git push -f origin HEAD:refs/heads/$new_branch + + else + echo "No changes to commit" + exit 0 + fi + - name: Apply or remove behaviour change label and comment if: ${{ github.event_name == 'pull_request_target' }} run: | diff --git a/.gitmodules b/.gitmodules index 55d09f842c8f..528c9ca4ae18 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "tests/Auto-GPT-test-cassettes"] path = tests/Auto-GPT-test-cassettes url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes - branch = master \ No newline at end of file + branch = ci-test-1 diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes index be280df43d6a..2017dcd8fe31 160000 --- a/tests/Auto-GPT-test-cassettes +++ b/tests/Auto-GPT-test-cassettes @@ -1 +1 @@ -Subproject commit be280df43d6a23b8074d9cba10d18ed8724a54c9 +Subproject commit 2017dcd8fe313403e90dde23563572d9f0439969 diff --git a/tests/integration/agent_factory.py b/tests/integration/agent_factory.py index 30d9cc13b2aa..a74b6533e6d9 100644 --- a/tests/integration/agent_factory.py +++ b/tests/integration/agent_factory.py @@ -81,8 +81,8 @@ def writer_agent(agent_test_config, memory_none: NoMemory, workspace: Workspace) ai_role="an AI designed to use the write_to_file command to write 'Hello World' into a file named \"hello_world.txt\" and then use the task_complete command to complete the task.", ai_goals=[ "Use the write_to_file command to write 'Hello World' into a file named \"hello_world.txt\".", - "Use the task_complete command to complete the task.", - "Do not use any other commands.", + "Use the task_complete command to complete the task!!!!", + "Do not use any other commands!", ], ) ai_config.command_registry = command_registry diff --git a/tests/integration/challenges/basic_abilities/test_browse_website.py b/tests/integration/challenges/basic_abilities/test_browse_website.py index 09e5ab2200a4..93e3ee241807 100644 --- a/tests/integration/challenges/basic_abilities/test_browse_website.py +++ b/tests/integration/challenges/basic_abilities/test_browse_website.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from tests.integration.challenges.challenge_decorator.challenge_decorator import ( @@ -15,7 +16,7 @@ @challenge def test_browse_website( browser_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, level_to_run: int, ) -> None: diff --git a/tests/integration/challenges/basic_abilities/test_write_file.py b/tests/integration/challenges/basic_abilities/test_write_file.py index cbbad514b6b6..393dbfd05d85 100644 --- a/tests/integration/challenges/basic_abilities/test_write_file.py +++ b/tests/integration/challenges/basic_abilities/test_write_file.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -17,7 +18,7 @@ @challenge def test_write_file( writer_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/current_score.json b/tests/integration/challenges/current_score.json index 726613991d48..519efe999dc1 100644 --- a/tests/integration/challenges/current_score.json +++ b/tests/integration/challenges/current_score.json @@ -17,8 +17,8 @@ }, "information_retrieval": { "information_retrieval_challenge_a": { - "max_level": 3, - "max_level_beaten": 1 + "max_level": 1, + "max_level_beaten": null }, "information_retrieval_challenge_b": { "max_level": 1, @@ -45,4 +45,4 @@ "max_level_beaten": 1 } } -} \ No newline at end of file +} diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py index 6b970e8b227d..2f61fef3902a 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py +++ b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.commands.file_operations import read_file from autogpt.config import Config @@ -19,7 +20,7 @@ def test_information_retrieval_challenge_a( information_retrieval_agents: Agent, monkeypatch: pytest.MonkeyPatch, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, config: Config, level_to_run: int, ) -> None: diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py index feac95a0f646..6461e13fbf4c 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py +++ b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py @@ -1,6 +1,7 @@ import contextlib import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -20,7 +21,7 @@ def test_information_retrieval_challenge_b( get_nobel_prize_agent: Agent, monkeypatch: pytest.MonkeyPatch, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, level_to_run: int, config: Config, ) -> None: diff --git a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py b/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py index 5fd280ac4bbb..ad658bae9183 100644 --- a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py +++ b/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py @@ -1,5 +1,6 @@ import pytest import yaml +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -21,6 +22,7 @@ def test_kubernetes_template_challenge_a( monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, + patched_api_requestor: MockerFixture, ) -> None: """ Test the challenge_a function in a given agent by mocking user inputs diff --git a/tests/integration/challenges/memory/test_memory_challenge_a.py b/tests/integration/challenges/memory/test_memory_challenge_a.py index 8919bf58c0eb..08f461bdd86d 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_a.py +++ b/tests/integration/challenges/memory/test_memory_challenge_a.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -15,7 +16,7 @@ @challenge def test_memory_challenge_a( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/memory/test_memory_challenge_b.py b/tests/integration/challenges/memory/test_memory_challenge_b.py index 5c28b330a3b4..c82763129aca 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_b.py +++ b/tests/integration/challenges/memory/test_memory_challenge_b.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -17,7 +18,7 @@ @challenge def test_memory_challenge_b( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/memory/test_memory_challenge_c.py b/tests/integration/challenges/memory/test_memory_challenge_c.py index 23c0217d6135..ab8ece105262 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_c.py +++ b/tests/integration/challenges/memory/test_memory_challenge_c.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -18,7 +19,7 @@ @challenge def test_memory_challenge_c( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/memory/test_json_file_memory.py b/tests/integration/memory/test_json_file_memory.py index 9134a06969f1..4de5819daabd 100644 --- a/tests/integration/memory/test_json_file_memory.py +++ b/tests/integration/memory/test_json_file_memory.py @@ -80,7 +80,9 @@ def test_json_memory_get(config: Config, memory_item: MemoryItem, mock_get_embed @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") -def test_json_memory_get_relevant(config: Config, patched_api_requestor: None) -> None: +def test_json_memory_get_relevant( + config: Config, patched_api_requestor: MockerFixture +) -> None: index = JSONFileMemory(config) mem1 = MemoryItem.from_text_file("Sample text", "sample.txt") mem2 = MemoryItem.from_text_file("Grocery list:\n- Pancake mix", "groceries.txt") From 368b1bfa4c4236d78192676c692daa5f7ea34471 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 31 May 2023 20:48:49 -0700 Subject: [PATCH 2/7] ci test --- .github/workflows/ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15582c6a94a7..cfd82889118c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,12 +111,6 @@ jobs: # sudo apt-get update # sudo apt-get install gh - - name: Authenticate with GitHub CLI - run: echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - - - name: Set base repository - run: gh repo clone ${BASE_REPOSITORY} - - name: Create Pull Request run: | echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token From 1abcfd40387624b4e70b6cdb2151d2e8e3ec71a7 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 31 May 2023 20:51:42 -0700 Subject: [PATCH 3/7] ci test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfd82889118c..ca53a16b4b3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: --head "${{ github.repository_owner }}:${{ env.NEW_BRANCH }}" \ --base "${{ github.event.pull_request.user.login }}:${{ github.event.pull_request.head.ref }}" \ --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \ - --repo ${{ github.repository }} + --repo ${{ github.event.pull_request.head.repo.full_name }} - name: Make some changes run: | From dd202743df215ec112acad77f2115534a0bfd01e Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 31 May 2023 20:54:50 -0700 Subject: [PATCH 4/7] ci test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca53a16b4b3d..f236bdb61598 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,7 @@ jobs: echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token gh pr create --title "Update Score In Pull Request Number ${{ env.PR_NUM }} ${{ env.CURRENT_DATE_TIME }}" \ --head "${{ github.repository_owner }}:${{ env.NEW_BRANCH }}" \ - --base "${{ github.event.pull_request.user.login }}:${{ github.event.pull_request.head.ref }}" \ + --base "${{ github.event.pull_request.head.ref }}" \ --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \ --repo ${{ github.event.pull_request.head.repo.full_name }} From 814c77667d1764ca40226b6fc9ffaf490613bb7a Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 31 May 2023 20:57:41 -0700 Subject: [PATCH 5/7] ci test --- .github/workflows/ci.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f236bdb61598..4d6edfa4f0b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,22 +100,16 @@ jobs: git config --global user.email "action@github.com" git config --global user.name "GitHub Action" - git add -A + git add tests/integration/challenges/current_score.json git commit -m "Update score" git checkout -b $new_branch git push origin $new_branch -# - name: Install GitHub CLI -# run: | -# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 -# sudo apt-add-repository https://cli.github.com/packages -# sudo apt-get update -# sudo apt-get install gh - name: Create Pull Request run: | echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token gh pr create --title "Update Score In Pull Request Number ${{ env.PR_NUM }} ${{ env.CURRENT_DATE_TIME }}" \ - --head "${{ github.repository_owner }}:${{ env.NEW_BRANCH }}" \ + --head "${{ env.NEW_BRANCH }}" \ --base "${{ github.event.pull_request.head.ref }}" \ --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \ --repo ${{ github.event.pull_request.head.repo.full_name }} From a9ea8aac28bda043f43bd313a2c8abf02593ff11 Mon Sep 17 00:00:00 2001 From: Merwane Hamadi Date: Wed, 31 May 2023 20:57:54 -0700 Subject: [PATCH 6/7] ci test From 93565b4ca1275e1d7a506a36ad92824ea769ff47 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 1 Jun 2023 03:58:18 +0000 Subject: [PATCH 7/7] Update score --- tests/integration/challenges/current_score.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/challenges/current_score.json b/tests/integration/challenges/current_score.json index 519efe999dc1..5186b2a7670d 100644 --- a/tests/integration/challenges/current_score.json +++ b/tests/integration/challenges/current_score.json @@ -46,3 +46,4 @@ } } } +update current score