diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5219b9826bd6..03923da53c4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,7 +119,7 @@ jobs: - name: Run pytest tests with coverage run: | - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term + pytest tests/integration/challenges/basic_abilities/test_write_file.py python tests/integration/challenges/utils/build_current_score.py env: CI: true @@ -127,9 +127,6 @@ jobs: AGENT_MODE: ${{ vars.AGENT_MODE }} AGENT_TYPE: ${{ vars.AGENT_TYPE }} - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - - name: Update cassette submodule to push target if push event if: ${{ github.event_name == 'push' }} run: | @@ -165,6 +162,9 @@ jobs: exit 0 fi + + + - name: Update cassette submodule to submodule branch if PR event if: ${{ github.event_name == 'pull_request_target' }} run: | @@ -193,6 +193,40 @@ jobs: echo "DIFF_EXISTS=false" >> $GITHUB_ENV fi + - name: Run pytest tests with coverage + if: ${{ env.DIFF_EXISTS == 'true' }} + run: | + pytest -n auto tests/integration/challenges --beat-challenges --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term + python tests/integration/challenges/utils/build_current_score.py + env: + CI: true + PROXY: ${{ secrets.PROXY }} + AGENT_MODE: ${{ vars.AGENT_MODE }} + AGENT_TYPE: ${{ vars.AGENT_TYPE }} + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + + - name: Beat challenges - Update cassette submodule to submodule branch if PR event + if: ${{ env.DIFF_EXISTS == 'true' }} + run: | + new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" + + cd tests/Auto-GPT-test-cassettes + git config --global user.name "Auto-GPT-Bot" + git config --global user.email "github-bot@agpt.co" + git add . + + # Check if there are any changes + if ! git diff-index --quiet HEAD; then + git commit -m "Auto-update cassettes after merging PR #$pr_number (Beat challenges)" + git push -f origin HEAD:refs/heads/$new_branch + + else + echo "No changes to commit" + exit 0 + fi + - name: Apply or remove behaviour change label and comment if: ${{ github.event_name == 'pull_request_target' }} run: | diff --git a/tests/integration/agent_factory.py b/tests/integration/agent_factory.py index 30d9cc13b2aa..f57162705d84 100644 --- a/tests/integration/agent_factory.py +++ b/tests/integration/agent_factory.py @@ -81,7 +81,7 @@ def writer_agent(agent_test_config, memory_none: NoMemory, workspace: Workspace) ai_role="an AI designed to use the write_to_file command to write 'Hello World' into a file named \"hello_world.txt\" and then use the task_complete command to complete the task.", ai_goals=[ "Use the write_to_file command to write 'Hello World' into a file named \"hello_world.txt\".", - "Use the task_complete command to complete the task.", + "Use the task_complete command to complete the task!!!!!", "Do not use any other commands.", ], ) diff --git a/tests/integration/challenges/current_score.json b/tests/integration/challenges/current_score.json index 726613991d48..ab2ea4123e06 100644 --- a/tests/integration/challenges/current_score.json +++ b/tests/integration/challenges/current_score.json @@ -1,48 +1,8 @@ { "basic_abilities": { - "browse_website": { - "max_level": 1, - "max_level_beaten": 1 - }, "write_file": { "max_level": 1, "max_level_beaten": 1 } - }, - "debug_code": { - "debug_code_challenge_a": { - "max_level": 1, - "max_level_beaten": 1 - } - }, - "information_retrieval": { - "information_retrieval_challenge_a": { - "max_level": 3, - "max_level_beaten": 1 - }, - "information_retrieval_challenge_b": { - "max_level": 1, - "max_level_beaten": 1 - } - }, - "kubernetes": { - "kubernetes_template_challenge_a": { - "max_level": 1, - "max_level_beaten": null - } - }, - "memory": { - "memory_challenge_a": { - "max_level": 3, - "max_level_beaten": 3 - }, - "memory_challenge_b": { - "max_level": 5, - "max_level_beaten": null - }, - "memory_challenge_c": { - "max_level": 5, - "max_level_beaten": 1 - } } } \ No newline at end of file