diff --git a/.github/workflows/add-cassettes.yml b/.github/workflows/add-cassettes.yml new file mode 100644 index 000000000000..cbb548626797 --- /dev/null +++ b/.github/workflows/add-cassettes.yml @@ -0,0 +1,26 @@ +name: Merge and Commit Cassettes + +on: + pull_request_target: + types: + - closed + +jobs: + update-cassettes: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 # This is necessary to fetch all branches and tags + + - name: Download diff artifact + uses: dawidd6/action-download-artifact@v2 + with: + github_token: ${{secrets.GITHUB_TOKEN}} + workflow: Python CI + name: cassette-diff + path: . + pr: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a08ecb02114b..e1d63f15c795 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,12 +3,12 @@ name: Python CI on: push: branches: [ master ] - pull_request: + pull_request_target: branches: [ master, stable ] concurrency: group: ${{ format('ci-{0}', github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} jobs: lint: @@ -43,12 +43,7 @@ jobs: test: permissions: - # Gives the action the necessary permissions for publishing new - # comments in pull requests. pull-requests: write - # Gives the action the necessary permissions for pushing data to the - # python-coverage-comment-action branch, and for editing existing - # comments (to avoid publishing multiple comments in the same PR) contents: write runs-on: ubuntu-latest strategy: @@ -58,6 +53,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -71,9 +68,26 @@ jobs: - name: Run unittest tests with coverage run: | - pytest --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term + pytest -s tests/integration/goal_oriented/test_write_file.py env: CI: true + PROXY: https://us-central1-prod-benchmarks-core.cloudfunctions.net/proxy_function + + - name: Stage new files + run: | + git add . + + - name: Create diff and save to file + run: | + git diff --cached > cassette-diff-${{ github.run_number }}-${{ github.sha }}.txt || echo "No diff to create" + + - name: Create new branch and commit changes + run: | + git checkout -b Auto-GPT-cassettes || git checkout Auto-GPT-cassettes + git config user.email "github-actions@github.com" + git config user.name "GitHub Actions" + git add cassette-diff-${{ github.run_number }}-${{ github.sha }}.txt + git commit -m "Add cassette diff" + git push origin Auto-GPT-cassettes + - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index e066641f04fb..9af6680cf5be 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -79,3 +79,5 @@ jobs: run: .github/workflows/scripts/docker-release-summary.sh >> $GITHUB_STEP_SUMMARY continue-on-error: true +# rand +# rand diff --git a/autogpt/prompts/prompt.py b/autogpt/prompts/prompt.py index 5da01c44ffea..b20bd5a1a8ee 100644 --- a/autogpt/prompts/prompt.py +++ b/autogpt/prompts/prompt.py @@ -10,9 +10,7 @@ CFG = Config() -DEFAULT_TRIGGERING_PROMPT = ( - "Determine which next command to use, and respond using the format specified above:" -) +DEFAULT_TRIGGERING_PROMPT = "Determindfsdfse which next command to use, and respond using the format specified above:" def build_default_prompt_generator() -> PromptGenerator: diff --git a/cassette-diff-353-f75fcc72ce20b5cacc240db20e6d35c07c1724bf.txt b/cassette-diff-353-f75fcc72ce20b5cacc240db20e6d35c07c1724bf.txt new file mode 100644 index 000000000000..d10fb8375c11 --- /dev/null +++ b/cassette-diff-353-f75fcc72ce20b5cacc240db20e6d35c07c1724bf.txt @@ -0,0 +1,116 @@ +diff --git a/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml b/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml +index ef390a5..2bde595 100644 +--- a/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml ++++ b/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml +@@ -690,4 +690,111 @@ interactions: + status: + code: 200 + message: OK ++- request: ++ body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "system", "content": "You ++ are write_to_file-GPT, an AI designed to use the write_to_file command to write ++ ''Hello World'' into a file named \"hello_world.txt\" and then use the task_complete ++ command to complete the task.\nYour decisions must always be made independently ++ without seeking user assistance. Play to your strengths as an LLM and pursue ++ simple strategies with no legal complications.\n\nGOALS:\n\n1. Use the write_to_file ++ command to write ''Hello World'' into a file named \"hello_world.txt\".\n2. ++ Use the task_complete command to complete the task.\n3. Do not use andfsfdsy ++ other commands.\n\n\nConstraints:\n1. ~4000 word limit for short term memory. ++ Your short term memory is short, so immediately save important information to ++ files.\n2. If you are unsure how you previously did something or want to recall ++ past events, thinking about similar events will help you remember.\n3. No user ++ assistance\n4. Exclusively use the commands listed in double quotes e.g. \"command ++ name\"\n\nCommands:\n1. append_to_file: Append to file, args: \"filename\": ++ \"\", \"text\": \"\"\n2. delete_file: Delete file, args: \"filename\": ++ \"\"\n3. list_files: List Files in Directory, args: \"directory\": ++ \"\"\n4. read_file: Read file, args: \"filename\": \"\"\n5. ++ write_to_file: Write to file, args: \"filename\": \"\", \"text\": ++ \"\"\n6. delete_agent: Delete GPT Agent, args: \"key\": \"\"\n7. ++ get_hyperlinks: Get text summary, args: \"url\": \"\"\n8. get_text_summary: ++ Get text summary, args: \"url\": \"\", \"question\": \"\"\n9. ++ list_agents: List GPT Agents, args: () -> str\n10. message_agent: Message GPT ++ Agent, args: \"key\": \"\", \"message\": \"\"\n11. start_agent: ++ Start GPT Agent, args: \"name\": \"\", \"task\": \"\", ++ \"prompt\": \"\"\n12. task_complete: Task Complete (Shutdown), args: ++ \"reason\": \"\"\n\nResources:\n1. Internet access for searches and ++ information gathering.\n2. Long Term memory management.\n3. GPT-3.5 powered ++ Agents for delegation of simple tasks.\n4. File output.\n\nPerformance Evaluation:\n1. ++ Continuously review and analyze your actions to ensure you are performing to ++ the best of your abilities.\n2. Constructively self-criticize your big-picture ++ behavior constantly.\n3. Reflect on past decisions and strategies to refine ++ your approach.\n4. Every command has a cost, so be smart and efficient. Aim ++ to complete tasks in the least number of steps.\n5. Write all code to a file.\n\nYou ++ should only respond in JSON format as described below \nResponse Format: \n{\n \"thoughts\": ++ {\n \"text\": \"thought\",\n \"reasoning\": \"reasoning\",\n \"plan\": ++ \"- short bulleted\\n- list that conveys\\n- long-term plan\",\n \"criticism\": ++ \"constructive self-criticism\",\n \"speak\": \"thoughts summary to say ++ to user\"\n },\n \"command\": {\n \"name\": \"command name\",\n \"args\": ++ {\n \"arg name\": \"value\"\n }\n }\n} \nEnsure the response ++ can be parsed by Python json.loads"}, {"role": "system", "content": "The current ++ time and date is Tue Jan 1 00:00:00 2000"}, {"role": "system", "content": "This ++ reminds you of these events from your past: \nI was created and nothing new ++ has happened."}, {"role": "user", "content": "Determine which next command to ++ use, and respond using the format specified above:"}, {"role": "assistant", ++ "content": "{\n \"thoughts\": {\n \"text\": \"I need to use the write_to_file ++ command to write ''Hello World'' into a file named ''hello_world.txt''.\",\n \"reasoning\": ++ \"The task requires me to write ''Hello World'' into a file named ''hello_world.txt'', ++ and the write_to_file command is the appropriate command for this task.\",\n \"plan\": ++ \"- Use the write_to_file command to write ''Hello World'' into a file named ++ ''hello_world.txt''.\",\n \"criticism\": \"I need to ensure that I use ++ the correct file name and text to write to the file.\",\n \"speak\": ++ \"I will use the write_to_file command to write ''Hello World'' into a file ++ named ''hello_world.txt''.\"\n },\n \"command\": {\n \"name\": ++ \"write_to_file\",\n \"args\": {\n \"filename\": \"hello_world.txt\",\n \"text\": ++ \"Hello World\"\n }\n }\n}"}, {"role": "system", "content": "Command ++ write_to_file returned: File written to successfully."}, {"role": "user", "content": ++ "Determine which next command to use, and respond using the format specified ++ above:"}], "temperature": 0, "max_tokens": 0}' ++ headers: ++ Accept: ++ - '*/*' ++ Accept-Encoding: ++ - gzip, deflate ++ Connection: ++ - keep-alive ++ Content-Length: ++ - '4660' ++ Content-Type: ++ - application/json ++ method: POST ++ uri: https://api.openai.com/v1/chat/completions ++ response: ++ body: ++ string: '{"id":"chatcmpl-7FMsZlCgaqEI6ebydODAT47LprpFA","object":"chat.completion","created":1683897223,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":1031,"completion_tokens":201,"total_tokens":1232},"choices":[{"message":{"role":"assistant","content":"{\n \"thoughts\": ++ {\n \"text\": \"I have successfully written ''Hello World'' into a ++ file named ''hello_world.txt''. The next step is to use the task_complete ++ command to complete the task.\",\n \"reasoning\": \"The task requires ++ me to write ''Hello World'' into a file named ''hello_world.txt'' and then ++ complete the task. Since I have successfully written to the file, the next ++ step is to use the task_complete command to complete the task.\",\n \"plan\": ++ \"- Use the task_complete command to complete the task.\",\n \"criticism\": ++ \"I need to ensure that I provide a valid reason for completing the task.\",\n \"speak\": ++ \"I will use the task_complete command to complete the task.\"\n },\n \"command\": ++ {\n \"name\": \"task_complete\",\n \"args\": {\n \"reason\": ++ \"I have successfully written ''Hello World'' into a file named ''hello_world.txt''.\"\n }\n }\n}"},"finish_reason":"stop","index":0}]}' ++ headers: ++ Alt-Svc: ++ - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 ++ - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 ++ Content-Length: ++ - '1221' ++ Content-Type: ++ - application/json ++ Date: ++ - Fri, 12 May 2023 13:14:02 GMT ++ Function-Execution-Id: ++ - ea42yneiu06g ++ Server: ++ - Google Frontend ++ X-Cloud-Trace-Context: ++ - 78cd3f6b10f6ed7cdd88e530a2abe3ee;o=1 ++ X-Powered-By: ++ - Express ++ status: ++ code: 200 ++ message: OK + version: 1 diff --git a/tests/conftest.py b/tests/conftest.py index da00058b7fe2..6e6f0ad380af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import pytest @@ -9,6 +10,8 @@ pytest_plugins = ["tests.integration.agent_factory"] +PROXY = os.environ.get("PROXY") + @pytest.fixture() def workspace_root(tmp_path: Path) -> Path: diff --git a/tests/integration/agent_factory.py b/tests/integration/agent_factory.py index b6168034aae0..24ee38ea9ffb 100644 --- a/tests/integration/agent_factory.py +++ b/tests/integration/agent_factory.py @@ -89,7 +89,7 @@ def writer_agent(agent_test_config, memory_none: NoMemory, workspace: Workspace) ai_goals=[ "Use the write_to_file command to write 'Hello World' into a file named \"hello_world.txt\".", "Use the task_complete command to complete the task.", - "Do not use any other commands.", + "Do not use andfsfdsy other commands.", ], ) ai_config.command_registry = command_registry diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py index a5f8fb4ce33b..b96e811adb9d 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py +++ b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py @@ -26,7 +26,7 @@ def input_generator(input_sequence: list) -> Generator[str, None, None]: @requires_api_key("OPENAI_API_KEY") @run_multiple_times(3) def test_information_retrieval_challenge_a( - get_company_revenue_agent, monkeypatch + get_company_revenue_agent, monkeypatch, patched_api_requestor ) -> None: """ Test the challenge_a function in a given agent by mocking user inputs and checking the output file content. diff --git a/tests/integration/challenges/memory/test_memory_challenge_a.py b/tests/integration/challenges/memory/test_memory_challenge_a.py index 895fc8febe3e..ab859a6c0914 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_a.py +++ b/tests/integration/challenges/memory/test_memory_challenge_a.py @@ -13,7 +13,7 @@ @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") def test_memory_challenge_a( - memory_management_agent: Agent, user_selected_level: int + memory_management_agent: Agent, user_selected_level: int, patched_api_requestor ) -> None: """ The agent reads a file containing a task_id. Then, it reads a series of other files. diff --git a/tests/integration/challenges/memory/test_memory_challenge_b.py b/tests/integration/challenges/memory/test_memory_challenge_b.py index 628b49893f90..21d46b38d6a8 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_b.py +++ b/tests/integration/challenges/memory/test_memory_challenge_b.py @@ -14,7 +14,7 @@ @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") def test_memory_challenge_b( - memory_management_agent: Agent, user_selected_level: int + memory_management_agent: Agent, user_selected_level: int, patched_api_requestor ) -> None: """ The agent reads a series of files, each containing a task_id and noise. After reading 'n' files, diff --git a/tests/integration/challenges/memory/test_memory_challenge_c.py b/tests/integration/challenges/memory/test_memory_challenge_c.py index edd3efe00ab8..634a24a3e5d5 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_c.py +++ b/tests/integration/challenges/memory/test_memory_challenge_c.py @@ -14,7 +14,7 @@ @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") def test_memory_challenge_c( - memory_management_agent: Agent, user_selected_level: int + memory_management_agent: Agent, user_selected_level: int, patched_api_requestor ) -> None: """ Instead of reading task Ids from files as with the previous challenges, the agent now must remember diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 009287021338..2ff867cf8509 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,7 +1,9 @@ import os +import openai import pytest +from tests.conftest import PROXY from tests.vcr.vcr_filter import before_record_request, before_record_response @@ -17,5 +19,25 @@ def vcr_config(): "X-OpenAI-Client-User-Agent", "User-Agent", ], - "match_on": ["method", "uri", "body"], + "match_on": ["method", "body"], } + + +def patch_api_base(requestor): + new_api_base = f"{PROXY}/v1" + requestor.api_base = new_api_base + return requestor + + +@pytest.fixture +def patched_api_requestor(mocker): + original_init = openai.api_requestor.APIRequestor.__init__ + + def patched_init(requestor, *args, **kwargs): + original_init(requestor, *args, **kwargs) + patch_api_base(requestor) + + if PROXY: + mocker.patch("openai.api_requestor.APIRequestor.__init__", new=patched_init) + + return mocker diff --git a/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml b/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml index 42160ea2b382..2bde595d7774 100644 --- a/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml +++ b/tests/integration/goal_oriented/cassettes/test_write_file/test_write_file.yaml @@ -573,4 +573,228 @@ interactions: status: code: 200 message: OK +- request: + body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "system", "content": "You + are write_to_file-GPT, an AI designed to use the write_to_file command to write + ''Hello World'' into a file named \"hello_world.txt\" and then use the task_complete + command to complete the task.\nYour decisions must always be made independently + without seeking user assistance. Play to your strengths as an LLM and pursue + simple strategies with no legal complications.\n\nGOALS:\n\n1. Use the write_to_file + command to write ''Hello World'' into a file named \"hello_world.txt\".\n2. + Use the task_complete command to complete the task.\n3. Do not use andfsfdsy + other commands.\n\n\nConstraints:\n1. ~4000 word limit for short term memory. + Your short term memory is short, so immediately save important information to + files.\n2. If you are unsure how you previously did something or want to recall + past events, thinking about similar events will help you remember.\n3. No user + assistance\n4. Exclusively use the commands listed in double quotes e.g. \"command + name\"\n\nCommands:\n1. append_to_file: Append to file, args: \"filename\": + \"\", \"text\": \"\"\n2. delete_file: Delete file, args: \"filename\": + \"\"\n3. list_files: List Files in Directory, args: \"directory\": + \"\"\n4. read_file: Read file, args: \"filename\": \"\"\n5. + write_to_file: Write to file, args: \"filename\": \"\", \"text\": + \"\"\n6. delete_agent: Delete GPT Agent, args: \"key\": \"\"\n7. + get_hyperlinks: Get text summary, args: \"url\": \"\"\n8. get_text_summary: + Get text summary, args: \"url\": \"\", \"question\": \"\"\n9. + list_agents: List GPT Agents, args: () -> str\n10. message_agent: Message GPT + Agent, args: \"key\": \"\", \"message\": \"\"\n11. start_agent: + Start GPT Agent, args: \"name\": \"\", \"task\": \"\", + \"prompt\": \"\"\n12. task_complete: Task Complete (Shutdown), args: + \"reason\": \"\"\n\nResources:\n1. Internet access for searches and + information gathering.\n2. Long Term memory management.\n3. GPT-3.5 powered + Agents for delegation of simple tasks.\n4. File output.\n\nPerformance Evaluation:\n1. + Continuously review and analyze your actions to ensure you are performing to + the best of your abilities.\n2. Constructively self-criticize your big-picture + behavior constantly.\n3. Reflect on past decisions and strategies to refine + your approach.\n4. Every command has a cost, so be smart and efficient. Aim + to complete tasks in the least number of steps.\n5. Write all code to a file.\n\nYou + should only respond in JSON format as described below \nResponse Format: \n{\n \"thoughts\": + {\n \"text\": \"thought\",\n \"reasoning\": \"reasoning\",\n \"plan\": + \"- short bulleted\\n- list that conveys\\n- long-term plan\",\n \"criticism\": + \"constructive self-criticism\",\n \"speak\": \"thoughts summary to say + to user\"\n },\n \"command\": {\n \"name\": \"command name\",\n \"args\": + {\n \"arg name\": \"value\"\n }\n }\n} \nEnsure the response + can be parsed by Python json.loads"}, {"role": "system", "content": "The current + time and date is Tue Jan 1 00:00:00 2000"}, {"role": "user", "content": "Determine + which next command to use, and respond using the format specified above:"}], + "temperature": 0, "max_tokens": 0}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '3412' + Content-Type: + - application/json + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA7yTS2/bMBCE7/0Vi734Qht2lTiObj00aC69NA+0UWEw1NpiTXFVclW7MPTfC0l+ + JCp6KtrrzmjmI7nao80xRVNoMWXlxlc31w/F44+Pn1Y3Pz9v3s3py/urGLZ3tngI96iQn7+RkcMX + E8Nl5Ugse1RoAmmhHNPZfJEsLi4XSaKw5JwcpriuZJxMLsdSh2ceT5PpDBXWUa8J0z1WgctKlsIb + 8hHTq3mi8Jx9mr+dzhQKi3an0fX8olFoCraGIqZPeywpHmMDO8IUdYw2ivbSQrIX8u0B9pkHAMhQ + Cq7XhcQMUzgMDwLtpB1meAueKAdhqCOBFATbYIWWwsuVdQSGy1L7ztAJMPpAzjE8cnD5CKwXBg2d + 1euSchgVrb7ctvpEdjKaZKhedgfSkb316x7griAQHTcQ6HttA0Uo6S/aFHS0fzyHjZ2oqypwFayW + s7TiAFK0Bh03Q+rKad8Dj+H+P92UCVassbEcPhX5WIeWQQvcnh7OcAhk5FzRXwXt5Mwk3Flby7At + VqQ3x6atde5fbkRf3Kjjph5Cf1vUNqFnegUxQNdhPdzxXmi954gBx6uQ4X/x4kxH2o74AJ75BhuF + K+ttLJb9SmOKUbhChdbntMN02nxt3vwCAAD//wMApFuuI4cEAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 7c5e050a3e7497eb-SJC + Cache-Control: + - no-cache, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 11 May 2023 22:57:24 GMT + Server: + - cloudflare + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400 + openai-model: + - gpt-3.5-turbo-0301 + openai-organization: + - user-adtx4fhfg1qsiyzdoaxciooj + openai-processing-ms: + - '11428' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=15724800; includeSubDomains + x-ratelimit-limit-requests: + - '3500' + x-ratelimit-limit-tokens: + - '90000' + x-ratelimit-remaining-requests: + - '3499' + x-ratelimit-remaining-tokens: + - '86497' + x-ratelimit-reset-requests: + - 17ms + x-ratelimit-reset-tokens: + - 2.335s + x-request-id: + - 9418ad50ca3fd9b6f08757827e32f729 + status: + code: 200 + message: OK +- request: + body: '{"model": "gpt-3.5-turbo", "messages": [{"role": "system", "content": "You + are write_to_file-GPT, an AI designed to use the write_to_file command to write + ''Hello World'' into a file named \"hello_world.txt\" and then use the task_complete + command to complete the task.\nYour decisions must always be made independently + without seeking user assistance. Play to your strengths as an LLM and pursue + simple strategies with no legal complications.\n\nGOALS:\n\n1. Use the write_to_file + command to write ''Hello World'' into a file named \"hello_world.txt\".\n2. + Use the task_complete command to complete the task.\n3. Do not use andfsfdsy + other commands.\n\n\nConstraints:\n1. ~4000 word limit for short term memory. + Your short term memory is short, so immediately save important information to + files.\n2. If you are unsure how you previously did something or want to recall + past events, thinking about similar events will help you remember.\n3. No user + assistance\n4. Exclusively use the commands listed in double quotes e.g. \"command + name\"\n\nCommands:\n1. append_to_file: Append to file, args: \"filename\": + \"\", \"text\": \"\"\n2. delete_file: Delete file, args: \"filename\": + \"\"\n3. list_files: List Files in Directory, args: \"directory\": + \"\"\n4. read_file: Read file, args: \"filename\": \"\"\n5. + write_to_file: Write to file, args: \"filename\": \"\", \"text\": + \"\"\n6. delete_agent: Delete GPT Agent, args: \"key\": \"\"\n7. + get_hyperlinks: Get text summary, args: \"url\": \"\"\n8. get_text_summary: + Get text summary, args: \"url\": \"\", \"question\": \"\"\n9. + list_agents: List GPT Agents, args: () -> str\n10. message_agent: Message GPT + Agent, args: \"key\": \"\", \"message\": \"\"\n11. start_agent: + Start GPT Agent, args: \"name\": \"\", \"task\": \"\", + \"prompt\": \"\"\n12. task_complete: Task Complete (Shutdown), args: + \"reason\": \"\"\n\nResources:\n1. Internet access for searches and + information gathering.\n2. Long Term memory management.\n3. GPT-3.5 powered + Agents for delegation of simple tasks.\n4. File output.\n\nPerformance Evaluation:\n1. + Continuously review and analyze your actions to ensure you are performing to + the best of your abilities.\n2. Constructively self-criticize your big-picture + behavior constantly.\n3. Reflect on past decisions and strategies to refine + your approach.\n4. Every command has a cost, so be smart and efficient. Aim + to complete tasks in the least number of steps.\n5. Write all code to a file.\n\nYou + should only respond in JSON format as described below \nResponse Format: \n{\n \"thoughts\": + {\n \"text\": \"thought\",\n \"reasoning\": \"reasoning\",\n \"plan\": + \"- short bulleted\\n- list that conveys\\n- long-term plan\",\n \"criticism\": + \"constructive self-criticism\",\n \"speak\": \"thoughts summary to say + to user\"\n },\n \"command\": {\n \"name\": \"command name\",\n \"args\": + {\n \"arg name\": \"value\"\n }\n }\n} \nEnsure the response + can be parsed by Python json.loads"}, {"role": "system", "content": "The current + time and date is Tue Jan 1 00:00:00 2000"}, {"role": "system", "content": "This + reminds you of these events from your past: \nI was created and nothing new + has happened."}, {"role": "user", "content": "Determine which next command to + use, and respond using the format specified above:"}, {"role": "assistant", + "content": "{\n \"thoughts\": {\n \"text\": \"I need to use the write_to_file + command to write ''Hello World'' into a file named ''hello_world.txt''.\",\n \"reasoning\": + \"The task requires me to write ''Hello World'' into a file named ''hello_world.txt'', + and the write_to_file command is the appropriate command for this task.\",\n \"plan\": + \"- Use the write_to_file command to write ''Hello World'' into a file named + ''hello_world.txt''.\",\n \"criticism\": \"I need to ensure that I use + the correct file name and text to write to the file.\",\n \"speak\": + \"I will use the write_to_file command to write ''Hello World'' into a file + named ''hello_world.txt''.\"\n },\n \"command\": {\n \"name\": + \"write_to_file\",\n \"args\": {\n \"filename\": \"hello_world.txt\",\n \"text\": + \"Hello World\"\n }\n }\n}"}, {"role": "system", "content": "Command + write_to_file returned: File written to successfully."}, {"role": "user", "content": + "Determine which next command to use, and respond using the format specified + above:"}], "temperature": 0, "max_tokens": 0}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '4660' + Content-Type: + - application/json + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: '{"id":"chatcmpl-7FMsZlCgaqEI6ebydODAT47LprpFA","object":"chat.completion","created":1683897223,"model":"gpt-3.5-turbo-0301","usage":{"prompt_tokens":1031,"completion_tokens":201,"total_tokens":1232},"choices":[{"message":{"role":"assistant","content":"{\n \"thoughts\": + {\n \"text\": \"I have successfully written ''Hello World'' into a + file named ''hello_world.txt''. The next step is to use the task_complete + command to complete the task.\",\n \"reasoning\": \"The task requires + me to write ''Hello World'' into a file named ''hello_world.txt'' and then + complete the task. Since I have successfully written to the file, the next + step is to use the task_complete command to complete the task.\",\n \"plan\": + \"- Use the task_complete command to complete the task.\",\n \"criticism\": + \"I need to ensure that I provide a valid reason for completing the task.\",\n \"speak\": + \"I will use the task_complete command to complete the task.\"\n },\n \"command\": + {\n \"name\": \"task_complete\",\n \"args\": {\n \"reason\": + \"I have successfully written ''Hello World'' into a file named ''hello_world.txt''.\"\n }\n }\n}"},"finish_reason":"stop","index":0}]}' + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Length: + - '1221' + Content-Type: + - application/json + Date: + - Fri, 12 May 2023 13:14:02 GMT + Function-Execution-Id: + - ea42yneiu06g + Server: + - Google Frontend + X-Cloud-Trace-Context: + - 78cd3f6b10f6ed7cdd88e530a2abe3ee;o=1 + X-Powered-By: + - Express + status: + code: 200 + message: OK version: 1 diff --git a/tests/integration/goal_oriented/test_browse_website.py b/tests/integration/goal_oriented/test_browse_website.py index ca433d80f99e..3ce85689f924 100644 --- a/tests/integration/goal_oriented/test_browse_website.py +++ b/tests/integration/goal_oriented/test_browse_website.py @@ -8,7 +8,7 @@ @requires_api_key("OPENAI_API_KEY") @pytest.mark.vcr -def test_browse_website(browser_agent: Agent) -> None: +def test_browse_website(browser_agent: Agent, patched_api_requestor) -> None: file_path = browser_agent.workspace.get_path("browse_website.txt") try: run_interaction_loop(browser_agent, 120) diff --git a/tests/integration/goal_oriented/test_write_file.py b/tests/integration/goal_oriented/test_write_file.py index da67235a8397..55db3f4a99a2 100644 --- a/tests/integration/goal_oriented/test_write_file.py +++ b/tests/integration/goal_oriented/test_write_file.py @@ -1,3 +1,6 @@ +import os + +import openai import pytest from autogpt.agent import Agent @@ -8,10 +11,10 @@ @requires_api_key("OPENAI_API_KEY") @pytest.mark.vcr -def test_write_file(writer_agent: Agent) -> None: +def test_write_file(writer_agent: Agent, patched_api_requestor) -> None: file_path = str(writer_agent.workspace.get_path("hello_world.txt")) try: - run_interaction_loop(writer_agent, 40) + run_interaction_loop(writer_agent, 200) # catch system exit exceptions except SystemExit: # the agent returns an exception when it shuts down content = read_file(file_path) diff --git a/tests/integration/test_llm_utils.py b/tests/integration/test_llm_utils.py index 553d369947f6..fefc239c1ada 100644 --- a/tests/integration/test_llm_utils.py +++ b/tests/integration/test_llm_utils.py @@ -41,7 +41,10 @@ def spy_create_embedding(mocker: MockerFixture): @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") def test_get_ada_embedding( - config: Config, api_manager: ApiManager, spy_create_embedding: MagicMock + config: Config, + api_manager: ApiManager, + spy_create_embedding: MagicMock, + patched_api_requestor, ): token_cost = COSTS[config.embedding_model]["prompt"] llm_utils.get_ada_embedding("test") diff --git a/tests/integration/test_local_cache.py b/tests/integration/test_local_cache.py index 5200e026e708..808f119a73ed 100644 --- a/tests/integration/test_local_cache.py +++ b/tests/integration/test_local_cache.py @@ -91,7 +91,7 @@ def test_get(LocalCache, config, mock_embed_with_ada): @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") -def test_get_relevant(LocalCache, config) -> None: +def test_get_relevant(LocalCache, config, patched_api_requestor) -> None: cache = LocalCache(config) text1 = "Sample text 1" text2 = "Sample text 2" diff --git a/tests/integration/test_memory_management.py b/tests/integration/test_memory_management.py index c9ab9fc94818..22ade7b0be0f 100644 --- a/tests/integration/test_memory_management.py +++ b/tests/integration/test_memory_management.py @@ -52,7 +52,10 @@ def expected_permanent_memory() -> str: @requires_api_key("OPENAI_API_KEY") @pytest.mark.vcr def test_save_memory_trimmed_from_context_window( - message_history_fixture, expected_permanent_memory, config: Config + message_history_fixture, + expected_permanent_memory, + config: Config, + patched_api_requestor, ): next_message_to_add_index = len(message_history_fixture) - 1 memory = get_memory(config, init=True) diff --git a/tests/integration/test_setup.py b/tests/integration/test_setup.py index b649bb14f157..444d94747f12 100644 --- a/tests/integration/test_setup.py +++ b/tests/integration/test_setup.py @@ -13,7 +13,7 @@ @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") -def test_generate_aiconfig_automatic_default(): +def test_generate_aiconfig_automatic_default(patched_api_requestor): user_inputs = [""] with patch("builtins.input", side_effect=user_inputs): ai_config = prompt_user() @@ -26,7 +26,7 @@ def test_generate_aiconfig_automatic_default(): @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") -def test_generate_aiconfig_automatic_typical(): +def test_generate_aiconfig_automatic_typical(patched_api_requestor): user_prompt = "Help me create a rock opera about cybernetic giraffes" ai_config = generate_aiconfig_automatic(user_prompt) @@ -38,7 +38,7 @@ def test_generate_aiconfig_automatic_typical(): @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") -def test_generate_aiconfig_automatic_fallback(): +def test_generate_aiconfig_automatic_fallback(patched_api_requestor): user_inputs = [ "T&GF£OIBECC()!*", "Chef-GPT", @@ -59,7 +59,7 @@ def test_generate_aiconfig_automatic_fallback(): @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") -def test_prompt_user_manual_mode(): +def test_prompt_user_manual_mode(patched_api_requestor): user_inputs = [ "--manual", "Chef-GPT", diff --git a/tests/test_image_gen.py b/tests/test_image_gen.py index 546cc758d7e8..33e6fa738594 100644 --- a/tests/test_image_gen.py +++ b/tests/test_image_gen.py @@ -19,7 +19,7 @@ def image_size(request): reason="The image is too big to be put in a cassette for a CI pipeline. We're looking into a solution." ) @requires_api_key("OPENAI_API_KEY") -def test_dalle(config, workspace, image_size): +def test_dalle(config, workspace, image_size, patched_api_requestor): """Test DALL-E image generation.""" generate_and_validate( config, diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index e3b874fb9470..12f407389463 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -10,7 +10,7 @@ @pytest.mark.vcr @pytest.mark.integration_test @requires_api_key("OPENAI_API_KEY") -def test_make_agent() -> None: +def test_make_agent(patched_api_requestor) -> None: """Test that an agent can be created""" # Use the mock agent manager to avoid creating a real agent with patch("openai.ChatCompletion.create") as mock: diff --git a/tests/utils.py b/tests/utils.py index 2a0d25d81821..2603dfe48de7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -24,11 +24,11 @@ def requires_api_key(env_var): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): - if not os.environ.get(env_var) and env_var == "OPENAI_API_KEY": - with dummy_openai_api_key(): - return func(*args, **kwargs) - else: - return func(*args, **kwargs) + if env_var == "OPENAI_API_KEY": + if not os.environ.get(env_var) and env_var == "OPENAI_API_KEY": + with dummy_openai_api_key(): + return func(*args, **kwargs) + return func(*args, **kwargs) return wrapper diff --git a/tests/vcr/vcr_filter.py b/tests/vcr/vcr_filter.py index 892b8021fb6a..3a58bee98633 100644 --- a/tests/vcr/vcr_filter.py +++ b/tests/vcr/vcr_filter.py @@ -1,7 +1,10 @@ import json +import os import re from typing import Any, Dict, List +from tests.conftest import PROXY + REPLACEMENTS: List[Dict[str, str]] = [ { "regex": r"\w{3} \w{3} {1,2}\d{1,2} \d{2}:\d{2}:\d{2} \d{4}", @@ -13,6 +16,19 @@ }, ] +ALLOWED_HOSTNAMES: List[str] = [ + "api.openai.com", + "localhost:50337", +] + +if PROXY: + ALLOWED_HOSTNAMES.append(PROXY) + ORIGINAL_URL = PROXY +else: + ORIGINAL_URL = "no_ci" + +NEW_URL = "api.openai.com" + def replace_message_content(content: str, replacements: List[Dict[str, str]]) -> str: for replacement in replacements: @@ -53,6 +69,8 @@ def before_record_response(response: Dict[str, Any]) -> Dict[str, Any]: def before_record_request(request: Any) -> Any: + request = replace_request_hostname(request, ORIGINAL_URL, NEW_URL) + filtered_request = filter_hostnames(request) filtered_request_without_dynamic_data = replace_timestamp_in_request( filtered_request @@ -60,14 +78,24 @@ def before_record_request(request: Any) -> Any: return filtered_request_without_dynamic_data -def filter_hostnames(request: Any) -> Any: - allowed_hostnames: List[str] = [ - "api.openai.com", - "localhost:50337", - ] +from urllib.parse import urlparse, urlunparse + +def replace_request_hostname(request: Any, original_url: str, new_hostname: str) -> Any: + parsed_url = urlparse(request.uri) + + if parsed_url.hostname in original_url: + new_path = parsed_url.path.replace("/proxy_function", "") + request.uri = urlunparse( + parsed_url._replace(netloc=new_hostname, path=new_path, scheme="https") + ) + + return request + + +def filter_hostnames(request: Any) -> Any: # Add your implementation here for filtering hostnames - if any(hostname in request.url for hostname in allowed_hostnames): + if any(hostname in request.url for hostname in ALLOWED_HOSTNAMES): return request else: return None