From 86af2f9f15603c8e5650026bfa6d2c59d1194a65 Mon Sep 17 00:00:00 2001 From: Shrey Modi Date: Fri, 21 Nov 2025 11:03:22 -0800 Subject: [PATCH 1/4] tests fix --- tests/pytest/test_openenv_browsergym_basic.py | 8 ++++++-- tests/pytest/test_openenv_browsergym_eval.py | 6 ++++++ tests/pytest/test_openenv_echo_hub.py | 18 +++++++++++++----- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/pytest/test_openenv_browsergym_basic.py b/tests/pytest/test_openenv_browsergym_basic.py index f87a663b..26dbeb34 100644 --- a/tests/pytest/test_openenv_browsergym_basic.py +++ b/tests/pytest/test_openenv_browsergym_basic.py @@ -12,6 +12,12 @@ # Skip these integration-heavy tests on CI runners by default pytestmark = pytest.mark.skipif(os.getenv("CI") == "true", reason="Skip OpenEnv integration tests on CI") +# Skip if OpenEnv not installed +try: + from envs.browsergym_env import BrowserGymEnv, BrowserGymAction # type: ignore +except ImportError: + pytest.skip("OpenEnv browsergym_env not installed", allow_module_level=True) + @pytest.mark.integration def test_openenv_browsergym_basic(): @@ -43,8 +49,6 @@ def test_openenv_browsergym_basic(): # Construct the processor with a trivial action_parser; the model output will still be generated # but we parse to a safe noop action to minimize flakiness for the environment step. - from envs.browsergym_env import BrowserGymAction, BrowserGymEnv # type: ignore - processor = OpenEnvRolloutProcessor( env_factory=None, prompt_builder=lambda obs, step, history: "Do nothing", diff --git a/tests/pytest/test_openenv_browsergym_eval.py b/tests/pytest/test_openenv_browsergym_eval.py index 505336ae..dabad0fb 100644 --- a/tests/pytest/test_openenv_browsergym_eval.py +++ b/tests/pytest/test_openenv_browsergym_eval.py @@ -10,6 +10,12 @@ # Skip these integration-heavy tests on CI runners by default pytestmark = pytest.mark.skipif(os.getenv("CI") == "true", reason="Skip OpenEnv integration tests on CI") +# Skip if OpenEnv not installed +try: + from envs.browsergym_env import BrowserGymEnv, BrowserGymAction # type: ignore +except ImportError: + pytest.skip("OpenEnv browsergym_env not installed", allow_module_level=True) + def openenv_dataset_to_rows(data: List[Dict[str, Any]]) -> List[EvaluationRow]: """ diff --git a/tests/pytest/test_openenv_echo_hub.py b/tests/pytest/test_openenv_echo_hub.py index ae9c2cdc..d6216853 100644 --- a/tests/pytest/test_openenv_echo_hub.py +++ b/tests/pytest/test_openenv_echo_hub.py @@ -9,13 +9,19 @@ import pytest -# Preferred import when using the monolithic `openenv` package -from envs.echo_env import EchoEnv # type: ignore - - # Skip these integration-heavy tests on CI runners by default pytestmark = pytest.mark.skipif(os.getenv("CI") == "true", reason="Skip OpenEnv integration tests on CI") +# Try to import, but skip if not available +try: + from envs.echo_env import EchoEnv # type: ignore + + _HAS_ECHO = True +except ImportError: + _HAS_ECHO = False + EchoEnv = None # type: ignore + pytest.skip("OpenEnv echo_env not installed", allow_module_level=True) + def echo_dataset_to_rows(data: List[Dict[str, Any]]) -> List[EvaluationRow]: """ @@ -79,7 +85,7 @@ def action_parser(response_text: str): rollout_processor=( OpenEnvRolloutProcessor( # Use HF Hub to launch the environment container automatically - env_client_cls=EchoEnv, # type: ignore + env_client_cls=EchoEnv if _HAS_ECHO else None, # type: ignore hub_repo_id=os.getenv("OPENENV_ECHO_REPO", "openenv/echo-env"), # Simple prompt+parser above prompt_builder=prompt_builder, @@ -88,6 +94,8 @@ def action_parser(response_text: str): timeout_ms=5000, num_generations=1, ) + if _HAS_ECHO + else None ), ) def test_openenv_echo_hub(row: EvaluationRow) -> EvaluationRow: From 135422e986e02d80e3c341af0130654df928cb2a Mon Sep 17 00:00:00 2001 From: Shrey Modi Date: Fri, 21 Nov 2025 14:02:35 -0800 Subject: [PATCH 2/4] echo hub test fix --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d29a04a3..f8bc9cc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,12 @@ jobs: - name: Install tau2 for testing run: uv pip install git+https://github.com/sierra-research/tau2-bench.git@main + - name: Install OpenEnv for integration tests + run: | + # Install OpenEnv core and echo environment + uv pip install "openenv-core" + uv pip install "openenv-echo-env @ git+https://github.com/meta-pytorch/OpenEnv.git#subdirectory=src/envs/echo_env" + - name: Run Core Tests with pytest-xdist env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -109,6 +115,9 @@ jobs: --ignore=tests/remote_server/test_remote_fireworks.py \ --ignore=tests/remote_server/test_remote_fireworks_propagate_status.py \ --ignore=tests/logging/test_elasticsearch_direct_http_handler.py \ + --ignore=tests/pytest/test_openenv_browsergym_basic.py \ + --ignore=tests/pytest/test_openenv_browsergym_eval.py \ + --ignore=tests/pytest/test_openenv_textarena_docker.py \ --ignore=eval_protocol/benchmarks/ \ --ignore=eval_protocol/quickstart/ \ --cov=eval_protocol --cov-append --cov-report=xml --cov-report=term-missing -v --durations=10 From ada54e4db4f0d269070f500177aa276e7e6d4c8e Mon Sep 17 00:00:00 2001 From: Shrey Modi Date: Fri, 21 Nov 2025 14:08:21 -0800 Subject: [PATCH 3/4] test fix --- tests/pytest/test_openenv_echo_hub.py | 52 +++++++-------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/tests/pytest/test_openenv_echo_hub.py b/tests/pytest/test_openenv_echo_hub.py index d6216853..b3aa8004 100644 --- a/tests/pytest/test_openenv_echo_hub.py +++ b/tests/pytest/test_openenv_echo_hub.py @@ -8,19 +8,8 @@ from eval_protocol.pytest.openenv_rollout_processor import OpenEnvRolloutProcessor import pytest - -# Skip these integration-heavy tests on CI runners by default -pytestmark = pytest.mark.skipif(os.getenv("CI") == "true", reason="Skip OpenEnv integration tests on CI") - -# Try to import, but skip if not available -try: - from envs.echo_env import EchoEnv # type: ignore - - _HAS_ECHO = True -except ImportError: - _HAS_ECHO = False - EchoEnv = None # type: ignore - pytest.skip("OpenEnv echo_env not installed", allow_module_level=True) +# Import OpenEnv Echo environment +from envs.echo_env import EchoEnv, EchoAction # type: ignore def echo_dataset_to_rows(data: List[Dict[str, Any]]) -> List[EvaluationRow]: @@ -45,23 +34,10 @@ def action_parser(response_text: str): """ Convert raw model response to EchoAction. """ - try: - from envs.echo_env import EchoAction # type: ignore - except Exception: - pytest.skip("OpenEnv (openenv.envs.echo_env) is not installed; skipping Echo hub test.") - raise text = response_text.strip() if isinstance(response_text, str) else "" return EchoAction(message=text or "hello") -# try: -# from envs.echo_env import EchoEnv # type: ignore - -# _HAS_ECHO = True -# except Exception: -# _HAS_ECHO = False - - # Inline test data ECHO_INLINE_DATA: List[Dict[str, Any]] = [ {"id": "echo-1", "prompt": "hello"}, @@ -82,20 +58,16 @@ def action_parser(response_text: str): num_runs=1, max_concurrent_rollouts=2, mode="pointwise", - rollout_processor=( - OpenEnvRolloutProcessor( - # Use HF Hub to launch the environment container automatically - env_client_cls=EchoEnv if _HAS_ECHO else None, # type: ignore - hub_repo_id=os.getenv("OPENENV_ECHO_REPO", "openenv/echo-env"), - # Simple prompt+parser above - prompt_builder=prompt_builder, - action_parser=action_parser, - # Keep defaults for timeouts/viewport/etc. (not relevant for echo) - timeout_ms=5000, - num_generations=1, - ) - if _HAS_ECHO - else None + rollout_processor=OpenEnvRolloutProcessor( + # Use HF Hub to launch the environment container automatically + env_client_cls=EchoEnv, + hub_repo_id=os.getenv("OPENENV_ECHO_REPO", "openenv/echo-env"), + # Simple prompt+parser above + prompt_builder=prompt_builder, + action_parser=action_parser, + # Keep defaults for timeouts/viewport/etc. (not relevant for echo) + timeout_ms=5000, + num_generations=1, ), ) def test_openenv_echo_hub(row: EvaluationRow) -> EvaluationRow: From 03dfcc8133341d33731790168d3dff821681903b Mon Sep 17 00:00:00 2001 From: Shrey Modi Date: Fri, 21 Nov 2025 14:23:41 -0800 Subject: [PATCH 4/4] update --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8bc9cc6..5e2db39d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: run: | # Install OpenEnv core and echo environment uv pip install "openenv-core" - uv pip install "openenv-echo-env @ git+https://github.com/meta-pytorch/OpenEnv.git#subdirectory=src/envs/echo_env" + uv pip install "openenv @ git+https://github.com/meta-pytorch/OpenEnv.git" - name: Run Core Tests with pytest-xdist env: