fix: ensure utf-8 encoding is used in all relevant open() calls

python/packages/agbench/benchmarks/GAIA/Scripts/custom_tabulate.py

@@ -131,7 +131,7 @@ def scorer(instance_dir):
         return None
 
     expected_answer = None
-    with open(expected_answer_file, "rt") as fh:
+    with open(expected_answer_file, "rt", encoding="utf-8") as fh:
         expected_answer = fh.read().strip()
 
     # Read the console
@@ -140,7 +140,7 @@ def scorer(instance_dir):
         return None
 
     console_log = ""
-    with open(console_log_file, "rt") as fh:
+    with open(console_log_file, "rt", encoding="utf-8") as fh:
         console_log = fh.read()
 
         final_answer = None 

python/packages/agbench/benchmarks/GAIA/Scripts/init_tasks.py

@@ -42,7 +42,7 @@ def create_jsonl(name, tasks, files_dir, template):
     if not os.path.isdir(TASKS_DIR):
         os.mkdir(TASKS_DIR)
 
-    with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt") as fh:
+    with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt", encoding="utf-8") as fh:
         for task in tasks:
             print(f"Converting: [{name}] {task['task_id']}")
 

python/packages/agbench/benchmarks/GAIA/Templates/MagenticOne/scenario.py

@@ -20,7 +20,7 @@
 async def main() -> None:
 
     # Load model configuration and create the model client.
-    with open("config.yaml", "r") as f:
+    with open("config.yaml", "r", encoding="utf-8") as f:
         config = yaml.safe_load(f)
 
     orchestrator_client = ChatCompletionClient.load_component(config["orchestrator_client"])
@@ -30,7 +30,7 @@ async def main() -> None:
 
     # Read the prompt
     prompt = ""
-    with open("prompt.txt", "rt") as fh:
+    with open("prompt.txt", "rt", encoding="utf-8") as fh:
         prompt = fh.read().strip()
     filename = "__FILE_NAME__".strip()
 

python/packages/agbench/benchmarks/GAIA/Templates/ParallelAgents/scenario.py

@@ -267,7 +267,7 @@ async def aggregate_final_answer(task: str, client: ChatCompletionClient, team_r
 async def main(num_teams: int, num_answers: int) -> None:
 
     # Load model configuration and create the model client.
-    with open("config.yaml", "r") as f:
+    with open("config.yaml", "r", encoding="utf-8") as f:
         config = yaml.safe_load(f)
 
     orchestrator_client = ChatCompletionClient.load_component(config["orchestrator_client"])
@@ -277,7 +277,7 @@ async def main(num_teams: int, num_answers: int) -> None:
 
     # Read the prompt
     prompt = ""
-    with open("prompt.txt", "rt") as fh:
+    with open("prompt.txt", "rt", encoding="utf-8") as fh:
         prompt = fh.read().strip()
     filename = "__FILE_NAME__".strip()
 

python/packages/agbench/benchmarks/GAIA/Templates/SelectorGroupChat/scenario.py

@@ -25,7 +25,7 @@
 async def main() -> None:
 
     # Load model configuration and create the model client.
-    with open("config.yaml", "r") as f:
+    with open("config.yaml", "r", encoding="utf-8") as f:
         config = yaml.safe_load(f)
 
     orchestrator_client = ChatCompletionClient.load_component(config["orchestrator_client"])
@@ -35,7 +35,7 @@ async def main() -> None:
 
     # Read the prompt
     prompt = ""
-    with open("prompt.txt", "rt") as fh:
+    with open("prompt.txt", "rt", encoding="utf-8") as fh:
         prompt = fh.read().strip()
     filename = "__FILE_NAME__".strip()
 

python/packages/agbench/benchmarks/HumanEval/Scripts/init_tasks.py

@@ -85,7 +85,7 @@ def create_jsonl(name, tasks, template):
         os.mkdir(TASKS_DIR)
 
     # Create the jsonl file
-    with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt") as fh:
+    with open(os.path.join(TASKS_DIR, name + ".jsonl"), "wt", encoding="utf-8") as fh:
         for task in tasks:
             print(f"Converting: [{name}] {task['task_id']}")
 

python/packages/agbench/benchmarks/HumanEval/Templates/AgentChat/custom_code_executor.py

@@ -17,7 +17,7 @@ def __init__(
     ) -> None:
         super().__init__(name=name, description=description, code_executor=code_executor, sources=sources)
         self._test_code = ""
-        with open("test.txt", "rt") as fh:
+        with open("test.txt", "rt", encoding="utf-8") as fh:
             self._test_code = fh.read()
 
 

python/packages/agbench/benchmarks/HumanEval/Templates/AgentChat/scenario.py

@@ -15,7 +15,7 @@
 async def main() -> None:
 
     # Load model configuration and create the model client.
-    with open("config.yaml", "r") as f:
+    with open("config.yaml", "r", encoding="utf-8") as f:
         config = yaml.safe_load(f)
     model_client = ChatCompletionClient.load_component(config["model_config"])
 
@@ -48,7 +48,7 @@ async def main() -> None:
     agent_team = RoundRobinGroupChat([coder_agent, executor], max_turns=12, termination_condition=termination)
 
     prompt = ""
-    with open("prompt.txt", "rt") as fh:
+    with open("prompt.txt", "rt", encoding="utf-8") as fh:
         prompt = fh.read()
 
     task = f"""Complete the following python function. Format your output as Markdown python code block containing the entire function definition:

python/packages/agbench/src/agbench/linter/cli.py

@@ -19,7 +19,7 @@ def prepend_line_numbers(lines: List[str]) -> List[str]:
 
 
 def load_log_file(path: str, prepend_numbers: bool = False) -> Document:
-    with open(path, "r") as f:
+    with open(path, "r", encoding="utf-8") as f:
         lines = f.readlines()
     if prepend_numbers:
         lines = prepend_line_numbers(lines)

python/packages/agbench/src/agbench/run_cmd.py

@@ -113,7 +113,7 @@ def run_scenarios(
             scenario_name_parts.pop()
             scenario_name = ".".join(scenario_name_parts)
             scenario_dir = os.path.dirname(os.path.realpath(scenario_file))
-            file_handle = open(scenario_file, "rt")
+            file_handle = open(scenario_file, "rt", encoding="utf-8")
 
         # Read all the lines, then subsample if needed
         lines = [line for line in file_handle]
@@ -241,13 +241,13 @@ def expand_scenario(
     for templated_file in substitutions.keys():  # Keys are relative file paths
         # Read the templated file into memory
         template_contents: List[str] = list()
-        with open(os.path.join(output_dir, templated_file), "rt") as fh:
+        with open(os.path.join(output_dir, templated_file), "rt", encoding="utf-8") as fh:
             for line in fh:
                 template_contents.append(line)
 
         # Rewrite the templated file with substitutions
         values = substitutions[templated_file]
-        with open(os.path.join(output_dir, templated_file), "wt") as fh:
+        with open(os.path.join(output_dir, templated_file), "wt", encoding="utf-8") as fh:
             for line in template_contents:
                 for k, v in values.items():
                     line = line.replace(k, v)
@@ -295,10 +295,10 @@ def get_scenario_env(token_provider: Optional[Callable[[], str]] = None, env_fil
     if env_file is None:
         # Env file was not specified, so read the default, or warn if the default file is missing.
         if os.path.isfile(DEFAULT_ENV_FILE_YAML):
-            with open(DEFAULT_ENV_FILE_YAML, "r") as fh:
+            with open(DEFAULT_ENV_FILE_YAML, "r", encoding="utf-8") as fh:
                 env_file_contents = yaml.safe_load(fh)
         elif os.path.isfile(DEFAULT_ENV_FILE_JSON):
-            with open(DEFAULT_ENV_FILE_JSON, "rt") as fh:
+            with open(DEFAULT_ENV_FILE_JSON, "rt", encoding="utf-8") as fh:
                 env_file_contents = json.loads(fh.read())
             logging.warning(f"JSON environment files are deprecated. Migrate to '{DEFAULT_ENV_FILE_YAML}'")
         else:
@@ -307,7 +307,7 @@ def get_scenario_env(token_provider: Optional[Callable[[], str]] = None, env_fil
             )
     else:
         # Env file was specified. Throw an error if the file can't be read.
-        with open(env_file, "rt") as fh:
+        with open(env_file, "rt", encoding="utf-8") as fh:
             if env_file.endswith(".json"):
                 logging.warning("JSON environment files are deprecated. Migrate to YAML")
                 env_file_contents = json.loads(fh.read())
@@ -396,7 +396,7 @@ def run_scenario_natively(work_dir: str, env: Dict[str, str], timeout: int = TAS
     print("\n\n" + os.getcwd() + "\n===================================================================")
 
     # Prepare the run script
-    with open(os.path.join("run.sh"), "wt") as f:
+    with open(os.path.join("run.sh"), "wt", encoding="utf-8") as f:
         f.write(
             f"""#
 echo RUN.SH STARTING !#!#
@@ -520,7 +520,7 @@ def run_scenario_in_docker(
                 print(f"Failed to pull image '{docker_image}'")
 
     # Prepare the run script
-    with open(os.path.join(work_dir, "run.sh"), "wt", newline="\n") as f:
+    with open(os.path.join(work_dir, "run.sh"), "wt", newline="\n", encoding="utf-8") as f:
         f.write(
             f"""#
 echo RUN.SH STARTING !#!#
@@ -737,7 +737,7 @@ def split_jsonl(file_path: str, num_parts: int) -> List[List[Dict[str, Any]]]:
     """
     Split a JSONL file into num_parts approximately equal parts.
     """
-    with open(file_path, "r") as f:
+    with open(file_path, "r", encoding="utf-8") as f:
         data = [json.loads(line) for line in f]
 
     random.shuffle(data)  # Shuffle the data for better distribution
@@ -942,7 +942,7 @@ def run_cli(args: Sequence[str]) -> None:
 
     if parsed_args.config is not None:
         # Make sure the config file is readable, so that we fail early
-        with open(parsed_args.config, "r"):
+        with open(parsed_args.config, "r", encoding="utf-8"):
             pass
 
     # don't support parallel and subsample together

python/packages/autogen-ext/examples/mcp_session_host_example.py

@@ -63,10 +63,10 @@ def load_model_client_from_config(config_path: str) -> ChatCompletionClient:
 
     # Load config based on file extension
     if config_file.suffix.lower() == ".json":
-        with open(config_file, "r") as f:
+        with open(config_file, "r", encoding="utf-8") as f:
             config_data = json.load(f)
     elif config_file.suffix.lower() in [".yml", ".yaml"]:
-        with open(config_file, "r") as f:
+        with open(config_file, "r", encoding="utf-8") as f:
             config_data = yaml.safe_load(f)
     else:
         raise ValueError(f"Unsupported config file type: {config_file.suffix}. Use .json, .yml, or .yaml")

python/packages/autogen-ext/src/autogen_ext/code_executors/docker_jupyter/_docker_jupyter.py

@@ -275,7 +275,7 @@ def _save_html(self, html_data: str) -> str:
         """Save html data to a file."""
         filename = f"{uuid.uuid4().hex}.html"
         path = os.path.join(str(self._output_dir), filename)
-        with open(path, "w") as f:
+        with open(path, "w", encoding="utf-8") as f:
             f.write(html_data)
         return os.path.abspath(path)
 

python/packages/autogen-ext/src/autogen_ext/experimental/task_centric_memory/utils/chat_completion_client_recorder.py

@@ -73,7 +73,7 @@ def __init__(
             # Load the previously recorded messages and responses from disk.
             self.logger.info("Replay mode enabled.\nRetrieving session from: " + self.session_file_path)
             try:
-                with open(self.session_file_path, "r") as f:
+                with open(self.session_file_path, "r", encoding="utf-8") as f:
                     self.records = json.load(f)
             except Exception as e:
                 error_str = f"\nFailed to load recorded session: '{self.session_file_path}': {e}"
@@ -211,7 +211,7 @@ def finalize(self) -> None:
                 # Create the directory if it doesn't exist.
                 os.makedirs(os.path.dirname(self.session_file_path), exist_ok=True)
                 # Write the records to disk.
-                with open(self.session_file_path, "w") as f:
+                with open(self.session_file_path, "w", encoding="utf-8") as f:
                     json.dump(self.records, f, indent=2)
                     self.logger.info("\nRecorded session was saved to: " + self.session_file_path)
             except Exception as e:

python/packages/autogen-ext/src/autogen_ext/experimental/task_centric_memory/utils/page_logger.py

@@ -117,7 +117,7 @@ def finalize(self) -> None:
         # Write the hash and other details to a file.
         hash_str, num_files, num_subdirs = hash_directory(self.log_dir)
         hash_path = os.path.join(self.log_dir, "hash.txt")
-        with open(hash_path, "w") as f:
+        with open(hash_path, "w", encoding="utf-8") as f:
             f.write(hash_str)
             f.write("\n")
             f.write("{} files\n".format(num_files))
@@ -386,7 +386,7 @@ def flush(self, finished: bool = False) -> None:
             return
         # Create a call tree of the log.
         call_tree_path = os.path.join(self.log_dir, self.name + ".html")
-        with open(call_tree_path, "w") as f:
+        with open(call_tree_path, "w", encoding="utf-8") as f:
             f.write(_html_opening("0 Call Tree", finished=finished))
             f.write(f"<h3>{self.name}</h3>")
             f.write("\n")
@@ -498,7 +498,7 @@ def flush(self) -> None:
         Writes the HTML page to disk.
         """
         page_path = os.path.join(self.page_logger.log_dir, self.index_str + ".html")
-        with open(page_path, "w") as f:
+        with open(page_path, "w", encoding="utf-8") as f:
             f.write(_html_opening(self.file_title, finished=self.finished))
             f.write(f"<h3>{self.file_title}</h3>\n")
             for line in self.lines:

python/packages/autogen-studio/autogenstudio/cli.py

@@ -69,7 +69,7 @@ def ui(
 
     # Create temporary env file to share configuration with uvicorn workers
     env_file_path = get_env_file_path()
-    with open(env_file_path, "w") as temp_env:
+    with open(env_file_path, "w", encoding="utf-8") as temp_env:
         for key, value in env_vars.items():
             temp_env.write(f"{key}={value}\n")
 

python/packages/autogen-studio/autogenstudio/database/schema_manager.py

@@ -75,7 +75,7 @@ def _update_configuration(self) -> None:
 
         # Update alembic.ini
         config_content = self._generate_alembic_ini_content()
-        with open(self.alembic_ini_path, "w") as f:
+        with open(self.alembic_ini_path, "w", encoding="utf-8") as f:
             f.write(config_content)
 
         # Update env.py
@@ -115,7 +115,7 @@ def _initialize_alembic(self) -> bool:
 
             # Create initial config file for alembic init
             config_content = self._generate_alembic_ini_content()
-            with open(self.alembic_ini_path, "w") as f:
+            with open(self.alembic_ini_path, "w", encoding="utf-8") as f:
                 f.write(config_content)
 
             # Use the config we just created
@@ -187,7 +187,7 @@ def run_migrations_online() -> None:
 else:
     run_migrations_online()"""
 
-        with open(env_path, "w") as f:
+        with open(env_path, "w", encoding="utf-8") as f:
             f.write(content)
 
     def _generate_alembic_ini_content(self) -> str:
@@ -239,7 +239,7 @@ def update_script_template(self):
         """Update the Alembic script template to include SQLModel."""
         template_path = self.alembic_dir / "script.py.mako"
         try:
-            with open(template_path, "r") as f:
+            with open(template_path, "r", encoding="utf-8") as f:
                 content = f.read()
 
             # Add sqlmodel import to imports section
@@ -248,7 +248,7 @@ def update_script_template(self):
 
             content = content.replace(import_section, new_imports)
 
-            with open(template_path, "w") as f:
+            with open(template_path, "w", encoding="utf-8") as f:
                 f.write(content)
 
             return True
@@ -265,7 +265,7 @@ def _update_env_py(self, env_path: Path) -> None:
             self._create_minimal_env_py(env_path)
             return
         try:
-            with open(env_path, "r") as f:
+            with open(env_path, "r", encoding="utf-8") as f:
                 content = f.read()
 
             # Add SQLModel import if not present
@@ -303,7 +303,7 @@ def _update_env_py(self, env_path: Path) -> None:
             )""",
             )
 
-            with open(env_path, "w") as f:
+            with open(env_path, "w", encoding="utf-8") as f:
                 f.write(content)
         except Exception as e:
             logger.error(f"Failed to update env.py: {e}")

python/packages/autogen-studio/autogenstudio/gallery/builder.py

@@ -630,5 +630,5 @@ def create_default_lite_team():
     gallery = create_default_gallery()
 
     # Save to file
-    with open("gallery_default.json", "w") as f:
+    with open("gallery_default.json", "w", encoding="utf-8") as f:
         f.write(gallery.model_dump_json(indent=2))

python/packages/autogen-studio/autogenstudio/lite/studio.py

@@ -151,7 +151,7 @@ def _setup_environment(self) -> str:
         }
 
         env_file_path = self._get_env_file_path()
-        with open(env_file_path, "w") as temp_env:
+        with open(env_file_path, "w", encoding="utf-8") as temp_env:
             for key, value in env_vars.items():
                 temp_env.write(f"{key}={value}\n")
 

python/packages/autogen-studio/autogenstudio/teammanager/teammanager.py

@@ -51,7 +51,7 @@ async def load_from_file(path: Union[str, Path]) -> Any:
         if not path.exists():
             raise FileNotFoundError(f"Config file not found: {path}")
 
-        async with aiofiles.open(path) as f:
+        async with aiofiles.open(path, mode="r", encoding="utf-8") as f:
             content = await f.read()
             if path.suffix == ".json":
                 return json.loads(content)

python/packages/autogen-studio/autogenstudio/web/auth/manager.py

@@ -117,7 +117,7 @@ def is_valid_token(self, token: str) -> bool:
     def from_yaml(cls, yaml_path: str) -> Self:
         """Create AuthManager from YAML config file."""
         try:
-            with open(yaml_path, "r") as f:
+            with open(yaml_path, "r", encoding="utf-8") as f:
                 config_data = yaml.safe_load(f)
             config = AuthConfig(**config_data)
             return cls(config)

python/packages/magentic-one-cli/src/magentic_one_cli/_m1.py

@@ -97,12 +97,12 @@ def main() -> None:
 
     if args.config is None:
         if os.path.isfile(DEFAULT_CONFIG_FILE):
-            with open(DEFAULT_CONFIG_FILE, "r") as f:
+            with open(DEFAULT_CONFIG_FILE, "r", encoding="utf-8") as f:
                 config = yaml.safe_load(f)
         else:
             config = yaml.safe_load(DEFAULT_CONFIG_CONTENTS)
     else:
-        with open(args.config if isinstance(args.config, str) else args.config[0], "r") as f:
+        with open(args.config if isinstance(args.config, str) else args.config[0], "r", encoding="utf-8") as f:
             config = yaml.safe_load(f)
 
     # Run the task