diff --git a/.gitignore b/.gitignore index 7afb58e9ee0..3ba0d9789e3 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,6 @@ tests/ # TacSL sensor **/tactile_record/* **/gelsight_r15_data/* + +# Downloaded assets +offline_assets/* diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 9fbfe7f1bf3..4bcc82be991 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -59,6 +59,7 @@ Guidelines for modifications: * Cathy Y. Li * Cheng-Rong Lai * Chenyu Yang +* Clayton Littlejohn * Connor Smith * CY (Chien-Ying) Chen * David Yang diff --git a/scripts/offline_setup/README.md b/scripts/offline_setup/README.md new file mode 100644 index 00000000000..cbf3361d267 --- /dev/null +++ b/scripts/offline_setup/README.md @@ -0,0 +1,145 @@ +# Isaac Lab Offline Training Setup +> Complete guide for training Isaac Lab environments offline using locally downloaded assets. + +## šŸŽÆ Overview +#### The offline training system enables you to train Isaac Lab environments without internet connectivity by using locally downloaded assets. This system: +- āœ… Works with any robot - No hardcoded paths needed +- āœ… Single flag - Add `--offline` to your training, tutorial, and demo commands +- āœ… Fallback flag - Add `--offline-permissive` to your commands to fallback to Nucleus for asset not found in `offline_assets` +- āœ… Maintains structure - Mirrors Nucleus directory organization locally + +## šŸ“¦ Requirements +- Isaac Lab installed and working +- Isaac Sim 5.0 or later +- 2-60 GB free disk space (depending on assets downloaded) +- Internet connection for initial asset download + +## šŸš€ Quick Start +### 1. Download essential assets (one-time, `all` ~60 GB) +#### Assets download to the `~/IsaacLab/offline_assets` directory: `cd ~/IsaacLab` +``` +./isaaclab.sh -p scripts/offline_setup/download_assets.py \ + --categories all +``` +#### _Alternative Note: Category fields can be specified separately_ +``` +./isaaclab.sh -p scripts/offline_setup/download_assets.py \ + --categories Robots --subset Unitree +``` + +### Successful Downloads +``` +====================================================================== +šŸ“Š Downloaded Assets Summary +====================================================================== +Location: ~/IsaacLab/offline_assets + +āœ“ Environments 34,437 files 17.0 GB +āœ“ IsaacLab 4,197 files 9.3 GB +āœ“ Materials 1,918 files 537.0 MB +āœ“ People 3,085 files 9.4 GB +āœ“ Props 2,507 files 4.2 GB +āœ“ Robots 4,687 files 5.3 GB +āœ“ Samples 6,601 files 10.4 GB +āœ“ Sensors 772 files 256.7 MB +====================================================================== +TOTAL 58,204 files 56.3 GB +====================================================================== + +āœ… Complete! Use --offline flag with Isaac Lab commands. + +Offline mirror: ~/IsaacLab/offline_assets +``` + +### 2. Train completely offline with any robot via the `--offline` flag (also works with `/play`) +#### Supported for: `rl_games`, `rsl_rl`, `sb3`, `skrl`, and `sim2transfer` +``` +./isaaclab.sh -p scripts/reinforcement_learning/rsl_rl/train.py \ + --task Isaac-Velocity-Rough-Unitree-Go2-v0 \ + --num_envs 64 \ + --max_iterations 10 \ + --offline + +./isaaclab.sh -p scripts/reinforcement_learning/skrl/train.py \ + --task Isaac-Velocity-Flat-H1-v0 \ + --num_envs 64 \ + --max_iterations 10 \ + --offline + +./isaaclab.sh -p scripts/reinforcement_learning/rsl_rl/play.py \ + --task Isaac-Velocity-Flat-Unitree-Go2-v0 \ + --num_envs 128 \ + --checkpoint logs/rsl_rl/_flat//model_.pt \ + --video \ + --video_length 1000 \ + --offline +``` +#### Run various demos and tutorials with `--offline` flag + +``` +./isaaclab.sh -p scripts/demos/quadrupeds.py --offline + +./isaaclab.sh -p scripts/demos/arms.py --offline + +./isaaclab.sh -p scripts/tutorials/01_assets/run_articulation.py --offline +``` + +#### Strict mode (default) - fails immediately if asset not found locally +``` +./isaaclab.sh -p train.py --task Go2 --offline +``` + +#### Permissive mode - warns and falls back to Nucleus if asset not found +``` +./isaaclab.sh -p train.py --task Go2 --offline-permissive +``` + +#### Environment variables work too +``` +OFFLINE=1 ./isaaclab.sh -p train.py --task Go2 +OFFLINE_PERMISSIVE=1 ./isaaclab.sh -p train.py --task Go2 +``` + +### Missing Assets +``` +====================================================================== +[OfflineAssetResolver] āœ— ASSET NOT FOUND (offline mode) +====================================================================== +Missing: IsaacLab/Robots/Unitree/Go2/go2.usd +Expected: ~/IsaacLab/offline_assets/IsaacLab/Robots/Unitree/Go2/go2.usd + +To download this asset, run: + ./isaaclab.sh -p scripts/offline_setup/download_assets.py --categories IsaacLab/Robots +``` + +#### _Note: For offline training, assets that cannot be found in `offline_assets` will attempted to be fetched from the [Nucleus Server](https://docs.omniverse.nvidia.com/nucleus/latest/index.html)_ when using the `--offline-permissive` flag. + +## šŸ“ Asset Layout +#### Offline assets are organized to mirror Nucleus (`ISAAC_NUCLEUS_DIR` & `ISAACLAB_NUCLEUS_DIR`) under the `offline_assets` directory, meaning that no code changes are required for offline running! We flatten `Isaac/IsaacLab/` to just the category names (`Robots/`, `Controllers/`, etc.) for cleaner local structure. This happens in `asset_resolver.py`, where the resolver maintains a 1:1 mapping between Nucleus and local storage. + +``` +IsaacLab/ +ā”œā”€ā”€ source/isaaclab/isaaclab/utils/ +ā”‚ ā””ā”€ā”€ asset_resolver.py # Core resolver +ā”œā”€ā”€ scripts/setup/ +ā”‚ ā””ā”€ā”€ download_assets.py # Asset downloader +ā””ā”€ā”€ offline_assets/ ā† Mirror of Isaac/ +ā”œā”€ā”€ IsaacLab/ +ā”‚ ā”œā”€ā”€ Robots/ +ā”‚ ā”‚ ā”œā”€ā”€ ANYbotics/ +ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ ANYmal-B/ +ā”‚ ā”‚ ā”‚ ā”œā”€ā”€ ANYmal-C/ +ā”‚ ā”‚ ā”‚ ā””ā”€ā”€ ANYmal-D/ +ā”‚ ā”‚ ā”œā”€ā”€ Unitree/ +ā”‚ ā”‚ ā””ā”€ā”€ FrankaEmika/ +ā”‚ ā”œā”€ā”€ ActuatorNets/ +ā”‚ ā”œā”€ā”€ Controllers/ +ā”‚ ā””ā”€ā”€ Policies/ +ā”œā”€ā”€ Props/ +ā”‚ ā””ā”€ā”€ UIElements/ +ā”œā”€ā”€ Environments/ +ā”‚ ā””ā”€ā”€ Grid/ +ā”œā”€ā”€ Materials/ +ā”‚ ā””ā”€ā”€ Textures/ +ā””ā”€ā”€ Robots/ ā† Isaac Sim robots (different from IsaacLab/Robots) +``` diff --git a/scripts/offline_setup/download_assets.py b/scripts/offline_setup/download_assets.py new file mode 100644 index 00000000000..d285d41f199 --- /dev/null +++ b/scripts/offline_setup/download_assets.py @@ -0,0 +1,307 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Download Isaac assets from Nucleus server for offline use. + +This script mirrors the Nucleus Isaac/ directory structure locally: + + Nucleus: .../Assets/Isaac/5.1/Isaac/ + Local: offline_assets/ + +Usage: + # Download Isaac Lab essentials (default) + ./isaaclab.sh -p scripts/offline_setup/download_assets.py + + # Download everything from Isaac/ + ./isaaclab.sh -p scripts/offline_setup/download_assets.py --categories all + + # Download specific directories + ./isaaclab.sh -p scripts/offline_setup/download_assets.py --categories IsaacLab Props + + # Download specific subdirectory + ./isaaclab.sh -p scripts/offline_setup/download_assets.py --categories IsaacLab/Robots --subset Unitree +""" + +import argparse +import os +from pathlib import Path + +from tqdm import tqdm + +from isaaclab.app import AppLauncher + +app_launcher = AppLauncher(headless=True) +simulation_app = app_launcher.app + +import carb +import omni.client + +ISAACLAB_PATH = os.environ.get("ISAACLAB_PATH", os.getcwd()) +OFFLINE_ASSETS_DIR = os.path.join(ISAACLAB_PATH, "offline_assets") + +settings = carb.settings.get_settings() +NUCLEUS_ASSET_ROOT = settings.get("/persistent/isaac/asset_root/default") +ISAAC_NUCLEUS_DIR = f"{NUCLEUS_ASSET_ROOT}/Isaac" + +KNOWN_CATEGORIES = { + "IsaacLab": "Isaac Lab assets (Robots, ActuatorNets, Controllers, Policies, etc.)", + "Props": "Props, markers, UI elements", + "Environments": "Environment assets, ground planes", + "Materials": "Materials, textures, HDRs", + "Robots": "Isaac Sim robots (separate from IsaacLab/Robots)", + "Sensors": "Sensor assets", + "People": "Human assets", + "Samples": "Sample scenes", +} + + +def format_size(bytes_size: int) -> str: + for unit in ["B", "KB", "MB", "GB", "TB"]: + if bytes_size < 1024.0: + return f"{bytes_size:.1f} {unit}" + bytes_size /= 1024.0 + return f"{bytes_size:.1f} PB" + + +def get_local_directory_size(path: str) -> int: + total_size = 0 + if os.path.exists(path): + for dirpath, _, filenames in os.walk(path): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + if os.path.exists(filepath): + total_size += os.path.getsize(filepath) + return total_size + + +def get_remote_directory_info(remote_path: str) -> tuple[int, int]: + file_count = 0 + total_size = 0 + + result, entries = omni.client.list(remote_path) + if result != omni.client.Result.OK: + return 0, 0 + + for entry in entries: + is_dir = entry.flags & omni.client.ItemFlags.CAN_HAVE_CHILDREN + remote_item = f"{remote_path}/{entry.relative_path}" + + if is_dir: + sub_count, sub_size = get_remote_directory_info(remote_item) + file_count += sub_count + total_size += sub_size + else: + file_count += 1 + stat_result, stat_entry = omni.client.stat(remote_item) + if stat_result == omni.client.Result.OK: + total_size += stat_entry.size + + return file_count, total_size + + +def list_remote_directories(remote_path: str) -> list[str]: + result, entries = omni.client.list(remote_path) + if result != omni.client.Result.OK: + return [] + return sorted([e.relative_path for e in entries if e.flags & omni.client.ItemFlags.CAN_HAVE_CHILDREN]) + + +def download_file(remote_path: str, local_path: str, overwrite: bool = False) -> bool: + try: + if os.path.exists(local_path) and not overwrite: + return True + os.makedirs(os.path.dirname(local_path), exist_ok=True) + result = omni.client.copy(remote_path, local_path) + return result == omni.client.Result.OK + except Exception as e: + print(f"Error downloading {remote_path}: {e}") + return False + + +def download_directory_recursive(remote_path: str, local_base: str, progress_bar, overwrite: bool = False) -> int: + downloaded = 0 + result, entries = omni.client.list(remote_path) + if result != omni.client.Result.OK: + return downloaded + + for entry in entries: + is_dir = entry.flags & omni.client.ItemFlags.CAN_HAVE_CHILDREN + remote_item = f"{remote_path}/{entry.relative_path}" + local_item = os.path.join(local_base, entry.relative_path) + + if is_dir: + os.makedirs(local_item, exist_ok=True) + downloaded += download_directory_recursive(remote_item, local_item, progress_bar, overwrite) + else: + progress_bar.set_description(f"Downloading {entry.relative_path[:50]}") + if download_file(remote_item, local_item, overwrite): + downloaded += 1 + progress_bar.update(1) + + return downloaded + + +def download_category(category: str, subset: str = None, overwrite: bool = False) -> None: + remote_dir = f"{ISAAC_NUCLEUS_DIR}/{category}" + local_dir = os.path.join(OFFLINE_ASSETS_DIR, category) + + if subset: + remote_dir = f"{remote_dir}/{subset}" + local_dir = os.path.join(local_dir, subset) + + base_category = category.split("/")[0] + desc = KNOWN_CATEGORIES.get(base_category, "Assets") + + print(f"\n{'=' * 70}") + print(f"šŸ“¦ {category}: {desc}") + print(f"{'=' * 70}") + print(f"Source: {remote_dir}") + print(f"Target: {local_dir}") + + result, _ = omni.client.stat(remote_dir) + if result != omni.client.Result.OK: + print(f"āš ļø Directory not found: {remote_dir}") + return + + print("šŸ“Š Analyzing...") + file_count, total_size = get_remote_directory_info(remote_dir) + + if file_count == 0: + print("āœ“ No files") + return + + print(f" Files: {file_count:,} | Size: {format_size(total_size)}") + + os.makedirs(local_dir, exist_ok=True) + with tqdm(total=file_count, unit="file", desc="Progress") as pbar: + download_directory_recursive(remote_dir, local_dir, pbar, overwrite) + + print(f"āœ“ Completed {category}") + + +def verify_downloads() -> None: + print("\n" + "=" * 70) + print("šŸ“Š Downloaded Assets Summary") + print("=" * 70) + print(f"Location: {OFFLINE_ASSETS_DIR}\n") + + total_size = 0 + total_files = 0 + + if not os.path.exists(OFFLINE_ASSETS_DIR): + print("āŒ Offline assets directory not found") + return + + for item in sorted(os.listdir(OFFLINE_ASSETS_DIR)): + item_path = os.path.join(OFFLINE_ASSETS_DIR, item) + if os.path.isdir(item_path): + size = get_local_directory_size(item_path) + files = sum(1 for _ in Path(item_path).rglob("*") if _.is_file()) + total_size += size + total_files += files + print(f"āœ“ {item:<20} {files:>8,} files {format_size(size):>10}") + + print("=" * 70) + print(f"{'TOTAL':<20} {total_files:>8,} files {format_size(total_size):>10}") + print("=" * 70) + + +def get_isaaclab_essential_categories() -> list[str]: + return ["IsaacLab", "Props", "Environments", "Materials"] + + +def main(): + parser = argparse.ArgumentParser( + description="Download Isaac assets for offline use", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--categories", nargs="+", default=["isaaclab-essentials"]) + parser.add_argument("--subset", type=str, help="Subset within category") + parser.add_argument("--verify-only", action="store_true") + parser.add_argument("--overwrite", action="store_true") + parser.add_argument("--list", action="store_true") + + args = parser.parse_args() + + try: + print("\n" + "=" * 70) + print("šŸš€ Isaac Lab Offline Asset Downloader") + print("=" * 70) + print(f"Nucleus: {ISAAC_NUCLEUS_DIR}") + print(f"Local: {OFFLINE_ASSETS_DIR}") + print("=" * 70) + + if args.list: + categories = list_remote_directories(ISAAC_NUCLEUS_DIR) + print("\nšŸ“‚ Available under Isaac/:") + for cat in categories: + desc = KNOWN_CATEGORIES.get(cat, "") + print(f" ā€¢ {cat:<20} {desc}") + return + + if args.verify_only: + verify_downloads() + return + + if "all" in args.categories: + categories = list_remote_directories(ISAAC_NUCLEUS_DIR) + elif "isaaclab-essentials" in args.categories: + categories = get_isaaclab_essential_categories() + print("\nšŸ“‹ Isaac Lab essentials:") + for cat in categories: + print(f" ā€¢ {cat}") + else: + categories = args.categories + + if args.subset: + print(f"šŸ” Subset: {args.subset}") + + print("\nšŸ“Š Calculating size...") + total_files = 0 + total_size = 0 + + for category in categories: + remote_dir = f"{ISAAC_NUCLEUS_DIR}/{category}" + if args.subset: + remote_dir = f"{remote_dir}/{args.subset}" + + result, _ = omni.client.stat(remote_dir) + if result == omni.client.Result.OK: + files, size = get_remote_directory_info(remote_dir) + total_files += files + total_size += size + print(f" {category}: {files:,} files ({format_size(size)})") + + if total_files == 0: + print("\nāŒ No files to download") + return + + print(f"\nšŸ“¦ Total: {total_files:,} files ({format_size(total_size)})") + + response = input("\nProceed? [y/N]: ") + if response.lower() not in ["y", "yes"]: + print("āŒ Cancelled") + return + + print("\nšŸ”½ Downloading...") + for category in categories: + try: + download_category(category, args.subset, args.overwrite) + except KeyboardInterrupt: + print("\nāš ļø Interrupted") + raise + except Exception as e: + print(f"āŒ Error: {category}: {e}") + + verify_downloads() + print("\nāœ… Complete! Use --offline flag with Isaac Lab commands.\n") + + finally: + simulation_app.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/reinforcement_learning/skrl/play.py b/scripts/reinforcement_learning/skrl/play.py index 089ec756197..3ad95406366 100644 --- a/scripts/reinforcement_learning/skrl/play.py +++ b/scripts/reinforcement_learning/skrl/play.py @@ -139,7 +139,7 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, expe if args_cli.ml_framework.startswith("jax"): skrl.config.jax.backend = "jax" if args_cli.ml_framework == "jax" else "numpy" - # randomly sample a seed if seed = -1 + # randomly sample a seed if seed = -1 if args_cli.seed == -1: args_cli.seed = random.randint(0, 10000) diff --git a/scripts/sim2sim_transfer/rsl_rl_transfer.py b/scripts/sim2sim_transfer/rsl_rl_transfer.py index 0ec1b389879..63036e2408d 100644 --- a/scripts/sim2sim_transfer/rsl_rl_transfer.py +++ b/scripts/sim2sim_transfer/rsl_rl_transfer.py @@ -148,7 +148,6 @@ def get_joint_mappings(args_cli, action_space_dim): @hydra_task_config(args_cli.task, args_cli.agent) def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agent_cfg: RslRlBaseRunnerCfg): """Play with RSL-RL agent with policy transfer capabilities.""" - # override configurations with non-hydra CLI arguments agent_cfg = cli_args.update_rsl_rl_cfg(agent_cfg, args_cli) env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index e986d4b664a..885af0d221c 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -115,6 +115,8 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa self._livestream: Literal[0, 1, 2] # 0: Disabled, 1: WebRTC public, 2: WebRTC private self._offscreen_render: bool # 0: Disabled, 1: Enabled self._sim_experience_file: str # Experience file to load + self._offline: bool # 0: Disabled, 1: Enabled + self._offline_strict: bool # True: fail on missing assets, False: fall back to Nucleus # Exposed to train scripts self.device_id: int # device ID for GPU simulation (defaults to 0) @@ -138,6 +140,10 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa # Set animation recording settings self._set_animation_recording_settings(launcher_args) + # Set up offline mode if enabled + if self._offline: + self._setup_offline_mode() + # Hide play button callback if the timeline is stopped import omni.timeline @@ -174,6 +180,16 @@ def app(self) -> SimulationApp: else: raise RuntimeError("The `AppLauncher.app` member cannot be retrieved until the class is initialized.") + @property + def offline(self) -> bool: + """Whether offline asset resolution is enabled.""" + return self._offline + + @property + def offline_strict(self) -> bool: + """Whether offline mode is strict (fails on missing assets) or permissive (falls back to Nucleus).""" + return self._offline_strict + """ Operations. """ @@ -369,6 +385,27 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: " exceeded, then the animation is not recorded." ), ) + arg_group.add_argument( + "--offline", + action="store_true", + default=AppLauncher._APPLAUNCHER_CFG_INFO["offline"][1], + help=( + "Enable offline asset resolution (strict mode). When enabled, asset paths from Nucleus/S3 " + "servers are automatically redirected to local storage (offline_assets/). " + "Fails immediately if an asset is not found locally. " + "Can also be enabled via the OFFLINE environment variable." + ), + ) + arg_group.add_argument( + "--offline-permissive", + action="store_true", + default=AppLauncher._APPLAUNCHER_CFG_INFO["offline_permissive"][1], + help=( + "Enable offline asset resolution (permissive mode). Same as --offline but falls back " + "to Nucleus if an asset is not found locally (may timeout if truly offline). " + "Can also be enabled via the OFFLINE_PERMISSIVE environment variable." + ), + ) # special flag for backwards compatibility # Corresponding to the beginning of the function, @@ -389,6 +426,8 @@ def add_app_launcher_args(parser: argparse.ArgumentParser) -> None: "device": ([str], "cuda:0"), "experience": ([str], ""), "rendering_mode": ([str], "balanced"), + "offline": ([bool], False), + "offline_permissive": ([bool], False), } """A dictionary of arguments added manually by the :meth:`AppLauncher.add_app_launcher_args` method. @@ -493,6 +532,9 @@ def _config_resolution(self, launcher_args: dict): # Handle device and distributed settings self._resolve_device_settings(launcher_args) + # Handle offline mode settings + self._resolve_offline_settings(launcher_args) + # Handle experience file settings self._resolve_experience_file(launcher_args) @@ -704,6 +746,38 @@ def _resolve_device_settings(self, launcher_args: dict): print(f"[INFO][AppLauncher]: Using device: {device}") + def _resolve_offline_settings(self, launcher_args: dict): + """Resolve offline mode related settings. + + This method checks both the --offline/--offline-permissive CLI arguments + and the OFFLINE/OFFLINE_PERMISSIVE environment variables. + CLI arguments take precedence. + """ + # Check environment variables + offline_env = os.environ.get("OFFLINE", "0").lower() in ("1", "true", "yes") + offline_permissive_env = os.environ.get("OFFLINE_PERMISSIVE", "0").lower() in ("1", "true", "yes") + + # Check CLI arguments (pop them so they don't get passed to SimulationApp) + offline_arg = launcher_args.pop("offline", False) + offline_permissive_arg = launcher_args.pop("offline_permissive", False) + + # Determine if offline mode is enabled (any flag/env var) + self._offline = offline_arg or offline_permissive_arg or offline_env or offline_permissive_env + + # Determine strict mode: strict unless permissive is explicitly requested + # --offline or OFFLINE=1 -> strict mode (fail on missing assets) + # --offline-permissive or OFFLINE_PERMISSIVE=1 -> permissive mode (fall back to Nucleus) + self._offline_strict = not (offline_permissive_arg or offline_permissive_env) + + if self._offline: + mode = "STRICT" if self._offline_strict else "PERMISSIVE" + print(f"[INFO][AppLauncher]: Offline mode ENABLED ({mode})") + + if self._offline_strict: + print("[INFO][AppLauncher]: Missing assets will cause an error (no Nucleus fallback)") + else: + print("[INFO][AppLauncher]: Missing assets will fall back to Nucleus (may timeout if offline)") + def _resolve_experience_file(self, launcher_args: dict): """Resolve experience file related settings.""" # Check if input keywords contain an 'experience' file setting @@ -763,6 +837,21 @@ def _resolve_experience_file(self, launcher_args: dict): self._sim_experience_file = os.path.abspath(self._sim_experience_file) print(f"[INFO][AppLauncher]: Loading experience file: {self._sim_experience_file}") + def _setup_offline_mode(self): + """Configure offline asset resolution. + + This method is called after the SimulationApp is created to ensure + all necessary modules are available. + """ + try: + from isaaclab.utils import setup_offline_mode + + setup_offline_mode(strict=self._offline_strict) + print("[INFO][AppLauncher]: Offline asset resolution configured") + except ImportError as e: + print(f"[WARN][AppLauncher]: Could not enable offline mode: {e}") + print("[WARN][AppLauncher]: Please ensure isaaclab.utils.asset_resolver is available") + def _resolve_anim_recording_settings(self, launcher_args: dict): """Resolve animation recording settings.""" diff --git a/source/isaaclab/isaaclab/utils/__init__.py b/source/isaaclab/isaaclab/utils/__init__.py index 1295715857f..58ab0cc3f2f 100644 --- a/source/isaaclab/isaaclab/utils/__init__.py +++ b/source/isaaclab/isaaclab/utils/__init__.py @@ -17,3 +17,4 @@ from .timer import Timer from .types import * from .version import * +from .asset_resolver import * diff --git a/source/isaaclab/isaaclab/utils/asset_resolver.py b/source/isaaclab/isaaclab/utils/asset_resolver.py new file mode 100644 index 00000000000..6d8384abae5 --- /dev/null +++ b/source/isaaclab/isaaclab/utils/asset_resolver.py @@ -0,0 +1,683 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Offline Asset Resolver for Isaac Lab. + +This module redirects Nucleus asset paths to a local mirror of the Isaac/ directory. + +Path Resolution: + Nucleus URL: .../Assets/Isaac/5.1/Isaac/{path} + Local Path: offline_assets/{path} + + Examples: + - /Isaac/IsaacLab/Robots/Unitree/Go2/go2.usd ā†’ offline_assets/IsaacLab/Robots/Unitree/Go2/go2.usd + - /Isaac/Props/UIElements/arrow_x.usd ā†’ offline_assets/Props/UIElements/arrow_x.usd + +Usage: + Automatically enabled via AppLauncher --offline flag, or manually: + + from isaaclab.utils import setup_offline_mode + setup_offline_mode() +""" + +from __future__ import annotations + +import os +import re + + +class OfflineAssetResolver: + """Singleton class to manage offline asset path resolution.""" + + _instance: OfflineAssetResolver | None = None + _enabled: bool = False + _strict: bool = True # If True, fail immediately when asset not found locally + _initialized: bool = False + _hooks_installed: bool = False + _offline_assets_dir: str | None = None + _failed_assets: list[tuple[str, str]] = [] + _warned_assets: set[str] = set() # Track assets we've already warned about + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._failed_assets = [] + cls._instance._warned_assets = set() + return cls._instance + + def _initialize(self): + """Initialize the resolver.""" + if self._initialized: + return + + self.isaaclab_path = os.environ.get("ISAACLAB_PATH", os.getcwd()) + self._offline_assets_dir = os.path.join(self.isaaclab_path, "offline_assets") + self._failed_assets = [] + self._warned_assets = set() + self._initialized = True + + print("[OfflineAssetResolver] Initialized") + print(f" Offline assets: {self._offline_assets_dir}") + + def enable(self, strict: bool = True): + """ + Enable offline asset resolution. + + Args: + strict: If True (default), raise an error when an asset is not found locally. + If False, fall back to Nucleus (may cause timeouts if offline). + """ + self._initialize() + self._enabled = True + self._strict = strict + self._failed_assets = [] + self._warned_assets = set() + + mode = "STRICT" if strict else "permissive" + print(f"[OfflineAssetResolver] Offline mode ENABLED ({mode})") + print(f" Local mirror of Isaac/ at: {self._offline_assets_dir}") + + if strict: + print(" Missing assets will cause an error (no Nucleus fallback)") + else: + print(" Missing assets will fall back to Nucleus (may timeout if offline)") + + if not os.path.exists(self._offline_assets_dir): + print("[OfflineAssetResolver] āš ļø WARNING: offline_assets directory not found!") + print(" Run: ./isaaclab.sh -p scripts/offline_setup/download_assets.py") + + def disable(self): + """Disable offline asset resolution.""" + self._enabled = False + print("[OfflineAssetResolver] Offline mode DISABLED") + + def is_enabled(self) -> bool: + return self._enabled + + def is_strict(self) -> bool: + return self._strict + + def are_hooks_installed(self) -> bool: + return self._hooks_installed + + def set_hooks_installed(self, value: bool = True): + self._hooks_installed = value + + def add_failed_asset(self, original_path: str, relative_path: str): + """Track a failed asset resolution.""" + for _, rel in self._failed_assets: + if rel == relative_path: + return + self._failed_assets.append((original_path, relative_path)) + + def get_failed_assets(self) -> list[tuple[str, str]]: + """Get list of (original_path, relative_path) for failed resolutions.""" + return self._failed_assets.copy() + + def resolve_path(self, asset_path: str) -> str: + """ + Resolve an asset path to offline storage. + """ + if not self._enabled or not isinstance(asset_path, str) or not asset_path: + return asset_path + + # Skip if already a local path + if os.path.exists(asset_path): + return asset_path + + # Extract the path relative to /Isaac/ + relative_path = self._extract_relative_path(asset_path) + + if relative_path: + offline_path = os.path.join(self._offline_assets_dir, relative_path) + + # Try exact path first + if os.path.exists(offline_path): + print(f"[OfflineAssetResolver] āœ“ Using offline: {relative_path}") + return offline_path + + # Try case-insensitive fallback + resolved = self._find_case_insensitive(relative_path) + if resolved: + print(f"[OfflineAssetResolver] āœ“ Using offline (case-adjusted): {resolved}") + return os.path.join(self._offline_assets_dir, resolved) + + # Asset not found locally + self._handle_missing_asset(asset_path, relative_path) + + return asset_path + + def _handle_missing_asset(self, original_path: str, relative_path: str): + """Handle a missing asset - either error (strict) or warn (permissive).""" + # Only warn once per asset + if relative_path in self._warned_assets: + return + self._warned_assets.add(relative_path) + self.add_failed_asset(original_path, relative_path) + + # Build download command + parts = relative_path.split("/") + if len(parts) >= 2 and parts[0] == "IsaacLab": + download_cmd = f"./isaaclab.sh -p scripts/offline_setup/download_assets.py --categories IsaacLab/{parts[1]}" + else: + download_cmd = f"./isaaclab.sh -p scripts/offline_setup/download_assets.py --categories {parts[0]}" + + if self._strict: + # Strict mode: raise an error + error_msg = f""" +{"=" * 70} +[OfflineAssetResolver] āœ— ASSET NOT FOUND (offline mode) +{"=" * 70} +Missing: {relative_path} +Expected: {os.path.join(self._offline_assets_dir, relative_path)} + +To download this asset, run: + {download_cmd} + +Or to allow Nucleus fallback (may timeout if offline), use: + --offline-permissive instead of --offline +{"=" * 70} +""" + raise FileNotFoundError(error_msg) + # Permissive mode: warn and fall back to Nucleus + print(f"\n[OfflineAssetResolver] āš ļø MISSING: {relative_path}") + print("[OfflineAssetResolver] Falling back to Nucleus (may timeout if offline)") + print(f"[OfflineAssetResolver] Download with: {download_cmd}\n") + + def _extract_relative_path(self, asset_path: str) -> str | None: + """Extract the path relative to /Isaac/ from a Nucleus URL.""" + match = re.search(r"/Assets/Isaac/[\d.]+/Isaac/(.+)$", asset_path) + if match: + return match.group(1) + return None + + def _find_case_insensitive(self, relative_path: str) -> str | None: + """Find a file using case-insensitive matching.""" + parts = relative_path.replace("\\", "/").split("/") + current_dir = self._offline_assets_dir + resolved_parts = [] + + for part in parts: + if not os.path.exists(current_dir): + return None + + try: + entries = os.listdir(current_dir) + except (OSError, PermissionError): + return None + + if part in entries: + resolved_parts.append(part) + current_dir = os.path.join(current_dir, part) + continue + + part_lower = part.lower() + part_normalized = part_lower.replace("-", "_") + + found = None + for entry in entries: + entry_lower = entry.lower() + entry_normalized = entry_lower.replace("-", "_") + + if entry_lower == part_lower or entry_normalized == part_normalized: + found = entry + break + + if found: + resolved_parts.append(found) + current_dir = os.path.join(current_dir, found) + else: + return None + + final_path = os.path.join(self._offline_assets_dir, *resolved_parts) + if os.path.exists(final_path): + return "/".join(resolved_parts) + + return None + + def get_offline_assets_dir(self) -> str: + self._initialize() + return self._offline_assets_dir + + +# Global resolver instance +_resolver = OfflineAssetResolver() + + +# ============================================================================= +# Public API Functions +# ============================================================================= + + +def enable_offline_mode(strict: bool = True): + """ + Enable offline asset resolution. + + Args: + strict: If True (default), raise an error when an asset is not found locally. + If False, fall back to Nucleus (may cause timeouts if offline). + """ + _resolver.enable(strict=strict) + + +def disable_offline_mode(): + _resolver.disable() + + +def is_offline_mode_enabled() -> bool: + return _resolver.is_enabled() + + +def is_offline_mode_strict() -> bool: + return _resolver.is_strict() + + +def resolve_asset_path(asset_path: str) -> str: + return _resolver.resolve_path(asset_path) + + +def get_offline_assets_dir() -> str: + return _resolver.get_offline_assets_dir() + + +def get_failed_assets() -> list[tuple[str, str]]: + return _resolver.get_failed_assets() + + +def _install_spawn_hooks(): + """Install hooks on spawn functions (Level 1).""" + try: + from isaaclab.sim.spawners.from_files import from_files as from_files_module + + if hasattr(from_files_module, "spawn_from_usd"): + original_spawn_from_usd = from_files_module.spawn_from_usd + + def patched_spawn_from_usd(prim_path, cfg, *args, **kwargs): + if is_offline_mode_enabled() and hasattr(cfg, "usd_path"): + cfg.usd_path = resolve_asset_path(cfg.usd_path) + return original_spawn_from_usd(prim_path, cfg, *args, **kwargs) + + from_files_module.spawn_from_usd = patched_spawn_from_usd + print("[OfflineAssetResolver] Installed spawn_from_usd hook") + + if hasattr(from_files_module, "spawn_from_urdf"): + original_spawn_from_urdf = from_files_module.spawn_from_urdf + + def patched_spawn_from_urdf(prim_path, cfg, *args, **kwargs): + if is_offline_mode_enabled() and hasattr(cfg, "asset_path"): + cfg.asset_path = resolve_asset_path(cfg.asset_path) + return original_spawn_from_urdf(prim_path, cfg, *args, **kwargs) + + from_files_module.spawn_from_urdf = patched_spawn_from_urdf + print("[OfflineAssetResolver] Installed spawn_from_urdf hook") + + except ImportError as e: + print(f"[OfflineAssetResolver] Could not install spawn hooks: {e}") + + +def _install_file_hooks(): + """Install hooks on file loading functions (Level 2).""" + try: + import isaaclab.utils.assets as assets_module + + if hasattr(assets_module, "read_file"): + original_read_file = assets_module.read_file + + def patched_read_file(path: str): + if is_offline_mode_enabled(): + return original_read_file(resolve_asset_path(path)) + return original_read_file(path) + + assets_module.read_file = patched_read_file + print("[OfflineAssetResolver] Installed read_file hook") + + if hasattr(assets_module, "retrieve_file_path"): + original_retrieve = assets_module.retrieve_file_path + + def patched_retrieve_file_path(path: str, *args, **kwargs): + if is_offline_mode_enabled(): + return original_retrieve(resolve_asset_path(path), *args, **kwargs) + return original_retrieve(path, *args, **kwargs) + + assets_module.retrieve_file_path = patched_retrieve_file_path + print("[OfflineAssetResolver] Installed retrieve_file_path hook") + + if hasattr(assets_module, "check_file_path"): + original_check_file_path = assets_module.check_file_path + + def patched_check_file_path(path: str, *args, **kwargs): + if is_offline_mode_enabled(): + return original_check_file_path(resolve_asset_path(path), *args, **kwargs) + return original_check_file_path(path, *args, **kwargs) + + assets_module.check_file_path = patched_check_file_path + print("[OfflineAssetResolver] Installed check_file_path hook") + + except ImportError: + pass + + +def _install_config_hooks(): + """Install hooks on config classes (Level 3).""" + try: + import isaaclab.sim as sim_utils + + if hasattr(sim_utils, "UsdFileCfg"): + original_usd_init = sim_utils.UsdFileCfg.__init__ + + def patched_usd_init(self, *args, **kwargs): + original_usd_init(self, *args, **kwargs) + if hasattr(self, "usd_path") and is_offline_mode_enabled(): + self.usd_path = resolve_asset_path(self.usd_path) + + sim_utils.UsdFileCfg.__init__ = patched_usd_init + print("[OfflineAssetResolver] Installed UsdFileCfg hook") + + if hasattr(sim_utils, "GroundPlaneCfg"): + original_ground_init = sim_utils.GroundPlaneCfg.__init__ + + def patched_ground_init(self, *args, **kwargs): + original_ground_init(self, *args, **kwargs) + if hasattr(self, "usd_path") and is_offline_mode_enabled(): + self.usd_path = resolve_asset_path(self.usd_path) + + sim_utils.GroundPlaneCfg.__init__ = patched_ground_init + print("[OfflineAssetResolver] Installed GroundPlaneCfg hook") + + if hasattr(sim_utils, "PreviewSurfaceCfg"): + original_surface_init = sim_utils.PreviewSurfaceCfg.__init__ + + def patched_surface_init(self, *args, **kwargs): + original_surface_init(self, *args, **kwargs) + if hasattr(self, "texture_file") and is_offline_mode_enabled() and self.texture_file: + self.texture_file = resolve_asset_path(self.texture_file) + + sim_utils.PreviewSurfaceCfg.__init__ = patched_surface_init + print("[OfflineAssetResolver] Installed PreviewSurfaceCfg hook") + + except ImportError: + pass + + +def _install_env_hooks(): + """Install hooks on environment classes (Level 4).""" + try: + from isaaclab.envs import ManagerBasedEnv + + original_manager_init = ManagerBasedEnv.__init__ + + def patched_manager_init(self, cfg, *args, **kwargs): + if is_offline_mode_enabled(): + patch_config_for_offline_mode(cfg) + original_manager_init(self, cfg, *args, **kwargs) + + ManagerBasedEnv.__init__ = patched_manager_init + print("[OfflineAssetResolver] Installed ManagerBasedEnv hook") + except ImportError: + pass + + try: + from isaaclab.envs import DirectRLEnv + + original_direct_init = DirectRLEnv.__init__ + + def patched_direct_init(self, cfg, *args, **kwargs): + if is_offline_mode_enabled(): + patch_config_for_offline_mode(cfg) + original_direct_init(self, cfg, *args, **kwargs) + + DirectRLEnv.__init__ = patched_direct_init + print("[OfflineAssetResolver] Installed DirectRLEnv hook") + except ImportError: + pass + + try: + from isaaclab.envs import DirectMARLEnv + + original_marl_init = DirectMARLEnv.__init__ + + def patched_marl_init(self, cfg, *args, **kwargs): + if is_offline_mode_enabled(): + patch_config_for_offline_mode(cfg) + original_marl_init(self, cfg, *args, **kwargs) + + DirectMARLEnv.__init__ = patched_marl_init + print("[OfflineAssetResolver] Installed DirectMARLEnv hook") + except ImportError: + pass + + +def _install_asset_hooks(): + """Install hooks on asset classes (Level 5).""" + try: + from isaaclab.assets import Articulation + + original_articulation_init = Articulation.__init__ + + def patched_articulation_init(self, cfg, *args, **kwargs): + if is_offline_mode_enabled(): + patch_config_for_offline_mode(cfg) + original_articulation_init(self, cfg, *args, **kwargs) + + Articulation.__init__ = patched_articulation_init + print("[OfflineAssetResolver] Installed Articulation hook") + except ImportError: + pass + + try: + from isaaclab.assets import RigidObject + + original_rigid_init = RigidObject.__init__ + + def patched_rigid_init(self, cfg, *args, **kwargs): + if is_offline_mode_enabled(): + patch_config_for_offline_mode(cfg) + original_rigid_init(self, cfg, *args, **kwargs) + + RigidObject.__init__ = patched_rigid_init + print("[OfflineAssetResolver] Installed RigidObject hook") + except ImportError: + pass + + +def install_hooks(): + """Install ALL path resolution hooks at multiple levels.""" + if _resolver.are_hooks_installed(): + return + + _install_spawn_hooks() + _install_file_hooks() + _install_config_hooks() + _install_env_hooks() + _install_asset_hooks() + + _resolver.set_hooks_installed(True) + + +def _is_nucleus_path(path: str) -> bool: + """Check if a path is a Nucleus/S3 URL.""" + if not isinstance(path, str): + return False + return ( + "omniverse-content-production" in path + or "nucleus" in path.lower() + or path.startswith("omniverse://") + or "/Assets/Isaac/" in path + ) + + +def _patch_object_recursive(obj, visited: set, depth: int = 0, max_depth: int = 15) -> int: + """Recursively patch asset paths in an object.""" + if depth > max_depth: + return 0 + + obj_id = id(obj) + if obj_id in visited: + return 0 + visited.add(obj_id) + + patches = 0 + + if obj is None or isinstance(obj, (str, int, float, bool, bytes)): + return 0 + + if isinstance(obj, dict): + for key, value in obj.items(): + if isinstance(value, str) and _is_nucleus_path(value): + resolved = resolve_asset_path(value) + if resolved != value: + obj[key] = resolved + patches += 1 + else: + patches += _patch_object_recursive(value, visited, depth + 1, max_depth) + return patches + + if isinstance(obj, (list, tuple)): + for i, item in enumerate(obj): + if isinstance(item, str) and _is_nucleus_path(item): + resolved = resolve_asset_path(item) + if resolved != item: + if isinstance(obj, list): + obj[i] = resolved + patches += 1 + else: + patches += _patch_object_recursive(item, visited, depth + 1, max_depth) + return patches + + if hasattr(obj, "__dict__"): + asset_attrs = [ + "usd_path", + "texture_file", + "asset_path", + "file_path", + "mesh_file", + "network_file", + "policy_path", + "checkpoint_path", + ] + + for attr in asset_attrs: + if hasattr(obj, attr): + value = getattr(obj, attr) + if isinstance(value, str) and _is_nucleus_path(value): + resolved = resolve_asset_path(value) + if resolved != value: + try: + setattr(obj, attr, resolved) + patches += 1 + except (AttributeError, TypeError): + pass + + try: + for attr_name in dir(obj): + if attr_name.startswith("_"): + continue + try: + attr_value = getattr(obj, attr_name) + if callable(attr_value) and not hasattr(attr_value, "__dict__"): + continue + patches += _patch_object_recursive(attr_value, visited, depth + 1, max_depth) + except (AttributeError, TypeError, RuntimeError): + continue + except (TypeError, RuntimeError): + pass + + return patches + + +def patch_config_for_offline_mode(cfg): + """Patch a configuration object to use offline assets.""" + if not is_offline_mode_enabled(): + return + + visited = set() + patches = _patch_object_recursive(cfg, visited) + + if patches > 0: + print(f"[OfflineAssetResolver] Patched {patches} config paths") + + +def setup_offline_mode(strict: bool = True): + """ + Set up offline mode with all hooks. + + Args: + strict: If True (default), raise an error when an asset is not found locally. + If False, fall back to Nucleus (may cause timeouts if offline). + """ + enable_offline_mode(strict=strict) + install_hooks() + print("[OfflineAssetResolver] Offline mode fully configured") + + +def print_summary(): + """Print summary of any failed asset resolutions.""" + failed = get_failed_assets() + if failed: + print("\n" + "=" * 70) + print("[OfflineAssetResolver] āš ļø MISSING ASSETS SUMMARY") + print("=" * 70) + print(f"The following {len(failed)} asset(s) were not found locally:\n") + + categories = {} + for original, relative in failed: + parts = relative.split("/") + if len(parts) >= 2 and parts[0] == "IsaacLab": + cat = f"IsaacLab/{parts[1]}" + else: + cat = parts[0] if parts else "Unknown" + + if cat not in categories: + categories[cat] = [] + categories[cat].append(relative) + + for cat, assets in sorted(categories.items()): + print(f" {cat}:") + for asset in assets: + print(f" - {asset}") + + print("\nTo download missing assets, run:") + for cat in sorted(categories.keys()): + print(f" ./isaaclab.sh -p scripts/offline_setup/download_assets.py --categories {cat}") + print("=" * 70 + "\n") + else: + print("[OfflineAssetResolver] āœ“ All assets resolved successfully") + + +# Legacy aliases +def install_spawn_hooks(): + install_hooks() + + +def install_file_hooks(): + install_hooks() + + +def install_env_hooks(): + install_hooks() + + +def install_path_hooks(): + install_hooks() + + +__all__ = [ + "enable_offline_mode", + "disable_offline_mode", + "is_offline_mode_enabled", + "is_offline_mode_strict", + "resolve_asset_path", + "get_offline_assets_dir", + "get_failed_assets", + "patch_config_for_offline_mode", + "install_hooks", + "setup_offline_mode", + "print_summary", + "install_spawn_hooks", + "install_file_hooks", + "install_env_hooks", + "install_path_hooks", +]