From ddaf379463b74e71603ebebcb09a5fb17adfb229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Tue, 3 Feb 2026 10:48:46 +0800 Subject: [PATCH 1/5] feat(server): support multi ingress gateway mode --- docs/architecture.md | 2 +- server/README.md | 16 ++ server/README_zh.md | 16 ++ server/example.config.toml | 10 + server/src/config.py | 149 ++++++++++++-- server/tests/test_auth_middleware.py | 6 +- server/tests/test_config.py | 287 +++++++++++++++++++++++++-- server/tests/test_docker_service.py | 4 +- server/tests/testdata/config.toml | 4 +- 9 files changed, 453 insertions(+), 41 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 50e97e54..8a39c080 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -224,7 +224,7 @@ The pluggable architecture allows implementing custom runtimes by: **Features:** - Dynamic endpoint generation based on sandbox ID and port -- Supports both domain-based and wildcard-domain routing +- Supports both domain-based and wildcard routing - Reverse proxy to sandbox container ports - Automatic cleanup when sandbox terminates diff --git a/server/README.md b/server/README.md index a37aab09..c4bb0af2 100644 --- a/server/README.md +++ b/server/README.md @@ -112,6 +112,22 @@ cp example.batchsandbox-template.yaml ~/batchsandbox-template.yaml ``` Further reading on Docker container security: https://docs.docker.com/engine/security/ +**Ingress exposure (tunnel | gateway)** + ```toml + [ingress] + mode = "tunnel" # docker runtime only supports tunnel + # gateway.address = "https://*.example.com" # scheme optional; defaults to http when omitted + # gateway.route.mode = "wildcard" # wildcard | header | uri + ``` + - `mode=tunnel`: default; required when `runtime.type=docker`. + - `mode=gateway`: configure external ingress. + - `gateway.address`: wildcard domain required when `gateway.route.mode=wildcard`; otherwise must be domain, IP, or IP:port (http/https optional; no userinfo). + - `gateway.route.mode`: `wildcard` (host-based wildcard), `header` (header-based), `uri` (path-prefix). + - Response format examples: + - `wildcard`: `-.example.com/path/to/request` + - `header`: `10.0.0.1:8000/path/to/request` with header `OPEN-SANDBOX-INGRESS: -` + - `uri`: `10.0.0.1:8000///path/to/request` + ### (Optional) Egress sidecar for `networkPolicy` - Configure the sidecar image (used only when requests include `networkPolicy`): diff --git a/server/README_zh.md b/server/README_zh.md index 80d425b1..36a38951 100644 --- a/server/README_zh.md +++ b/server/README_zh.md @@ -117,6 +117,22 @@ seccomp_profile = "" # 配置文件路径或名称；为空使用 Docker ``` 更多 Docker 安全参考：https://docs.docker.com/engine/security/ +**Ingress 暴露（tunnel | gateway）** +```toml +[ingress] +mode = "tunnel" # Docker 运行时仅支持 tunnel +# gateway.address = "https://*.example.com" # scheme 可选，缺省按 http +# gateway.route.mode = "wildcard" # wildcard | header | uri +``` +- `mode=tunnel`：默认；当 `runtime.type=docker` 时必须使用。 +- `mode=gateway`：配置外部入口。 + - `gateway.address`：当 `gateway.route.mode=wildcard` 时必须是泛域名；其他模式需为域名/IP 或 IP:port，可选 http/https（不允许带用户信息）。 + - `gateway.route.mode`：`wildcard`（域名泛匹配）、`header`（基于请求头）、`uri`（基于路径前缀）。 + - 返回示例： + - `wildcard`：`-.example.com/path/to/request` + - `header`：`10.0.0.1:8000/path/to/request`，请求头 `OPEN-SANDBOX-INGRESS: -` + - `uri`：`10.0.0.1:8000///path/to/request` + ### （可选）Egress sidecar 配置与使用 - 配置镜像（仅在请求携带 `networkPolicy` 时注入）： diff --git a/server/example.config.toml b/server/example.config.toml index deec9c44..a456413f 100644 --- a/server/example.config.toml +++ b/server/example.config.toml @@ -28,6 +28,16 @@ log_level = "INFO" api_key = "" +[ingress] +# Ingress exposure mode: tunnel (default) or gateway +mode = "tunnel" +# Gateway-only settings (uncomment when mode = "gateway") +# gateway.address = "https://gateway.opensandbox.io" # scheme optional; defaults to http when omitted +# gateway.route.mode = "wildcard" # wildcard | header | uri +# gateway.route.header-name = "X-SANDBOX-ID" +# gateway.route.uri-prefix = "/sandbox" + + [runtime] # Runtime selection (docker | kubernetes) # ----------------------------------------------------------------- diff --git a/server/src/config.py b/server/src/config.py index b1f24366..509eac04 100644 --- a/server/src/config.py +++ b/server/src/config.py @@ -21,10 +21,13 @@ from __future__ import annotations +import ipaddress import logging import os +import re from pathlib import Path from typing import Any, Literal, Optional +from urllib.parse import urlparse from pydantic import BaseModel, Field, ValidationError, model_validator @@ -38,31 +41,139 @@ CONFIG_ENV_VAR = "SANDBOX_CONFIG_PATH" DEFAULT_CONFIG_PATH = Path.home() / ".sandbox.toml" +_DOMAIN_RE = re.compile(r"^(?=.{1,253}$)(?!-)[A-Za-z0-9-]{1,63}(?:\.[A-Za-z0-9-]{1,63})+$") +_WILDCARD_DOMAIN_RE = re.compile(r"^\*\.(?!-)[A-Za-z0-9-]{1,63}(?:\.[A-Za-z0-9-]{1,63})+$") +_IPV4_WITH_PORT_RE = re.compile(r"^(?P(?:\d{1,3}\.){3}\d{1,3})(?::(?P\d{1,5}))?$") -class RouterConfig(BaseModel): - """Configuration for external sandbox router endpoints.""" +INGRESS_MODE_TUNNEL = "tunnel" +INGRESS_MODE_GATEWAY = "gateway" +GATEWAY_ROUTE_MODE_WILDCARD = "wildcard" +GATEWAY_ROUTE_MODE_HEADER = "header" +GATEWAY_ROUTE_MODE_URI = "uri" - domain: Optional[str] = Field( - default=None, - description="Base domain used to expose sandbox endpoints (e.g., 'opensandbox.io').", + +def _is_valid_ip(host: str) -> bool: + try: + ipaddress.ip_address(host) + return True + except ValueError: + return False + + +def _is_valid_ip_or_ip_port(address: str) -> bool: + match = _IPV4_WITH_PORT_RE.match(address) + if not match: + return False + ip_str = match.group("ip") + if not _is_valid_ip(ip_str): + return False + port_str = match.group("port") + if port_str is None: + return True + try: + port = int(port_str) + except ValueError: + return False + return 1 <= port <= 65535 + + +def _is_valid_domain(host: str) -> bool: + return bool(_DOMAIN_RE.match(host)) + + +def _is_wildcard_domain(host: str) -> bool: + return bool(_WILDCARD_DOMAIN_RE.match(host)) + + +class GatewayRouteModeConfig(BaseModel): + """Routing strategy for gateway ingress exposure.""" + + mode: Literal[ + GATEWAY_ROUTE_MODE_WILDCARD, + GATEWAY_ROUTE_MODE_HEADER, + GATEWAY_ROUTE_MODE_URI, + ] = Field( + ..., + description="Routing mode used by the gateway (wildcard, header, uri).", + ) + + class Config: + populate_by_name = True + + +class GatewayConfig(BaseModel): + """Gateway mode configuration for ingress exposure.""" + + address: str = Field( + ..., + description=( + "Gateway host used to expose sandboxes. Supports domain, IP, or IP:port. " + "Scheme (http/https) is optional; when omitted, http is assumed." + ), min_length=1, ) - wildcard_domain: Optional[str] = Field( + route: GatewayRouteModeConfig = Field( + ..., + description="Routing mode configuration used by the gateway.", + ) + + +class IngressConfig(BaseModel): + """Configuration for exposing sandbox ingress.""" + + mode: Literal["tunnel", "gateway"] = Field( + default=INGRESS_MODE_TUNNEL, + description="Ingress exposure mode (tunnel or gateway).", + ) + gateway: Optional[GatewayConfig] = Field( default=None, - alias="wildcard-domain", - description="Wildcard domain pattern (e.g., '*.opensandbox.io') used for sandbox endpoints.", - min_length=1, + description="Gateway configuration required when mode = 'gateway'.", ) @model_validator(mode="after") - def validate_domain_choice(self) -> "RouterConfig": - if bool(self.domain) == bool(self.wildcard_domain): - raise ValueError("Exactly one of domain or wildcard-domain must be specified in [router].") + def validate_ingress_mode(self) -> "IngressConfig": + if self.mode == INGRESS_MODE_GATEWAY and self.gateway is None: + raise ValueError("gateway block must be provided when ingress.mode = 'gateway'.") + if self.mode == INGRESS_MODE_TUNNEL and self.gateway is not None: + raise ValueError("gateway block must be omitted unless ingress.mode = 'gateway'.") + + if self.mode == INGRESS_MODE_GATEWAY and self.gateway: + route_mode = self.gateway.route.mode + address_raw = self.gateway.address + scheme = "http" + hostport = address_raw + if "://" in address_raw: + parsed = urlparse(address_raw) + if parsed.scheme not in {"http", "https"}: + raise ValueError( + "ingress.gateway.address scheme must be http or https when provided." + ) + if parsed.username or parsed.password: + raise ValueError("ingress.gateway.address must not include userinfo.") + if not parsed.hostname: + raise ValueError("ingress.gateway.address must include a host.") + scheme = parsed.scheme + hostport = parsed.hostname + if parsed.port: + hostport = f"{hostport}:{parsed.port}" + + if route_mode == GATEWAY_ROUTE_MODE_WILDCARD: + if not _is_wildcard_domain(hostport): + raise ValueError( + "ingress.gateway.address must be a wildcard domain (e.g., *.example.com) " + "when gateway.route.mode is wildcard." + ) + else: + if "*" in hostport: + raise ValueError( + "ingress.gateway.address must not contain wildcard when gateway.route.mode is not wildcard." + ) + if not (_is_valid_domain(hostport) or _is_valid_ip_or_ip_port(hostport)): + raise ValueError( + "ingress.gateway.address must be a valid domain, IP, or IP:port when gateway.route.mode is not wildcard." + ) return self - class Config: - populate_by_name = True - class ServerConfig(BaseModel): """FastAPI server configuration.""" @@ -203,7 +314,7 @@ class AppConfig(BaseModel): runtime: RuntimeConfig = Field(..., description="Sandbox runtime configuration.") kubernetes: Optional[KubernetesRuntimeConfig] = None agent_sandbox: Optional["AgentSandboxRuntimeConfig"] = None - router: Optional[RouterConfig] = None + ingress: Optional[IngressConfig] = None docker: DockerConfig = Field(default_factory=DockerConfig) @model_validator(mode="after") @@ -213,6 +324,8 @@ def validate_runtime_blocks(self) -> "AppConfig": raise ValueError("Kubernetes block must be omitted when runtime.type = 'docker'.") if self.agent_sandbox is not None: raise ValueError("agent_sandbox block must be omitted when runtime.type = 'docker'.") + if self.ingress is not None and self.ingress.mode != INGRESS_MODE_TUNNEL: + raise ValueError("ingress.mode must be 'tunnel' when runtime.type = 'docker'.") elif self.runtime.type == "kubernetes": if self.kubernetes is None: self.kubernetes = KubernetesRuntimeConfig() @@ -314,7 +427,9 @@ def get_config_path() -> Path: "AppConfig", "ServerConfig", "RuntimeConfig", - "RouterConfig", + "IngressConfig", + "GatewayConfig", + "GatewayRouteModeConfig", "DockerConfig", "KubernetesRuntimeConfig", "DEFAULT_CONFIG_PATH", diff --git a/server/tests/test_auth_middleware.py b/server/tests/test_auth_middleware.py index b9c6fc0a..ef12165e 100644 --- a/server/tests/test_auth_middleware.py +++ b/server/tests/test_auth_middleware.py @@ -15,15 +15,15 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from src.config import AppConfig, RouterConfig, RuntimeConfig, ServerConfig +from src.config import AppConfig, IngressConfig, RuntimeConfig, ServerConfig from src.middleware.auth import AuthMiddleware def _app_config_with_api_key() -> AppConfig: return AppConfig( server=ServerConfig(api_key="secret-key"), - runtime=RuntimeConfig(type="docker", execd_image="ghcr.io/opensandbox/platform:latest"), - router=RouterConfig(domain="opensandbox.io"), + runtime=RuntimeConfig(type="docker", execd_image="opensandbox/execd:latest"), + ingress=IngressConfig(mode="tunnel"), ) diff --git a/server/tests/test_config.py b/server/tests/test_config.py index 9fa34b2a..e862e7f2 100644 --- a/server/tests/test_config.py +++ b/server/tests/test_config.py @@ -17,7 +17,14 @@ import pytest from src import config as config_module -from src.config import AppConfig, RouterConfig, RuntimeConfig, ServerConfig +from src.config import ( + AppConfig, + GatewayConfig, + GatewayRouteModeConfig, + IngressConfig, + RuntimeConfig, + ServerConfig, +) def _reset_config(monkeypatch): @@ -36,11 +43,13 @@ def test_load_config_from_file(tmp_path, monkeypatch): api_key = "secret" [runtime] - type = "docker" - execd_image = "ghcr.io/opensandbox/platform:test" + type = "kubernetes" + execd_image = "opensandbox/execd:test" - [router] - domain = "opensandbox.io" + [ingress] + mode = "gateway" + gateway.address = "https://*.opensandbox.io" + gateway.route.mode = "wildcard" """ ) config_path = tmp_path / "config.toml" @@ -51,11 +60,14 @@ def test_load_config_from_file(tmp_path, monkeypatch): assert loaded.server.port == 9000 assert loaded.server.log_level == "DEBUG" assert loaded.server.api_key == "secret" - assert loaded.runtime.type == "docker" - assert loaded.runtime.execd_image == "ghcr.io/opensandbox/platform:test" - assert loaded.router is not None - assert loaded.router.domain == "opensandbox.io" - assert loaded.docker.network_mode == "host" + assert loaded.runtime.type == "kubernetes" + assert loaded.runtime.execd_image == "opensandbox/execd:test" + assert loaded.ingress is not None + assert loaded.ingress.mode == "gateway" + assert loaded.ingress.gateway is not None + assert loaded.ingress.gateway.address == "https://*.opensandbox.io" + assert loaded.ingress.gateway.route.mode == "wildcard" + assert loaded.kubernetes is not None def test_docker_runtime_disallows_kubernetes_block(): @@ -68,15 +80,258 @@ def test_docker_runtime_disallows_kubernetes_block(): def test_kubernetes_runtime_fills_missing_block(): server_cfg = ServerConfig() - runtime_cfg = RuntimeConfig(type="kubernetes", execd_image="ghcr.io/opensandbox/platform:latest") + runtime_cfg = RuntimeConfig(type="kubernetes", execd_image="opensandbox/execd:latest") app_cfg = AppConfig(server=server_cfg, runtime=runtime_cfg) assert app_cfg.kubernetes is not None -def test_router_requires_exactly_one_domain(): +def test_ingress_gateway_requires_gateway_block(): with pytest.raises(ValueError): - RouterConfig(domain=None, wildcard_domain=None) + IngressConfig(mode="gateway") + cfg = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="uri"), + ), + ) + assert cfg.gateway.route.mode == "uri" + + +def test_gateway_address_validation_for_wildcard_mode(): + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + cfg = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://*.opensandbox.io", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + assert cfg.gateway.address == "https://*.opensandbox.io" + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="10.0.0.1", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="http://10.0.0.1:8080", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="10.0.0.1:8080", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + + +def test_gateway_route_mode_allows_wildcard_alias(): + cfg = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://*.opensandbox.io", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + assert cfg.gateway.route.mode == "wildcard" + + +def test_gateway_address_validation_for_non_wildcard_mode(): + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="*.opensandbox.io", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="not a host", + route=GatewayRouteModeConfig(mode="uri"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="gateway.opensandbox.io:8080", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="10.0.0.1:70000", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="ftp://gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="http://", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="http://user:pass@gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="http://gateway.opensandbox.io:8080", + route=GatewayRouteModeConfig(mode="header"), + ), + ) with pytest.raises(ValueError): - RouterConfig(domain="opensandbox.io", wildcard_domain="*.opensandbox.io") - cfg = RouterConfig(domain="opensandbox.io") - assert cfg.domain == "opensandbox.io" + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="10.0.0.1:0", + route=GatewayRouteModeConfig(mode="uri"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="10.0.0.1:abc", + route=GatewayRouteModeConfig(mode="uri"), + ), + ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="http://[::1]", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + cfg = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="uri"), + ), + ) + assert cfg.gateway.address == "gateway.opensandbox.io" + cfg_scheme = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + assert cfg_scheme.gateway.address == "https://gateway.opensandbox.io" + cfg_ip = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="10.0.0.1", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + assert cfg_ip.gateway.address == "10.0.0.1" + cfg_ip_port = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="10.0.0.1:8080", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + assert cfg_ip_port.gateway.address == "10.0.0.1:8080" + cfg_ip_port_scheme = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="http://10.0.0.1:8080", + route=GatewayRouteModeConfig(mode="uri"), + ), + ) + assert cfg_ip_port_scheme.gateway.address == "http://10.0.0.1:8080" + + +def test_gateway_address_allows_scheme_less_defaults(): + cfg = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="*.example.com", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + assert cfg.gateway.address == "*.example.com" + cfg2 = IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://*.example.com", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + assert cfg2.gateway.address == "https://*.example.com" + + +def test_tunnel_mode_rejects_gateway_block(): + with pytest.raises(ValueError): + IngressConfig( + mode="tunnel", + gateway=GatewayConfig( + address="gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + + +def test_docker_runtime_rejects_gateway_ingress(): + server_cfg = ServerConfig() + runtime_cfg = RuntimeConfig(type="docker", execd_image="busybox:latest") + with pytest.raises(ValueError): + AppConfig( + server=server_cfg, + runtime=runtime_cfg, + ingress=IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="gateway.opensandbox.io", + route=GatewayRouteModeConfig(mode="header"), + ), + ), + ) + # tunnel remains valid + app_cfg = AppConfig( + server=server_cfg, + runtime=runtime_cfg, + ingress=IngressConfig(mode="tunnel"), + ) + assert app_cfg.ingress.mode == "tunnel" diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py index fdd77b06..75564385 100644 --- a/server/tests/test_docker_service.py +++ b/server/tests/test_docker_service.py @@ -18,7 +18,7 @@ import pytest from fastapi import HTTPException, status -from src.config import AppConfig, RouterConfig, RuntimeConfig, ServerConfig +from src.config import AppConfig, IngressConfig, RuntimeConfig, ServerConfig from src.services.constants import SANDBOX_ID_LABEL, SandboxErrorCodes from src.services.docker import DockerSandboxService, PendingSandbox from src.services.helpers import parse_memory_limit, parse_nano_cpus, parse_timestamp @@ -39,7 +39,7 @@ def _app_config() -> AppConfig: return AppConfig( server=ServerConfig(), runtime=RuntimeConfig(type="docker", execd_image="ghcr.io/opensandbox/platform:latest"), - router=RouterConfig(domain="opensandbox.io"), + ingress=IngressConfig(mode="tunnel"), ) diff --git a/server/tests/testdata/config.toml b/server/tests/testdata/config.toml index 1a31baf5..953dc988 100644 --- a/server/tests/testdata/config.toml +++ b/server/tests/testdata/config.toml @@ -22,5 +22,5 @@ api_key = "test-api-key-12345" type = "docker" execd_image = "ghcr.io/opensandbox/platform:latest" -[router] -domain = "opensandbox.io" +[ingress] +mode = "tunnel" From b032495b6b0b924a811c23586fdfa53c4d868005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Tue, 3 Feb 2026 12:48:18 +0800 Subject: [PATCH 2/5] feat(server): rename `tunnel` mode to `direct` mode --- server/README.md | 10 ++--- server/README_zh.md | 10 ++--- server/example.config.toml | 6 +-- server/src/config.py | 36 +++++----------- server/tests/test_auth_middleware.py | 2 +- server/tests/test_config.py | 64 ++++++++++++---------------- server/tests/test_docker_service.py | 2 +- server/tests/testdata/config.toml | 2 +- 8 files changed, 55 insertions(+), 77 deletions(-) diff --git a/server/README.md b/server/README.md index c4bb0af2..b9fb7e3f 100644 --- a/server/README.md +++ b/server/README.md @@ -112,16 +112,16 @@ cp example.batchsandbox-template.yaml ~/batchsandbox-template.yaml ``` Further reading on Docker container security: https://docs.docker.com/engine/security/ -**Ingress exposure (tunnel | gateway)** +**Ingress exposure (direct | gateway)** ```toml [ingress] - mode = "tunnel" # docker runtime only supports tunnel - # gateway.address = "https://*.example.com" # scheme optional; defaults to http when omitted + mode = "direct" # docker runtime only supports direct + # gateway.address = "*.example.com" # host only (domain or IP[:port]); scheme is not allowed # gateway.route.mode = "wildcard" # wildcard | header | uri ``` - - `mode=tunnel`: default; required when `runtime.type=docker`. + - `mode=direct`: default; required when `runtime.type=docker` (client ↔ sandbox direct reachability, no L7 gateway). - `mode=gateway`: configure external ingress. - - `gateway.address`: wildcard domain required when `gateway.route.mode=wildcard`; otherwise must be domain, IP, or IP:port (http/https optional; no userinfo). + - `gateway.address`: wildcard domain required when `gateway.route.mode=wildcard`; otherwise must be domain, IP, or IP:port. Do not include scheme; clients decide http/https. - `gateway.route.mode`: `wildcard` (host-based wildcard), `header` (header-based), `uri` (path-prefix). - Response format examples: - `wildcard`: `-.example.com/path/to/request` diff --git a/server/README_zh.md b/server/README_zh.md index 36a38951..eaae0b9d 100644 --- a/server/README_zh.md +++ b/server/README_zh.md @@ -117,16 +117,16 @@ seccomp_profile = "" # 配置文件路径或名称；为空使用 Docker ``` 更多 Docker 安全参考：https://docs.docker.com/engine/security/ -**Ingress 暴露（tunnel | gateway）** +**Ingress 暴露（direct | gateway）** ```toml [ingress] -mode = "tunnel" # Docker 运行时仅支持 tunnel -# gateway.address = "https://*.example.com" # scheme 可选，缺省按 http +mode = "direct" # Docker 运行时仅支持 direct（直连，无 L7 网关） +# gateway.address = "*.example.com" # 仅主机（域名/IP 或 IP:port），不允许带 scheme # gateway.route.mode = "wildcard" # wildcard | header | uri ``` -- `mode=tunnel`：默认；当 `runtime.type=docker` 时必须使用。 +- `mode=direct`：默认；当 `runtime.type=docker` 时必须使用（客户端与 sandbox 直连，不经过网关）。 - `mode=gateway`：配置外部入口。 - - `gateway.address`：当 `gateway.route.mode=wildcard` 时必须是泛域名；其他模式需为域名/IP 或 IP:port，可选 http/https（不允许带用户信息）。 + - `gateway.address`：当 `gateway.route.mode=wildcard` 时必须是泛域名；其他模式需为域名/IP 或 IP:port。不允许携带 scheme，客户端自行选择 http/https。 - `gateway.route.mode`：`wildcard`（域名泛匹配）、`header`（基于请求头）、`uri`（基于路径前缀）。 - 返回示例： - `wildcard`：`-.example.com/path/to/request` diff --git a/server/example.config.toml b/server/example.config.toml index a456413f..0877eff9 100644 --- a/server/example.config.toml +++ b/server/example.config.toml @@ -29,10 +29,10 @@ api_key = "" [ingress] -# Ingress exposure mode: tunnel (default) or gateway -mode = "tunnel" +# Ingress exposure mode: direct (default) or gateway +mode = "direct" # Gateway-only settings (uncomment when mode = "gateway") -# gateway.address = "https://gateway.opensandbox.io" # scheme optional; defaults to http when omitted +# gateway.address = "gateway.opensandbox.io" # host (domain/IP or IP:port), no scheme # gateway.route.mode = "wildcard" # wildcard | header | uri # gateway.route.header-name = "X-SANDBOX-ID" # gateway.route.uri-prefix = "/sandbox" diff --git a/server/src/config.py b/server/src/config.py index 509eac04..3ce56cb3 100644 --- a/server/src/config.py +++ b/server/src/config.py @@ -45,7 +45,7 @@ _WILDCARD_DOMAIN_RE = re.compile(r"^\*\.(?!-)[A-Za-z0-9-]{1,63}(?:\.[A-Za-z0-9-]{1,63})+$") _IPV4_WITH_PORT_RE = re.compile(r"^(?P(?:\d{1,3}\.){3}\d{1,3})(?::(?P\d{1,5}))?$") -INGRESS_MODE_TUNNEL = "tunnel" +INGRESS_MODE_DIRECT = "direct" INGRESS_MODE_GATEWAY = "gateway" GATEWAY_ROUTE_MODE_WILDCARD = "wildcard" GATEWAY_ROUTE_MODE_HEADER = "header" @@ -106,10 +106,7 @@ class GatewayConfig(BaseModel): address: str = Field( ..., - description=( - "Gateway host used to expose sandboxes. Supports domain, IP, or IP:port. " - "Scheme (http/https) is optional; when omitted, http is assumed." - ), + description="Gateway host used to expose sandboxes (domain or IP, may include :port; scheme is not allowed).", min_length=1, ) route: GatewayRouteModeConfig = Field( @@ -121,9 +118,9 @@ class GatewayConfig(BaseModel): class IngressConfig(BaseModel): """Configuration for exposing sandbox ingress.""" - mode: Literal["tunnel", "gateway"] = Field( - default=INGRESS_MODE_TUNNEL, - description="Ingress exposure mode (tunnel or gateway).", + mode: Literal[INGRESS_MODE_DIRECT, INGRESS_MODE_GATEWAY] = Field( + default=INGRESS_MODE_DIRECT, + description="Ingress exposure mode (direct or gateway).", ) gateway: Optional[GatewayConfig] = Field( default=None, @@ -134,28 +131,15 @@ class IngressConfig(BaseModel): def validate_ingress_mode(self) -> "IngressConfig": if self.mode == INGRESS_MODE_GATEWAY and self.gateway is None: raise ValueError("gateway block must be provided when ingress.mode = 'gateway'.") - if self.mode == INGRESS_MODE_TUNNEL and self.gateway is not None: + if self.mode == INGRESS_MODE_DIRECT and self.gateway is not None: raise ValueError("gateway block must be omitted unless ingress.mode = 'gateway'.") if self.mode == INGRESS_MODE_GATEWAY and self.gateway: route_mode = self.gateway.route.mode address_raw = self.gateway.address - scheme = "http" hostport = address_raw if "://" in address_raw: - parsed = urlparse(address_raw) - if parsed.scheme not in {"http", "https"}: - raise ValueError( - "ingress.gateway.address scheme must be http or https when provided." - ) - if parsed.username or parsed.password: - raise ValueError("ingress.gateway.address must not include userinfo.") - if not parsed.hostname: - raise ValueError("ingress.gateway.address must include a host.") - scheme = parsed.scheme - hostport = parsed.hostname - if parsed.port: - hostport = f"{hostport}:{parsed.port}" + raise ValueError("ingress.gateway.address must not include a scheme; clients choose http/https.") if route_mode == GATEWAY_ROUTE_MODE_WILDCARD: if not _is_wildcard_domain(hostport): @@ -324,8 +308,8 @@ def validate_runtime_blocks(self) -> "AppConfig": raise ValueError("Kubernetes block must be omitted when runtime.type = 'docker'.") if self.agent_sandbox is not None: raise ValueError("agent_sandbox block must be omitted when runtime.type = 'docker'.") - if self.ingress is not None and self.ingress.mode != INGRESS_MODE_TUNNEL: - raise ValueError("ingress.mode must be 'tunnel' when runtime.type = 'docker'.") + if self.ingress is not None and self.ingress.mode != INGRESS_MODE_DIRECT: + raise ValueError("ingress.mode must be 'direct' when runtime.type = 'docker'.") elif self.runtime.type == "kubernetes": if self.kubernetes is None: self.kubernetes = KubernetesRuntimeConfig() @@ -430,6 +414,8 @@ def get_config_path() -> Path: "IngressConfig", "GatewayConfig", "GatewayRouteModeConfig", + "INGRESS_MODE_DIRECT", + "INGRESS_MODE_GATEWAY", "DockerConfig", "KubernetesRuntimeConfig", "DEFAULT_CONFIG_PATH", diff --git a/server/tests/test_auth_middleware.py b/server/tests/test_auth_middleware.py index ef12165e..ee142279 100644 --- a/server/tests/test_auth_middleware.py +++ b/server/tests/test_auth_middleware.py @@ -23,7 +23,7 @@ def _app_config_with_api_key() -> AppConfig: return AppConfig( server=ServerConfig(api_key="secret-key"), runtime=RuntimeConfig(type="docker", execd_image="opensandbox/execd:latest"), - ingress=IngressConfig(mode="tunnel"), + ingress=IngressConfig(mode="direct"), ) diff --git a/server/tests/test_config.py b/server/tests/test_config.py index e862e7f2..31b87377 100644 --- a/server/tests/test_config.py +++ b/server/tests/test_config.py @@ -48,7 +48,7 @@ def test_load_config_from_file(tmp_path, monkeypatch): [ingress] mode = "gateway" - gateway.address = "https://*.opensandbox.io" + gateway.address = "*.opensandbox.io" gateway.route.mode = "wildcard" """ ) @@ -65,7 +65,7 @@ def test_load_config_from_file(tmp_path, monkeypatch): assert loaded.ingress is not None assert loaded.ingress.mode == "gateway" assert loaded.ingress.gateway is not None - assert loaded.ingress.gateway.address == "https://*.opensandbox.io" + assert loaded.ingress.gateway.address == "*.opensandbox.io" assert loaded.ingress.gateway.route.mode == "wildcard" assert loaded.kubernetes is not None @@ -91,7 +91,7 @@ def test_ingress_gateway_requires_gateway_block(): cfg = IngressConfig( mode="gateway", gateway=GatewayConfig( - address="https://gateway.opensandbox.io", + address="gateway.opensandbox.io", route=GatewayRouteModeConfig(mode="uri"), ), ) @@ -103,18 +103,18 @@ def test_gateway_address_validation_for_wildcard_mode(): IngressConfig( mode="gateway", gateway=GatewayConfig( - address="https://gateway.opensandbox.io", + address="gateway.opensandbox.io", route=GatewayRouteModeConfig(mode="wildcard"), ), ) cfg = IngressConfig( mode="gateway", gateway=GatewayConfig( - address="https://*.opensandbox.io", + address="*.opensandbox.io", route=GatewayRouteModeConfig(mode="wildcard"), ), ) - assert cfg.gateway.address == "https://*.opensandbox.io" + assert cfg.gateway.address == "*.opensandbox.io" with pytest.raises(ValueError): IngressConfig( mode="gateway", @@ -139,13 +139,21 @@ def test_gateway_address_validation_for_wildcard_mode(): route=GatewayRouteModeConfig(mode="wildcard"), ), ) + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://*.opensandbox.io", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) def test_gateway_route_mode_allows_wildcard_alias(): cfg = IngressConfig( mode="gateway", gateway=GatewayConfig( - address="https://*.opensandbox.io", + address="*.opensandbox.io", route=GatewayRouteModeConfig(mode="wildcard"), ), ) @@ -249,14 +257,6 @@ def test_gateway_address_validation_for_non_wildcard_mode(): ), ) assert cfg.gateway.address == "gateway.opensandbox.io" - cfg_scheme = IngressConfig( - mode="gateway", - gateway=GatewayConfig( - address="https://gateway.opensandbox.io", - route=GatewayRouteModeConfig(mode="header"), - ), - ) - assert cfg_scheme.gateway.address == "https://gateway.opensandbox.io" cfg_ip = IngressConfig( mode="gateway", gateway=GatewayConfig( @@ -273,14 +273,6 @@ def test_gateway_address_validation_for_non_wildcard_mode(): ), ) assert cfg_ip_port.gateway.address == "10.0.0.1:8080" - cfg_ip_port_scheme = IngressConfig( - mode="gateway", - gateway=GatewayConfig( - address="http://10.0.0.1:8080", - route=GatewayRouteModeConfig(mode="uri"), - ), - ) - assert cfg_ip_port_scheme.gateway.address == "http://10.0.0.1:8080" def test_gateway_address_allows_scheme_less_defaults(): @@ -292,20 +284,20 @@ def test_gateway_address_allows_scheme_less_defaults(): ), ) assert cfg.gateway.address == "*.example.com" - cfg2 = IngressConfig( - mode="gateway", - gateway=GatewayConfig( - address="https://*.example.com", - route=GatewayRouteModeConfig(mode="wildcard"), - ), - ) - assert cfg2.gateway.address == "https://*.example.com" + with pytest.raises(ValueError): + IngressConfig( + mode="gateway", + gateway=GatewayConfig( + address="https://*.example.com", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) -def test_tunnel_mode_rejects_gateway_block(): +def test_direct_mode_rejects_gateway_block(): with pytest.raises(ValueError): IngressConfig( - mode="tunnel", + mode="direct", gateway=GatewayConfig( address="gateway.opensandbox.io", route=GatewayRouteModeConfig(mode="header"), @@ -328,10 +320,10 @@ def test_docker_runtime_rejects_gateway_ingress(): ), ), ) - # tunnel remains valid + # direct remains valid app_cfg = AppConfig( server=server_cfg, runtime=runtime_cfg, - ingress=IngressConfig(mode="tunnel"), + ingress=IngressConfig(mode="direct"), ) - assert app_cfg.ingress.mode == "tunnel" + assert app_cfg.ingress.mode == "direct" diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py index 75564385..fa6101f6 100644 --- a/server/tests/test_docker_service.py +++ b/server/tests/test_docker_service.py @@ -39,7 +39,7 @@ def _app_config() -> AppConfig: return AppConfig( server=ServerConfig(), runtime=RuntimeConfig(type="docker", execd_image="ghcr.io/opensandbox/platform:latest"), - ingress=IngressConfig(mode="tunnel"), + ingress=IngressConfig(mode="direct"), ) diff --git a/server/tests/testdata/config.toml b/server/tests/testdata/config.toml index 953dc988..a3d190f7 100644 --- a/server/tests/testdata/config.toml +++ b/server/tests/testdata/config.toml @@ -23,4 +23,4 @@ type = "docker" execd_image = "ghcr.io/opensandbox/platform:latest" [ingress] -mode = "tunnel" +mode = "direct" From ff15bdaaf8238687c8d25005daf0600d5ff04ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Tue, 3 Feb 2026 15:34:37 +0800 Subject: [PATCH 3/5] feat(server): ingress mode for kubernetes service --- server/README.md | 5 +- server/README_zh.md | 4 +- server/example.config.k8s.toml | 4 ++ server/example.config.k8s.zh.toml | 4 ++ server/example.config.toml | 20 ++------ server/example.config.zh.toml | 10 ++-- server/src/config.py | 1 - server/src/services/helpers.py | 46 ++++++++++++++++++ .../services/k8s/agent_sandbox_provider.py | 12 ++++- .../src/services/k8s/batchsandbox_provider.py | 34 ++++++------- server/src/services/k8s/kubernetes_service.py | 6 ++- server/src/services/k8s/provider_factory.py | 15 ++++-- server/src/services/k8s/workload_provider.py | 3 +- .../tests/k8s/test_agent_sandbox_provider.py | 4 +- .../tests/k8s/test_batchsandbox_provider.py | 10 ++-- server/tests/test_ingress.py | 48 +++++++++++++++++++ 16 files changed, 170 insertions(+), 56 deletions(-) create mode 100644 server/tests/test_ingress.py diff --git a/server/README.md b/server/README.md index b9fb7e3f..3f494969 100644 --- a/server/README.md +++ b/server/README.md @@ -117,15 +117,14 @@ cp example.batchsandbox-template.yaml ~/batchsandbox-template.yaml [ingress] mode = "direct" # docker runtime only supports direct # gateway.address = "*.example.com" # host only (domain or IP[:port]); scheme is not allowed - # gateway.route.mode = "wildcard" # wildcard | header | uri + # gateway.route.mode = "wildcard" # wildcard | uri (header not yet supported) ``` - `mode=direct`: default; required when `runtime.type=docker` (client ↔ sandbox direct reachability, no L7 gateway). - `mode=gateway`: configure external ingress. - `gateway.address`: wildcard domain required when `gateway.route.mode=wildcard`; otherwise must be domain, IP, or IP:port. Do not include scheme; clients decide http/https. - - `gateway.route.mode`: `wildcard` (host-based wildcard), `header` (header-based), `uri` (path-prefix). + - `gateway.route.mode`: `wildcard` (host-based wildcard), `uri` (path-prefix). `header` is not yet supported. - Response format examples: - `wildcard`: `-.example.com/path/to/request` - - `header`: `10.0.0.1:8000/path/to/request` with header `OPEN-SANDBOX-INGRESS: -` - `uri`: `10.0.0.1:8000///path/to/request` ### (Optional) Egress sidecar for `networkPolicy` diff --git a/server/README_zh.md b/server/README_zh.md index eaae0b9d..c8ad00b3 100644 --- a/server/README_zh.md +++ b/server/README_zh.md @@ -122,12 +122,12 @@ seccomp_profile = "" # 配置文件路径或名称；为空使用 Docker [ingress] mode = "direct" # Docker 运行时仅支持 direct（直连，无 L7 网关） # gateway.address = "*.example.com" # 仅主机（域名/IP 或 IP:port），不允许带 scheme -# gateway.route.mode = "wildcard" # wildcard | header | uri +# gateway.route.mode = "wildcard" # wildcard | uri（header 暂未支持） ``` - `mode=direct`：默认；当 `runtime.type=docker` 时必须使用（客户端与 sandbox 直连，不经过网关）。 - `mode=gateway`：配置外部入口。 - `gateway.address`：当 `gateway.route.mode=wildcard` 时必须是泛域名；其他模式需为域名/IP 或 IP:port。不允许携带 scheme，客户端自行选择 http/https。 - - `gateway.route.mode`：`wildcard`（域名泛匹配）、`header`（基于请求头）、`uri`（基于路径前缀）。 + - `gateway.route.mode`：`wildcard`（域名泛匹配）、`uri`（基于路径前缀）；`header` 模式暂未支持。 - 返回示例： - `wildcard`：`-.example.com/path/to/request` - `header`：`10.0.0.1:8000/path/to/request`，请求头 `OPEN-SANDBOX-INGRESS: -` diff --git a/server/example.config.k8s.toml b/server/example.config.k8s.toml index 48fc8cae..9f584093 100644 --- a/server/example.config.k8s.toml +++ b/server/example.config.k8s.toml @@ -47,3 +47,7 @@ workload_provider = "batchsandbox" # Path to the BatchSandbox template file # Replace with your path batchsandbox_template_file = "~/batchsandbox-template.yaml" + +[ingress] +# Ingress exposure mode: direct (default) or gateway +mode = "direct" diff --git a/server/example.config.k8s.zh.toml b/server/example.config.k8s.zh.toml index 4231a9ca..3c528e56 100644 --- a/server/example.config.k8s.zh.toml +++ b/server/example.config.k8s.zh.toml @@ -47,3 +47,7 @@ workload_provider = "batchsandbox" # Path to the BatchSandbox template file # Replace with your path batchsandbox_template_file = "~/batchsandbox-template.yaml" + +[ingress] +# Ingress exposure mode: direct (default) or gateway +mode = "direct" diff --git a/server/example.config.toml b/server/example.config.toml index 0877eff9..04f678a2 100644 --- a/server/example.config.toml +++ b/server/example.config.toml @@ -21,22 +21,8 @@ # ----------------------------------------------------------------- host = "127.0.0.1" port = 8080 - log_level = "INFO" - -# Shared API key for the OPEN-SANDBOX-API-KEY header (leave empty only for dev) -api_key = "" - - -[ingress] -# Ingress exposure mode: direct (default) or gateway -mode = "direct" -# Gateway-only settings (uncomment when mode = "gateway") -# gateway.address = "gateway.opensandbox.io" # host (domain/IP or IP:port), no scheme -# gateway.route.mode = "wildcard" # wildcard | header | uri -# gateway.route.header-name = "X-SANDBOX-ID" -# gateway.route.uri-prefix = "/sandbox" - +# api_key = "your-secret-api-key" # Optional: Uncomment to enable API key authentication [runtime] # Runtime selection (docker | kubernetes) @@ -59,3 +45,7 @@ apparmor_profile = "" pids_limit = 512 # Seccomp profile: empty string uses Docker default; set to an absolute path for a custom profile seccomp_profile = "" + +[ingress] +# Ingress exposure mode: direct (default) or gateway +mode = "direct" diff --git a/server/example.config.zh.toml b/server/example.config.zh.toml index 9b78a7dc..3c246775 100644 --- a/server/example.config.zh.toml +++ b/server/example.config.zh.toml @@ -21,12 +21,8 @@ # ----------------------------------------------------------------- host = "127.0.0.1" port = 8080 - log_level = "INFO" - -# Shared API key for the OPEN-SANDBOX-API-KEY header (leave empty only for dev) -api_key = "" - +# api_key = "your-secret-api-key" # Optional: Uncomment to enable API key authentication [runtime] # Runtime selection (docker | kubernetes) @@ -49,3 +45,7 @@ apparmor_profile = "" pids_limit = 512 # Seccomp profile: empty string uses Docker default; set to an absolute path for a custom profile seccomp_profile = "" + +[ingress] +# Ingress exposure mode: direct (default) or gateway +mode = "direct" diff --git a/server/src/config.py b/server/src/config.py index 3ce56cb3..5db90c1d 100644 --- a/server/src/config.py +++ b/server/src/config.py @@ -27,7 +27,6 @@ import re from pathlib import Path from typing import Any, Literal, Optional -from urllib.parse import urlparse from pydantic import BaseModel, Field, ValidationError, model_validator diff --git a/server/src/services/helpers.py b/server/src/services/helpers.py index 96ae7c8d..cd9e55d0 100644 --- a/server/src/services/helpers.py +++ b/server/src/services/helpers.py @@ -27,6 +27,13 @@ from typing import Dict, Optional from src.api.schema import Sandbox, SandboxFilter +from src.config import ( + GATEWAY_ROUTE_MODE_HEADER, + GATEWAY_ROUTE_MODE_URI, + GATEWAY_ROUTE_MODE_WILDCARD, + INGRESS_MODE_GATEWAY, + IngressConfig, +) logger = logging.getLogger(__name__) @@ -141,9 +148,48 @@ def matches_filter(sandbox: Sandbox, filter_: SandboxFilter) -> bool: return True +# ============================================================================ +# Ingress helpers +# ============================================================================ +def format_ingress_endpoint( + ingress_config: Optional[IngressConfig], + sandbox_id: str, + port: int, +) -> Optional[str]: + """ + Build an ingress-based endpoint string for a sandbox. + + Returns None when ingress is not in gateway mode or when the route mode is + not supported (e.g., header mode is intentionally skipped until Endpoint + schema can carry headers). + """ + if not ingress_config or ingress_config.mode != INGRESS_MODE_GATEWAY: + return None + gateway_cfg = ingress_config.gateway + if gateway_cfg is None: + return None + + address = gateway_cfg.address + route_mode = gateway_cfg.route.mode + + if route_mode == GATEWAY_ROUTE_MODE_WILDCARD: + base = address[2:] if address.startswith("*.") else address + return f"{sandbox_id}-{port}.{base}" + + if route_mode == GATEWAY_ROUTE_MODE_URI: + return f"{address}/{sandbox_id}/{port}" + + if route_mode == GATEWAY_ROUTE_MODE_HEADER: + # TODO(Pangjiping): Header mode intentionally not emitted until Endpoint schema supports headers. + return None + + return None + + __all__ = [ "parse_memory_limit", "parse_nano_cpus", "parse_timestamp", + "format_ingress_endpoint", "matches_filter", ] diff --git a/server/src/services/k8s/agent_sandbox_provider.py b/server/src/services/k8s/agent_sandbox_provider.py index 4bf8bf59..b3b292a9 100644 --- a/server/src/services/k8s/agent_sandbox_provider.py +++ b/server/src/services/k8s/agent_sandbox_provider.py @@ -29,6 +29,8 @@ ) from src.api.schema import ImageSpec +from src.config import IngressConfig +from src.services.helpers import format_ingress_endpoint from src.services.constants import SANDBOX_ID_LABEL from src.services.k8s.agent_sandbox_template import AgentSandboxTemplateManager from src.services.k8s.client import K8sClient @@ -48,6 +50,7 @@ def __init__( template_file_path: Optional[str] = None, shutdown_policy: str = "Delete", service_account: Optional[str] = None, + ingress_config: Optional[IngressConfig] = None, ): self.k8s_client = k8s_client self.custom_api = k8s_client.get_custom_objects_api() @@ -60,6 +63,7 @@ def __init__( self.shutdown_policy = shutdown_policy self.service_account = service_account self.template_manager = AgentSandboxTemplateManager(template_file_path) + self.ingress_config = ingress_config def create_workload( self, @@ -405,7 +409,12 @@ def _pod_state_from_selector(self, workload: Dict[str, Any]) -> Optional[tuple[s return None - def get_endpoint_info(self, workload: Dict[str, Any], port: int) -> Optional[str]: + def get_endpoint_info(self, workload: Dict[str, Any], port: int, sandbox_id: str) -> Optional[str]: + # ingress-based endpoint if configured (gateway) + ingress_endpoint = format_ingress_endpoint(self.ingress_config, sandbox_id, port) + if ingress_endpoint: + return ingress_endpoint + status = workload.get("status", {}) selector = status.get("selector") namespace = workload.get("metadata", {}).get("namespace") @@ -426,3 +435,4 @@ def get_endpoint_info(self, workload: Dict[str, Any], port: int) -> Optional[str return f"{service_fqdn}:{port}" return None + diff --git a/server/src/services/k8s/batchsandbox_provider.py b/server/src/services/k8s/batchsandbox_provider.py index f1bfe0dc..4f57d67f 100644 --- a/server/src/services/k8s/batchsandbox_provider.py +++ b/server/src/services/k8s/batchsandbox_provider.py @@ -30,6 +30,8 @@ ) from src.api.schema import ImageSpec +from src.config import IngressConfig, INGRESS_MODE_GATEWAY +from src.services.helpers import format_ingress_endpoint from src.services.constants import SANDBOX_ID_LABEL from src.services.k8s.batchsandbox_template import BatchSandboxTemplateManager from src.services.k8s.client import K8sClient @@ -46,7 +48,12 @@ class BatchSandboxProvider(WorkloadProvider): and provides additional features like task management. """ - def __init__(self, k8s_client: K8sClient, template_file_path: Optional[str] = None): + def __init__( + self, + k8s_client: K8sClient, + template_file_path: Optional[str] = None, + ingress_config: Optional[IngressConfig] = None, + ): """ Initialize BatchSandbox provider. @@ -56,6 +63,7 @@ def __init__(self, k8s_client: K8sClient, template_file_path: Optional[str] = No """ self.k8s_client = k8s_client self.custom_api = k8s_client.get_custom_objects_api() + self.ingress_config = ingress_config # CRD constants self.group = "sandbox.opensandbox.io" @@ -668,30 +676,24 @@ def get_status(self, workload: Dict[str, Any]) -> Dict[str, Any]: "last_transition_at": creation_timestamp, } - def get_endpoint_info(self, workload: Dict[str, Any], port: int) -> Optional[str]: + def get_endpoint_info(self, workload: Dict[str, Any], port: int, sandbox_id: str) -> Optional[str]: """ Get endpoint information from BatchSandbox. - - Reads Pod IP from sandbox.opensandbox.io/endpoints annotation. - The annotation contains a JSON array of IP addresses. - - Args: - workload: BatchSandbox dict - port: Port number - - Returns: - Endpoint string in format "IP:PORT" or None if not available + - gateway mode: use ingress config to format endpoint + - direct/default: resolve Pod IP from annotation """ import json - - # Get annotations + + if self.ingress_config and self.ingress_config.mode == INGRESS_MODE_GATEWAY: + return format_ingress_endpoint(self.ingress_config, sandbox_id, port) + annotations = workload.get("metadata", {}).get("annotations", {}) # Get endpoints from annotation endpoints_str = annotations.get("sandbox.opensandbox.io/endpoints") if not endpoints_str: return None - + try: # Parse JSON array of IPs endpoints = json.loads(endpoints_str) @@ -701,5 +703,5 @@ def get_endpoint_info(self, workload: Dict[str, Any], port: int) -> Optional[str return f"{pod_ip}:{port}" except (json.JSONDecodeError, IndexError, TypeError): return None - + return None diff --git a/server/src/services/k8s/kubernetes_service.py b/server/src/services/k8s/kubernetes_service.py index b2c59483..40a374b1 100644 --- a/server/src/services/k8s/kubernetes_service.py +++ b/server/src/services/k8s/kubernetes_service.py @@ -82,6 +82,9 @@ def __init__(self, config: Optional[AppConfig] = None): if not self.app_config.kubernetes: raise ValueError("Kubernetes configuration is required") + # Ingress configuration (direct/gateway) if provided + self.ingress_config = self.app_config.ingress + self.namespace = self.app_config.kubernetes.namespace self.execd_image = runtime_config.execd_image self.service_account = self.app_config.kubernetes.service_account @@ -108,6 +111,7 @@ def __init__(self, config: Optional[AppConfig] = None): k8s_client=self.k8s_client, k8s_config=self.app_config.kubernetes, agent_sandbox_config=self.app_config.agent_sandbox, + ingress_config=self.ingress_config, ) logger.info( f"Initialized workload provider: {self.workload_provider.__class__.__name__}" @@ -621,7 +625,7 @@ def get_endpoint( }, ) - endpoint_str = self.workload_provider.get_endpoint_info(workload, port) + endpoint_str = self.workload_provider.get_endpoint_info(workload, port, sandbox_id) if not endpoint_str: raise HTTPException( diff --git a/server/src/services/k8s/provider_factory.py b/server/src/services/k8s/provider_factory.py index 751e59aa..4b2e7e2c 100644 --- a/server/src/services/k8s/provider_factory.py +++ b/server/src/services/k8s/provider_factory.py @@ -19,7 +19,7 @@ import logging from typing import Dict, Type, Optional -from src.config import KubernetesRuntimeConfig, AgentSandboxRuntimeConfig +from src.config import KubernetesRuntimeConfig, AgentSandboxRuntimeConfig, IngressConfig from src.services.k8s.workload_provider import WorkloadProvider from src.services.k8s.batchsandbox_provider import BatchSandboxProvider from src.services.k8s.agent_sandbox_provider import AgentSandboxProvider @@ -45,6 +45,7 @@ def create_workload_provider( k8s_client: K8sClient, k8s_config: Optional[KubernetesRuntimeConfig] = None, agent_sandbox_config: Optional[AgentSandboxRuntimeConfig] = None, + ingress_config: Optional[IngressConfig] = None, ) -> WorkloadProvider: """ Create a WorkloadProvider instance based on the provider type. @@ -85,11 +86,15 @@ def create_workload_provider( logger.info(f"Creating workload provider: {provider_class.__name__}") # Special handling for BatchSandboxProvider - pass template file path - if provider_type_lower == PROVIDER_TYPE_BATCHSANDBOX and k8s_config: - template_file = k8s_config.batchsandbox_template_file + if provider_type_lower == PROVIDER_TYPE_BATCHSANDBOX: + template_file = k8s_config.batchsandbox_template_file if k8s_config else None if template_file: logger.info(f"Using BatchSandbox template file: {template_file}") - return provider_class(k8s_client, template_file_path=template_file) + return provider_class( + k8s_client, + template_file_path=template_file, + ingress_config=ingress_config, + ) # Special handling for AgentSandboxProvider - pass agent-specific settings if provider_type_lower == PROVIDER_TYPE_AGENT_SANDBOX: @@ -99,8 +104,10 @@ def create_workload_provider( template_file_path=agent_config.template_file, shutdown_policy=agent_config.shutdown_policy, service_account=k8s_config.service_account if k8s_config else None, + ingress_config=ingress_config, ) + # Providers without ingress-specific needs return provider_class(k8s_client) diff --git a/server/src/services/k8s/workload_provider.py b/server/src/services/k8s/workload_provider.py index 146135bf..12ce245b 100644 --- a/server/src/services/k8s/workload_provider.py +++ b/server/src/services/k8s/workload_provider.py @@ -153,13 +153,14 @@ def get_status(self, workload: Any) -> Dict[str, Any]: pass @abstractmethod - def get_endpoint_info(self, workload: Any, port: int) -> Optional[str]: + def get_endpoint_info(self, workload: Any, port: int, sandbox_id: str) -> Optional[str]: """ Get endpoint information from workload. Args: workload: Workload object port: Port number + sandbox_id: Sandbox identifier for ingress-based endpoints Returns: Endpoint string (e.g., "10.244.0.5:8080") or None if not available diff --git a/server/tests/k8s/test_agent_sandbox_provider.py b/server/tests/k8s/test_agent_sandbox_provider.py index c0a3bf8d..5ca3f64c 100644 --- a/server/tests/k8s/test_agent_sandbox_provider.py +++ b/server/tests/k8s/test_agent_sandbox_provider.py @@ -229,7 +229,7 @@ def test_get_endpoint_info_prefers_running_pod(self, mock_k8s_client): "metadata": {"namespace": "test-ns"}, } - endpoint = provider.get_endpoint_info(workload, 8080) + endpoint = provider.get_endpoint_info(workload, 8080, "sandbox-123") assert endpoint == "10.0.0.9:8080" @@ -245,6 +245,6 @@ def test_get_endpoint_info_falls_back_to_service_fqdn(self, mock_k8s_client): "metadata": {"namespace": "test-ns"}, } - endpoint = provider.get_endpoint_info(workload, 9000) + endpoint = provider.get_endpoint_info(workload, 9000, "sandbox-123") assert endpoint == "svc.example.com:9000" diff --git a/server/tests/k8s/test_batchsandbox_provider.py b/server/tests/k8s/test_batchsandbox_provider.py index b51e6596..886a8afb 100644 --- a/server/tests/k8s/test_batchsandbox_provider.py +++ b/server/tests/k8s/test_batchsandbox_provider.py @@ -670,7 +670,7 @@ def test_get_endpoint_info_parses_json_annotation(self): } } - result = provider.get_endpoint_info(workload, 8080) + result = provider.get_endpoint_info(workload, 8080, "sandbox-123") assert result == "10.0.0.1:8080" @@ -687,7 +687,7 @@ def test_get_endpoint_info_uses_first_ip(self): } } - result = provider.get_endpoint_info(workload, 8080) + result = provider.get_endpoint_info(workload, 8080, "sandbox-123") assert result == "10.0.0.1:8080" @@ -698,7 +698,7 @@ def test_get_endpoint_info_returns_none_when_missing(self): provider = BatchSandboxProvider(MagicMock()) workload = {"metadata": {"annotations": {}}} - result = provider.get_endpoint_info(workload, 8080) + result = provider.get_endpoint_info(workload, 8080, "sandbox-123") assert result is None @@ -715,7 +715,7 @@ def test_get_endpoint_info_returns_none_on_invalid_json(self): } } - result = provider.get_endpoint_info(workload, 8080) + result = provider.get_endpoint_info(workload, 8080, "sandbox-123") assert result is None @@ -732,7 +732,7 @@ def test_get_endpoint_info_returns_none_on_empty_array(self): } } - result = provider.get_endpoint_info(workload, 8080) + result = provider.get_endpoint_info(workload, 8080, "sandbox-123") assert result is None diff --git a/server/tests/test_ingress.py b/server/tests/test_ingress.py new file mode 100644 index 00000000..cce96d23 --- /dev/null +++ b/server/tests/test_ingress.py @@ -0,0 +1,48 @@ + +from src.config import ( + GatewayConfig, + GatewayRouteModeConfig, + IngressConfig, + INGRESS_MODE_DIRECT, + INGRESS_MODE_GATEWAY, +) +from src.services.helpers import format_ingress_endpoint + + +def test_format_ingress_endpoint_returns_none_when_not_gateway(): + cfg = IngressConfig(mode=INGRESS_MODE_DIRECT) + assert format_ingress_endpoint(cfg, "sid", 8080) is None + assert format_ingress_endpoint(None, "sid", 8080) is None + + +def test_format_ingress_endpoint_wildcard(): + cfg = IngressConfig( + mode=INGRESS_MODE_GATEWAY, + gateway=GatewayConfig( + address="*.example.com", + route=GatewayRouteModeConfig(mode="wildcard"), + ), + ) + assert format_ingress_endpoint(cfg, "sid", 8080) == "sid-8080.example.com" + + +def test_format_ingress_endpoint_uri(): + cfg = IngressConfig( + mode=INGRESS_MODE_GATEWAY, + gateway=GatewayConfig( + address="gateway.example.com", + route=GatewayRouteModeConfig(mode="uri"), + ), + ) + assert format_ingress_endpoint(cfg, "sid", 9000) == "gateway.example.com/sid/9000" + + +def test_format_ingress_endpoint_header_returns_none(): + cfg = IngressConfig( + mode=INGRESS_MODE_GATEWAY, + gateway=GatewayConfig( + address="gateway.example.com", + route=GatewayRouteModeConfig(mode="header"), + ), + ) + assert format_ingress_endpoint(cfg, "sid", 9000) is None From 69d369d6cdb0469bc6e3717060dd491500e3ef27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Wed, 4 Feb 2026 14:09:14 +0800 Subject: [PATCH 4/5] chore(server): raise exception when header mode configured --- server/src/services/helpers.py | 2 +- server/tests/test_ingress.py | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/server/src/services/helpers.py b/server/src/services/helpers.py index cd9e55d0..67ba0e84 100644 --- a/server/src/services/helpers.py +++ b/server/src/services/helpers.py @@ -181,7 +181,7 @@ def format_ingress_endpoint( if route_mode == GATEWAY_ROUTE_MODE_HEADER: # TODO(Pangjiping): Header mode intentionally not emitted until Endpoint schema supports headers. - return None + raise RuntimeError(f"Unsupported route mode: {route_mode}") return None diff --git a/server/tests/test_ingress.py b/server/tests/test_ingress.py index cce96d23..95e7e083 100644 --- a/server/tests/test_ingress.py +++ b/server/tests/test_ingress.py @@ -1,3 +1,17 @@ +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from src.config import ( GatewayConfig, @@ -34,15 +48,4 @@ def test_format_ingress_endpoint_uri(): route=GatewayRouteModeConfig(mode="uri"), ), ) - assert format_ingress_endpoint(cfg, "sid", 9000) == "gateway.example.com/sid/9000" - - -def test_format_ingress_endpoint_header_returns_none(): - cfg = IngressConfig( - mode=INGRESS_MODE_GATEWAY, - gateway=GatewayConfig( - address="gateway.example.com", - route=GatewayRouteModeConfig(mode="header"), - ), - ) - assert format_ingress_endpoint(cfg, "sid", 9000) is None + assert format_ingress_endpoint(cfg, "sid", 9000) == "gateway.example.com/sid/9000" \ No newline at end of file From ce9780f070c4440d35845d539e73ddfb10b34f8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Mon, 9 Feb 2026 11:06:55 +0800 Subject: [PATCH 5/5] chore(server): remove unsupported header mode desc from readme --- server/README_zh.md | 1 - 1 file changed, 1 deletion(-) diff --git a/server/README_zh.md b/server/README_zh.md index 4138028a..d3395358 100644 --- a/server/README_zh.md +++ b/server/README_zh.md @@ -140,7 +140,6 @@ mode = "direct" # Docker 运行时仅支持 direct（直连，无 L7 网关） - `gateway.route.mode`：`wildcard`（域名泛匹配）、`uri`（基于路径前缀）；`header` 模式暂未支持。 - 返回示例： - `wildcard`：`-.example.com/path/to/request` - - `header`：`10.0.0.1:8000/path/to/request`，请求头 `OPEN-SANDBOX-INGRESS: -` - `uri`：`10.0.0.1:8000///path/to/request` ### Egress sidecar 配置与使用