LCORE-858: Hermetic build x86_64 + ARM platform

.tekton/lightspeed-stack-pull-request.yaml

@@ -30,6 +30,13 @@ spec:
   - name: build-platforms
     value:
     - linux/x86_64
+    - linux-c6gd2xlarge/arm64
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.x86_64.txt", "requirements.aarch64.txt", "requirements.hermetic.txt"]}]'
+  - name: hermetic
+    value: 'true'
   - name: dockerfile
     value: Containerfile
   pipelineSpec:
@@ -625,6 +632,9 @@ spec:
       optional: true
     - name: netrc
       optional: true
+  timeouts:
+    pipeline: 4h
+    tasks: 4h
   taskRunTemplate:
     serviceAccountName: build-pipeline-lightspeed-stack
   workspaces:

.tekton/lightspeed-stack-push.yaml

@@ -27,6 +27,13 @@ spec:
   - name: build-platforms
     value:
     - linux/x86_64
+    - linux-c6gd2xlarge/arm64
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true", "requirements_files": ["requirements.x86_64.txt", "requirements.aarch64.txt", "requirements.hermetic.txt"]}]'
+  - name: hermetic
+    value: 'true'
   - name: dockerfile
     value: Containerfile
   pipelineSpec:
@@ -622,6 +629,9 @@ spec:
       optional: true
     - name: netrc
       optional: true
+  timeouts:
+    pipeline: 4h
+    tasks: 4h
   taskRunTemplate:
     serviceAccountName: build-pipeline-lightspeed-stack
   workspaces:

Containerfile

@@ -22,10 +22,18 @@ RUN pip3.12 install "uv==0.8.15"
 # Add explicit files and directories
 # (avoid accidental inclusion of local directories or env files or credentials)
 COPY ${LSC_SOURCE_DIR}/src ./src
-COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ./
+COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ${LSC_SOURCE_DIR}/requirements.*.txt ./
 
 # Bundle additional dependencies for library mode.
-RUN uv sync --locked --no-dev --group llslibdev
+# Source cachi2 environment for hermetic builds if available, otherwise use normal installation
+# cachi2.env has these env vars:
+# PIP_FIND_LINKS=/cachi2/output/deps/pip
+# PIP_NO_INDEX=true
+RUN if [ -f /cachi2/cachi2.env ]; then \
+    . /cachi2/cachi2.env && uv venv --seed --no-index --find-links ${PIP_FIND_LINKS} && . .venv/bin/activate && pip install --no-index --find-links ${PIP_FIND_LINKS} -r requirements.$(uname -m).txt; \
+    else \
+    uv sync --locked --no-dev --group llslibdev; \
+    fi
 
 # Explicitly remove some packages to mitigate some CVEs
 # - GHSA-wj6h-64fc-37mp: python-ecdsa package won't fix it upstream.

pyproject.toml

@@ -87,8 +87,23 @@ Issues = "https://github.com/lightspeed-core/lightspeed-stack/issues"
 name = "pytorch-cpu"
 url = "https://download.pytorch.org/whl/cpu"
 explicit = true
+
+[[tool.uv.index]]
+name = "pypi-default"
+url = "https://pypi.org/simple"
+explicit = true
 [tool.uv.sources]
-torch = [{ index = "pytorch-cpu" }]
+torch = [
+    { index = "pytorch-cpu", group = "llslibdev" },
+    { index = "pypi-default", group = "llslibdev-hermetic" }
+]
+[tool.uv]
+conflicts = [
+    [
+        { group = "llslibdev" },
+        { group = "llslibdev-hermetic" },
+    ],
+]
 
 [dependency-groups]
 dev = [
@@ -159,6 +174,50 @@ llslibdev = [
     "blobfile>=3.0.0",
     "psutil>=7.0.0",
 ]
+llslibdev-hermetic = [
+   # the same as llslibdev, just using default index.
+   "matplotlib>=3.10.0",
+   "pillow>=11.1.0",
+   "pandas>=2.2.3",
+   "scikit-learn>=1.5.2",
+   "psycopg2-binary>=2.9.10",
+   # API eval: inline::meta-reference
+   "tree_sitter>=0.24.0",
+   "pythainlp>=3.0.10",
+   "langdetect>=1.0.9",
+   "emoji>=2.1.0",
+   "nltk>=3.8.1",
+   # API inference: remote::gemini
+   "litellm>=1.75.5.post1",
+   # API inference: inline::sentence-transformers
+   "sentence-transformers>=5.0.0",
+   # API vector_io: inline::faiss
+   "faiss-cpu>=1.11.0",
+   # API scoring: inline::basic
+   "requests>=2.32.4",
+   # API datasetio: inline::localfs
+   "aiosqlite>=0.21.0",
+   # API datasetio: remote::huggingface
+   "datasets>=3.6.0",
+   # API telemetry: inline::meta-reference
+   "opentelemetry-sdk>=1.34.1",
+   "opentelemetry-exporter-otlp>=1.34.1",
+   # API tool_runtime: inline::rag-runtime
+   "transformers>=4.34.0",
+   "numpy==2.2.6",
+   # API tool_runtime: remote::model-context-protocol
+   "mcp>=1.9.4",
+   # API post_training: inline::huggingface
+   "torch==2.7.1; sys_platform == 'linux'",
+   "trl>=0.18.2",
+   "peft>=0.15.2",
+   # Other
+   "autoevals>=0.0.129",
+   "fire>=0.7.0",
+   "opentelemetry-instrumentation>=0.55b0",
+   "blobfile>=3.0.0",
+   "psutil>=7.0.0",
+]
 
 build = [
     "build>=1.2.2.post1",