Feat/refine template

.github/workflows/validate-template.yaml

@@ -0,0 +1,52 @@
+name: Validation of template
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'pyproject.toml'
+      - 'pixi.lock'
+      - 'projects/**'
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches: [ main ]
+    paths:
+      - 'pyproject.toml'
+      - 'pixi.lock'
+      - 'projects/**'
+  workflow_dispatch:
+
+concurrency:
+  group: tutorial-main-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validation:
+    name: Validate tutorial
+    runs-on: ubuntu-latest
+    env:
+      # dagster dbt needs to run in prepare_if_dev to parse the project
+      DAGSTER_IS_DEV_CLI: "True"
+      CATALOG_DB_USER: "dummy"
+      CATALOG_DB_PASSWORD: "dummy"
+      OBJECT_STORE_ROOT_USER: "dummy"
+      OBJECT_STORE_ROOT_PASSWORD: "dummy"
+      OBJECT_STORE_REGION: "dummy"
+      OPENAI_API_KEY: "dummy"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - uses: prefix-dev/setup-pixi@v0.9.0
+        with:
+          pixi-version: v0.55.0
+          cache: true
+          frozen: true
+
+     # Create an instance of the template
+      - run: pixi run render-dev
+
+     # check the instance can start
+      - run: cd rendered-template/local_data_stack/ && pixi run -e dev test
+
+    # possibly add further testing later

.gitignore

@@ -1,3 +1,4 @@
 .pixi/envs/
 .pixi/solve-group-envs/
 local_data_stack
+key.txt

README.md

@@ -30,6 +30,8 @@ Additionally included for secure handling of secrets:
 - sops
 - age
 
+![](img/ducking-the-lake.png)
+
 ## usage
 
 Prerequisites: 
@@ -74,4 +76,11 @@ pixi run start-dev
 Post install:
 
 - update the secrets in the `.env` files by executing: `openssl rand -base64 32` and setting a suitable secret
-- ensure the `.env.enc` can be created by following the instructions in [documentation/secops]
+- ensure the `.env.enc` can be created by following the instructions in [documentation/secops]
+
+## Developing the template
+
+To render an instance of the project:
+```bash
+pixi run render-dev
+```

pyproject.toml

@@ -37,3 +37,10 @@ env = { RUST_LOG = "warn" }
 cmd = "pixi run -e template cruft create https://github.com/l-mds/local-data-stack.git"
 description = "Initialize template with cruft"
 
+[tool.pixi.tasks.render-dev]
+cmd = "pixi run --frozen -e template cruft create . --no-input --overwrite-if-exists --output-dir rendered-template && ./scripts/handle-dev-secrets.sh"
+description = "locally render a development instance of the template"
+
+[tool.pixi.tasks.render-dev-no-secrets]
+cmd = "pixi run --frozen -e template cruft create . --no-input --overwrite-if-exists --output-dir rendered-template"
+description = "locally render a development instance of the template; no secrets needed (for CI)"

scripts/handle-dev-secrets.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cp key.txt rendered-template/local_data_stack/
+cd rendered-template/local_data_stack
+make secrets-decrypt
+cd ../../

{{ cookiecutter.project_slug }}/Dockerfile

@@ -1,7 +1,5 @@
-# syntax=docker/dockerfile:1.10.0
-
 # for CUDA support ensure docker is nvidia-docker enabled!
-FROM ghcr.io/prefix-dev/pixi:0.41.1-bookworm AS builder_base
+FROM ghcr.io/prefix-dev/pixi:0.55.0-bookworm AS builder_base
 
 FROM builder_base AS build
 ENV DAGSTER_HOME=/opt/dagster/dagster_home/

{{ cookiecutter.project_slug }}/Makefile

@@ -35,13 +35,13 @@ fmt-unsafe:
 .PHONY: lint
 ## Ruff based flake8 style linting plus type checking via pyright
 lint:
-	pixi run -e ci-validation lint
+	pixi run -e dev lint
 
 
 .PHONY: test
 ## Execute tests with coverage
 test:
-	pixi run -e ci-validation test
+	pixi run -e dev test
 
 
 

{{ cookiecutter.project_slug }}/README.md

@@ -12,7 +12,7 @@ pixi update
 
 
 # launch a shell
-pixi shell --frozen -e ci-validation
+pixi shell --frozen -e dev
 cd src/code_location_{{ cookiecutter.project_slug }}/code_location_{{ cookiecutter.project_slug }}_dbt/ && dbt deps
 cd ../../../
 ```
@@ -23,11 +23,11 @@ set `DAGSTER_HOME` to have persistent logs
 
 ```bash
 # for a single code location
-pixi shell --frozen -e ci-validation
+pixi shell --frozen -e dev
 
 dagster dev
 # alternatively:
-pixi run -e ci-validation start-dev
+pixi run -e dev start-dev
 
 
 dagster job list --location foo

{{ cookiecutter.project_slug }}/prototyping/proxy/services/proxy/rules/middlewares.yml

@@ -10,7 +10,7 @@ http:
     middlewares-rate-limit:
       rateLimit:
         average: 100
-        burst: 50
+        burst: 500
 
     middlewares-https-redirectscheme:
       redirectScheme:

{{ cookiecutter.project_slug }}/pyproject.toml

@@ -3,7 +3,7 @@ name = "{{ cookiecutter.project_slug }}"
 version = "1.0.0"
 description = "{{ cookiecutter.project_name }}"
 authors = [{ name = "{{ cookiecutter.author }}", email = "{{ cookiecutter.author_email }}" }]
-requires-python = "== 3.12"
+requires-python = ">=3.12,<3.13"
 
 [tool.pixi.project]
 platforms = ["linux-64", "osx-arm64", "win-64", "linux-aarch64"]
@@ -28,18 +28,7 @@ codelocation-{{ cookiecutter.project_slug_pixi }} = { features = [
   "dagster-basics",
   "shared",
 ], solve-group = "codelocation-{{ cookiecutter.project_slug_pixi }}" }
-ci-validation = { features = [
-  "ci-basics",
-  "codelocation-{{ cookiecutter.project_slug_pixi }}",
-  "codelocation-foo",
-  "shared-library",
-  "shared",
-  "dagster-webserver",
-  "notebookdev",
-  "dagster-daemon",
-  "dagster-basics",
 
-], solve-group = "default" }
 dev = { features = [
   "ci-basics",
   "codelocation-{{ cookiecutter.project_slug_pixi }}",
@@ -50,7 +39,7 @@ dev = { features = [
   "dagster-webserver",
   "dagster-daemon",
   "dagster-basics",
-
+  "dagster-scaffolding",
 ], solve-group = "default" }
 dagster-webserver = { features = [
   "dagster-webserver",
@@ -61,6 +50,14 @@ dagster-daemon = { features = [
   "dagster-basics",
 ], solve-group = "default" }
 
+[tool.pixi.feature.dagster-scaffolding.dependencies]
+yarn = "~=4.9.2"
+
+[tool.pixi.feature.dagster-scaffolding.pypi-dependencies]
+dagster-dg-cli = ">=1.11.10,<1.12"
+create-dagster = ">=1.11.10,<1.12"
+
+
 [tool.pixi.feature.ci-basics.dependencies]
 yamllint = ">=1.35.1,<2"
 taplo = ">=0.9.3,<0.10"
@@ -85,16 +82,16 @@ ipywidgets = "~=8.1.5"
 [tool.pixi.feature.dagster-basics.dependencies]
 
 [tool.pixi.feature.dagster-basics.pypi-dependencies]
-dagster = ">=1.9.11,<1.10"
-dagster-postgres = ">=0.25.11,<0.26"
-dagster-cloud = ">=1.9.11,<1.10"
+dagster = ">=1.11.11,<1.12"
+dagster-postgres = ">=0.27.11,<0.28"
+dagster-cloud = ">=1.11.11,<1.12"
 
 [tool.pixi.feature.dagster-daemon.pypi-dependencies]
-dagster-docker = ">=0.25.11,<0.26"
+dagster-docker = ">=0.27.11,<0.28"
 
 [tool.pixi.feature.dagster-webserver.pypi-dependencies]
-dagster-webserver = ">=1.9.11,<1.10"
-dagster-docker = ">=0.25.11,<0.26"
+dagster-webserver = ">=1.11.11,<1.12"
+dagster-docker = ">=0.27.11,<0.28"
 
 [tool.pixi.feature.shared-library.dependencies]
 [tool.pixi.feature.shared-library.pypi-dependencies]
@@ -117,10 +114,10 @@ quickstart_etl = { path = "./src/code_location_foo", editable = true }
 [tool.pixi.feature.codelocation-{{ cookiecutter.project_slug_pixi }}.pypi-dependencies]
 dbt-core = "~=1.9.2"
 dbt-duckdb = "~=1.9.1"
-dagster-dbt = ">=0.25.11,<0.26"
-dagster-duckdb = ">=0.25.11,<0.26"
-dagster-duckdb-polars = ">=0.25.11,<0.26"
-dagster-duckdb-pandas = ">=0.25.11,<0.26"
+dagster-dbt = ">=0.27.11,<0.28"
+dagster-duckdb = ">=0.27.11,<0.28"
+dagster-duckdb-polars = ">=0.27.11,<0.28"
+dagster-duckdb-pandas = ">=0.27.11,<0.28"
 code_location_{{ cookiecutter.project_slug }} = { path = "./src/code_location_{{ cookiecutter.project_slug }}", editable = true }
 
 [tool.pixi.feature.template.dependencies]

{{ cookiecutter.project_slug }}/scripts/secops_config.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
 
 # Define files to encrypt/decrypt
-FILES_TO_ENCRYPT=".env prototyping/proxy/.env"
+FILES_TO_ENCRYPT="\
+.env
+prototyping/proxy/.env
+"

{{ cookiecutter.project_slug }}/src/code_location_foo/pyproject.toml

@@ -3,5 +3,12 @@ requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [tool.dagster]
-module_name = "quickstart_etl.definitions"
+module_name = "quickstart_etl"
 code_location_name = "quickstart_etl"
+
+[tool.dg]
+directory_type = "project"
+
+[tool.dg.project]
+root_module = "quickstart_etl"
+registry_modules = ["quickstart_etl.components.*"]

{{ cookiecutter.project_slug }}/src/code_location_foo/quickstart_etl/__init__.py

@@ -1,52 +1,4 @@
-from pathlib import Path
+# ruff: noqa: F401
+from .definitions import defs
 
-from dagster import (
-    Definitions,
-    ScheduleDefinition,
-    define_asset_job,
-    graph_asset,
-    link_code_references_to_git,
-    load_assets_from_package_module,
-    op,
-    with_source_code_references,
-)
-from dagster._core.definitions.metadata.source_code import AnchorBasedFilePathMapping
-
-from . import assets
-
-daily_refresh_schedule = ScheduleDefinition(
-    job=define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *"
-)
-
-
-@op
-def foo_op():
-    return 5
-
-
-@graph_asset
-def my_asset():
-    return foo_op()
-
-
-my_assets = with_source_code_references(
-    [
-        my_asset,
-        *load_assets_from_package_module(assets),
-    ]
-)
-
-my_assets = link_code_references_to_git(
-    assets_defs=my_assets,
-    git_url="https://github.com/{{ cookiecutter.organization }}/{{ cookiecutter.project_slug }}/",
-    git_branch="main",
-    file_path_mapping=AnchorBasedFilePathMapping(
-        local_file_anchor=Path(__file__).parent,
-        file_anchor_path_in_repository="src/code_location_foo/quickstart_etl",
-    ),
-)
-
-defs = Definitions(
-    assets=my_assets,
-    schedules=[daily_refresh_schedule],
-)
+__all__ = ["defs"]

{{ cookiecutter.project_slug }}/src/code_location_foo/quickstart_etl/definitions.py

@@ -0,0 +1,42 @@
+# ruff: noqa: E402
+import warnings
+
+import dagster as dg
+from dagster._utils import warnings as dagster_warnings
+
+warnings.filterwarnings("ignore", category=dagster_warnings.BetaWarning)
+warnings.filterwarnings("ignore", category=dagster_warnings.PreviewWarning)
+from pathlib import Path
+
+@dg.definitions
+def defs():
+    root_dir = __file__
+    base = dg.load_from_defs_folder(path_within_project=Path(root_dir).parent.parent)
+
+    materializable = [a for a in base.assets if isinstance(a, dg.AssetsDefinition)]  # type: ignore
+    passthrough = [
+        a
+        for a in base.assets  # type: ignore
+        if not isinstance(a, dg.AssetsDefinition)
+    ]  # SourceAsset
+
+    with_refs = dg.with_source_code_references(materializable)
+
+    linked = dg.link_code_references_to_git(
+        assets_defs=with_refs,
+        git_url="https://github.com/{{ cookiecutter.organization }}/{{ cookiecutter.project_slug }}/",
+        git_branch="main",
+        file_path_mapping=dg.AnchorBasedFilePathMapping(
+            local_file_anchor=Path(root_dir).parent,
+            file_anchor_path_in_repository="src/code_location_foo/quickstart_etl"
+        ),
+    )
+
+    return dg.Definitions(
+        assets=[*linked, *passthrough],
+        jobs=base.jobs,
+        schedules=base.schedules,
+        sensors=base.sensors,
+        resources=base.resources,
+        asset_checks=base.asset_checks,
+    )

{{ cookiecutter.project_slug }}/src/code_location_foo/quickstart_etl/defs/hn/component.py

@@ -0,0 +1,26 @@
+import dagster as dg
+
+daily_refresh_schedule = dg.ScheduleDefinition(
+    job=dg.define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *"
+)
+
+
+@dg.op
+def foo_op():
+    return 5
+
+
+@dg.graph_asset(automation_condition=dg.AutomationCondition.eager())
+def my_asset():
+    return foo_op()
+
+
+
+class HnComponent(dg.Component, dg.Model, dg.Resolvable):
+    def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
+        return dg.Definitions()
+
+
+@dg.component_instance
+def load(context: dg.ComponentLoadContext) -> HnComponent:
+    return HnComponent()