From 93267e9bd81505d737b191eac25304483484e87d Mon Sep 17 00:00:00 2001 From: ruby Date: Fri, 13 Feb 2026 13:32:44 +0400 Subject: [PATCH 1/2] feat(config,migrations): replace env.cluster_name with migrations.table; add replicated cluster override --- README.md | 27 +++-- docs/configuration/env-and-vars.mdx | 18 +-- docs/configuration/examples.mdx | 9 +- docs/configuration/overview.mdx | 23 +++- docs/migrations/faq.mdx | 105 ++++++++++++++++-- docs/migrations/templates.mdx | 9 +- example/clisma.hcl | 14 ++- package-lock.json | 22 +--- packages/cli/package.json | 4 +- packages/cli/src/cli.ts | 12 +- packages/cli/tsconfig.json | 10 +- packages/config/package.json | 2 +- packages/config/src/config.ts | 50 +++++---- packages/config/src/hcl-schema.ts | 44 +++++--- .../config/src/tests/parse-config.test.ts | 77 +++++++++++++ packages/core/package.json | 4 +- packages/core/src/migrations/repository.ts | 95 ++++++++++------ packages/core/src/migrations/runner.ts | 20 ++-- packages/core/src/migrations/types.ts | 1 + 19 files changed, 400 insertions(+), 146 deletions(-) diff --git a/README.md b/README.md index 476763c..63b3503 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,22 @@ variable "ttl_days" { env "production" { url = env("CLICKHOUSE_PROD_URL") - cluster_name = "prod-cluster" migrations { dir = "migrations" + + table { + name = "schema_migrations" + + is_replicated = true + + # Optional: force a specific cluster for ON CLUSTER. + cluster_name = "prod-cluster" + + # If replication_path is set, is_replicated can be omitted. + replication_path = "/clickhouse/tables/cluster-{cluster}/shard-{shard}/{database}/schema_migrations" + } + vars = { is_replicated = true create_table_options = "ON CLUSTER prod-cluster" @@ -81,9 +93,12 @@ env "production" { } ``` -**`cluster_name`** affects how the migrations tracking table is created (replicated or not). And the CLI will warn if the actual cluster does not match the config. +**`migrations.table`** controls the tracking table: -#### If your ClickHouse server has clusters configured, `cluster_name` is required +- `name` sets a custom table name. +- `is_replicated = true` enables replicated tracking. +- `cluster_name` optionally selects cluster for `ON CLUSTER`. +- `replication_path` overrides the default replication path (and also enables replicated mode if `is_replicated` is omitted). ### TLS certificates (custom CA and mTLS) @@ -91,7 +106,7 @@ If your ClickHouse endpoint uses a self-signed certificate, add a `tls` block so ```hcl env "production" { - url = env("CLICKHOUSE_URL") # e.g. https://user:pass@host:8443/db?secure=true + url = env("CLICKHOUSE_URL") # e.g. https://user:pass@host:8443/db tls { ca_file = env("CLICKHOUSE_CA_FILE") @@ -108,12 +123,10 @@ Notes: - `ca_file` is required when `tls` is set. - `cert_file` and `key_file` must be provided together. - Relative paths are resolved from the directory where `clisma.hcl` lives. -- URL query params (like `?secure=true`) are preserved. ## 🧪 Templates -Templates are [Handlebars](https://handlebarsjs.com/guide/expressions.html). Variables come from `migrations.vars` (and -`cluster_name` is available as `{{cluster_name}}`). +Templates are [Handlebars](https://handlebarsjs.com/guide/expressions.html). Variables come from `migrations.vars`. ```sql CREATE TABLE IF NOT EXISTS events {{create_table_options}} ( diff --git a/docs/configuration/env-and-vars.mdx b/docs/configuration/env-and-vars.mdx index 5d09417..a7a7792 100644 --- a/docs/configuration/env-and-vars.mdx +++ b/docs/configuration/env-and-vars.mdx @@ -38,20 +38,4 @@ env "local" { } } } -``` - -## Cluster name - -If you want the migrations table to be replicated and use cluster-aware templates, set -`cluster_name` in the environment: - -```hcl -env "production" { - url = env("CLICKHOUSE_PROD_URL") - cluster_name = "prod-cluster" - - migrations { - dir = "migrations" - } -} -``` +``` \ No newline at end of file diff --git a/docs/configuration/examples.mdx b/docs/configuration/examples.mdx index d92c015..269c622 100644 --- a/docs/configuration/examples.mdx +++ b/docs/configuration/examples.mdx @@ -24,6 +24,7 @@ env "staging" { migrations { dir = "migrations" + vars = { replication_factor = 2 ttl_days = 30 @@ -37,7 +38,6 @@ env "staging" { ```hcl env "production" { url = env("CLICKHOUSE_PROD_URL") - cluster_name = "prod-cluster" exclude = ["system.*", "_tmp_*"] tls { @@ -48,7 +48,12 @@ env "production" { migrations { dir = "migrations" - table_name = "schema_migrations" + + table { + name = "schema_migrations" + is_replicated = true + } + vars = { replication_factor = var.replication_factor ttl_days = 90 diff --git a/docs/configuration/overview.mdx b/docs/configuration/overview.mdx index 6e96ad4..79ca7df 100644 --- a/docs/configuration/overview.mdx +++ b/docs/configuration/overview.mdx @@ -29,7 +29,6 @@ env "local" { env "production" { url = "http://default:password@localhost:8123/mydb" - cluster_name = "dwh" tls { ca_file = "certs/ca.pem" @@ -37,8 +36,18 @@ env "production" { migrations { dir = "migrations" - table_name = "schema_migrations" - replication_path = "/clickhouse/tables/{cluster}/schema_migrations" + + table { + name = "schema_migrations" + + is_replicated = true + + # Optional: force a specific cluster for ON CLUSTER. + cluster_name = "dwh" + + # If replication_path is set, is_replicated can be omitted. + replication_path = "/clickhouse/tables/{cluster}/schema_migrations" + } vars = { ttl_days = 30 @@ -50,7 +59,6 @@ env "production" { Fields: - `url` (string) Connection string. Supports `env("NAME")`. -- `cluster_name` (string, optional) Cluster name used for template context and replicated tracking. Required when your server has clusters configured. - `exclude` (list\, optional) Patterns to ignore. - `migrations` (block) Migration settings. - `tls` (block, optional) TLS certificate settings for custom CA and mTLS: @@ -61,8 +69,11 @@ Fields: ## migrations - `dir` (string) Path to migrations directory. -- `table_name` (string, optional) Custom tracking table. -- `replication_path` (string, optional) Replication path for the tracking table. +- `table` (block, optional) Tracking table settings. + - `table.name` (string, optional) Custom tracking table. + - `table.is_replicated` (bool, optional, default `false`) Use replicated tracking table with `ON CLUSTER`. + - `table.cluster_name` (string, optional) Cluster name override for `ON CLUSTER`. Useful when multiple clusters exist. + - `table.replication_path` (string, optional) Replication path for the tracking table. If set, replicated mode is enabled even when `table.is_replicated` is omitted. - `vars` (object, optional) Variables for Handlebars templates. ## variable "name" diff --git a/docs/migrations/faq.mdx b/docs/migrations/faq.mdx index f25382d..dc8b412 100644 --- a/docs/migrations/faq.mdx +++ b/docs/migrations/faq.mdx @@ -14,7 +14,10 @@ env "local" { migrations { dir = "migrations" - table_name = "custom_schema_migrations" + + table { + name = "custom_schema_migrations" + } } } ``` @@ -26,12 +29,21 @@ In clustered setups you can override the migrations table replication path used ```hcl env "production" { url = env("CLICKHOUSE_PROD_URL") - cluster_name = "prod-cluster" migrations { dir = "migrations" - table_name = "custom_schema_migrations" - replication_path = "/clickhouse/foo/bar/custom_schema_migrations" + + table { + name = "custom_schema_migrations" + + is_replicated = true + + # Optional: force a specific cluster for ON CLUSTER. + cluster_name = "prod-cluster" + + # If replication_path is set, is_replicated can be omitted. + replication_path = "/clickhouse/foo/bar/custom_schema_migrations" + } } } ``` @@ -39,14 +51,91 @@ env "production" { Default replication path: ```text -/clickhouse/tables/cluster-{cluster}/shard-{shard}/{database}/${migrations.table_name} +/clickhouse/tables/cluster-{cluster}/shard-{shard}/{database}/${migrations.table.name} ``` ### Summary -If `env.cluster_name` is set, clisma treats the environment as clustered and creates the migrations table using `ReplicatedReplacingMergeTree(...)` with `ON CLUSTER ""`. +If `migrations.table.is_replicated` is `true`, clisma treats the environment as clustered and creates the migrations table using `ReplicatedReplacingMergeTree(...)` with `ON CLUSTER ""`. + +If `migrations.table.is_replicated` is not set (or `false`), it uses `ReplacingMergeTree()` in standalone mode. + +If `migrations.table.replication_path` is set, clisma also enables replicated mode even when `migrations.table.is_replicated` is omitted. + +If your ClickHouse has multiple non-default clusters, set `migrations.table.cluster_name` to choose which cluster should be used in `ON CLUSTER`. + +## How config affects migrations table DDL + +### 1) Standalone (default) + +Config: + +```hcl +migrations { + dir = "migrations" +} +``` + +DDL shape: + +```sql +CREATE TABLE IF NOT EXISTS schema_migrations ( + ... +) ENGINE = ReplacingMergeTree() +ORDER BY version; +``` + +### 2) Replicated (auto cluster detection) -If `cluster_name` is not set in config and ClickHouse reports configured clusters, clisma stops and asks you to set `cluster_name`. If there are no clusters, it uses `ReplacingMergeTree()` in standalone mode. +Config: + +```hcl +migrations { + dir = "migrations" + table { + is_replicated = true + } +} +``` + +DDL shape: + +```sql +CREATE TABLE IF NOT EXISTS schema_migrations ON CLUSTER "" ( + ... +) ENGINE = ReplicatedReplacingMergeTree( + '/clickhouse/tables/cluster-{cluster}/shard-{shard}/{database}/schema_migrations', + '{replica}' +) +ORDER BY version; +``` + +### 3) Replicated with explicit cluster and path + +Config: + +```hcl +migrations { + dir = "migrations" + table { + name = "custom_schema_migrations" + cluster_name = "prod-cluster" + replication_path = "/clickhouse/foo/bar/custom_schema_migrations" + } +} +``` + +DDL shape: + +```sql +CREATE TABLE IF NOT EXISTS custom_schema_migrations ON CLUSTER "prod-cluster" ( + ... +) ENGINE = ReplicatedReplacingMergeTree( + '/clickhouse/foo/bar/custom_schema_migrations', + '{replica}' +) +ORDER BY version; +``` ## Does clisma support down migrations? @@ -61,4 +150,4 @@ Yes. You can include multiple SQL statements in one file. Separate statements wi If a migration file changes after being applied, clisma will fail with a checksum mismatch. You have two options: - Revert the migration file back to the applied version and create a new forward migration. -- If you really need to override the checksum, update the stored checksum in the migrations table (use `clisma checksum `). This is risky and should only be done if you fully understand the consequences. \ No newline at end of file +- If you really need to override the checksum, update the stored checksum in the migrations table (use `clisma checksum `). This is risky and should only be done if you fully understand the consequences. diff --git a/docs/migrations/templates.mdx b/docs/migrations/templates.mdx index d6bfe93..018ca5e 100644 --- a/docs/migrations/templates.mdx +++ b/docs/migrations/templates.mdx @@ -5,7 +5,7 @@ icon: "notes-medical" --- Template variables come from `migrations.vars` in your config. If you set -`cluster_name` on the environment, it is also available as `{{cluster_name}}`. +`migrations.table.cluster_name`, it is also available as `{{cluster_name}}`. > If you have used Atlas, this will feel familiar. The idea is the same as > [Atlas templated migrations](https://atlasgo.io/concepts/migrations#template), @@ -31,10 +31,15 @@ env "local" { env "production" { url = env("CLICKHOUSE_PROD_URL") - cluster_name = "prod-cluster" migrations { dir = "migrations" + + table { + is_replicated = true + cluster_name = "prod-cluster" + } + vars = { is_replicated = true create_table_options = "ON CLUSTER prod-cluster" diff --git a/example/clisma.hcl b/example/clisma.hcl index afa9acb..55322d4 100644 --- a/example/clisma.hcl +++ b/example/clisma.hcl @@ -8,7 +8,6 @@ env "local" { env "production" { url = "http://default:password@localhost:8123/default" - cluster_name = "prod-cluster" tls { ca_file = "certs/ca.pem" @@ -18,6 +17,19 @@ env "production" { migrations { dir = "migrations" + + table { + name = "schema_migrations" + + is_replicated = true + + # Optional: force a specific cluster for ON CLUSTER. + cluster_name = "prod-cluster" + + # If replication_path is set, is_replicated can be omitted. + replication_path = "/clickhouse/tables/cluster-{cluster}/shard-{shard}/{database}/schema_migrations" + } + vars = { is_replicated = true create_table_options = "ON CLUSTER prod-cluster" diff --git a/package-lock.json b/package-lock.json index b19a8f7..8a6c115 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1662,10 +1662,10 @@ }, "packages/cli": { "name": "clisma", - "version": "0.2.1", + "version": "0.3.0", "license": "MIT", "dependencies": { - "@clisma/core": "0.2.0", + "@clisma/core": "0.3.0", "@dotenvx/dotenvx": "^1.51.4", "@inquirer/prompts": "^8.2.0", "commander": "^12.1.0", @@ -1679,21 +1679,9 @@ "oxlint": "^0.12.0" } }, - "packages/cli/node_modules/@clisma/core": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@clisma/core/-/core-0.2.0.tgz", - "integrity": "sha512-g0JXKYgez6z2OS7c1uC0GlUieXBG4Ts3UbcQoFhxHH+x/902fp3YWSR+zx7s4njTgvnmja6pZwt+mMr3Q0gO0w==", - "dependencies": { - "@clickhouse/client": "^1.11.2", - "@clisma/config": "*", - "handlebars": "^4.7.8", - "kleur": "^4.1.5", - "ora": "^9.1.0" - } - }, "packages/config": { "name": "@clisma/config", - "version": "0.2.1", + "version": "0.3.0", "dependencies": { "@cdktf/hcl2json": "^0.21.0" }, @@ -1704,10 +1692,10 @@ }, "packages/core": { "name": "@clisma/core", - "version": "0.2.1", + "version": "0.3.0", "dependencies": { "@clickhouse/client": "^1.11.2", - "@clisma/config": "0.2.1", + "@clisma/config": "0.3.0", "handlebars": "^4.7.8", "kleur": "^4.1.5", "ora": "^9.1.0" diff --git a/packages/cli/package.json b/packages/cli/package.json index 9bc206b..5a2a432 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "clisma", - "version": "0.2.1", + "version": "0.3.0", "description": "ClickHouse migration CLI", "author": "github.com/StopMakingThatBigFace", "license": "MIT", @@ -33,7 +33,7 @@ "access": "public" }, "dependencies": { - "@clisma/core": "0.2.0", + "@clisma/core": "0.3.0", "@dotenvx/dotenvx": "^1.51.4", "@inquirer/prompts": "^8.2.0", "commander": "^12.1.0", diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index f649084..74890aa 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -105,6 +105,7 @@ const runCommand = async ( let migrationsDir: string; let connectionString: string; let tableName: string | undefined; + let isReplicated = false; let clusterName: string | undefined; let replicationPath: string | undefined; let tls: MigrationRunnerTLSOptions | undefined; @@ -142,10 +143,10 @@ const runCommand = async ( envConfig.migrations.dir, ); - // Extract table_name, cluster_name, and replication_path from config - tableName = envConfig.migrations.table_name; - clusterName = envConfig.cluster_name; - replicationPath = envConfig.migrations.replication_path; + tableName = envConfig.migrations.table?.name; + isReplicated = envConfig.migrations.table?.is_replicated || false; + clusterName = envConfig.migrations.table?.cluster_name; + replicationPath = envConfig.migrations.table?.replication_path; if (envConfig.tls) { const caPath = resolveFromConfigPath(configPath, envConfig.tls.ca_file); @@ -166,7 +167,7 @@ const runCommand = async ( templateVars = { ...(envConfig.migrations.vars || {}), - ...(envConfig.cluster_name ? { cluster_name: envConfig.cluster_name } : {}), + ...(clusterName ? { cluster_name: clusterName } : {}), }; console.log( @@ -180,6 +181,7 @@ const runCommand = async ( migrationsDir, connectionString, tableName, + isReplicated, clusterName, replicationPath, tls, diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index c9c8001..40580cb 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -3,7 +3,15 @@ "compilerOptions": { "composite": true, "outDir": "dist", - "rootDir": "src" + "rootDir": "src", + "paths": { + "@clisma/core": [ + "../core/src/index.ts" + ], + "@clisma/config": [ + "../config/src/index.ts" + ] + } }, "references": [ { "path": "../config" }, diff --git a/packages/config/package.json b/packages/config/package.json index 9680545..08771d5 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@clisma/config", - "version": "0.2.1", + "version": "0.3.0", "description": "Configuration parsing and schema generation for clisma", "type": "module", "main": "dist/config.js", diff --git a/packages/config/src/config.ts b/packages/config/src/config.ts index c5869d5..6f3d5d1 100644 --- a/packages/config/src/config.ts +++ b/packages/config/src/config.ts @@ -10,8 +10,12 @@ import { export type MigrationConfig = { dir: string; - table_name?: string; - replication_path?: string; + table?: { + name: string; + is_replicated: boolean; + cluster_name?: string; + replication_path?: string; + }; vars?: Record; }; @@ -24,7 +28,6 @@ export type TlsConfig = { export type EnvConfig = { url: string; exclude?: string[]; - cluster_name?: string; tls?: TlsConfig; migrations: MigrationConfig; }; @@ -43,7 +46,6 @@ export type ClismaConfig = { type HCLEnvBlock = { url: string[] | string; exclude?: string[][] | string[]; - cluster_name?: string[] | string; tls?: Array<{ ca_file?: string[] | string; cert_file?: string[] | string; @@ -51,8 +53,12 @@ type HCLEnvBlock = { }>; migrations: Array<{ dir: string[] | string; - table_name?: string[] | string; - replication_path?: string[] | string; + table?: Array<{ + name?: string[] | string; + is_replicated?: boolean[] | boolean; + cluster_name?: string[] | string; + replication_path?: string[] | string; + }>; vars?: Record[] | Record; }>; }; @@ -147,17 +153,25 @@ export const parseConfig = async ( } } - // Parse optional table_name - const tableName = migrationBlock.table_name - ? extractValue(migrationBlock.table_name, "schema_migrations") + const tableBlock = migrationBlock.table?.[0]; + const tableName = tableBlock?.name + ? extractValue(tableBlock.name, "schema_migrations") : "schema_migrations"; - const replicationPath = migrationBlock.replication_path + const clusterName = tableBlock?.cluster_name + ? resolveValue(extractValue(tableBlock.cluster_name, ""), env, variables) || + undefined + : undefined; + const replicationPath = tableBlock?.replication_path ? resolveValue( - extractValue(migrationBlock.replication_path, ""), + extractValue(tableBlock.replication_path, ""), env, variables, ) || undefined : undefined; + const hasIsReplicated = tableBlock?.is_replicated !== undefined; + const isReplicated = hasIsReplicated + ? extractValue(tableBlock?.is_replicated, false) + : Boolean(replicationPath || clusterName); // Parse URL const url = extractValue(envBlock.url, ""); @@ -168,11 +182,6 @@ export const parseConfig = async ( ); } - const clusterNameValue = envBlock.cluster_name - ? resolveValue(extractValue(envBlock.cluster_name, ""), env, variables) - : ""; - - const clusterName = clusterNameValue || undefined; const tlsBlock = envBlock.tls?.[0]; let tls: TlsConfig | undefined; @@ -215,12 +224,15 @@ export const parseConfig = async ( return { url: resolveValue(url, env, variables), exclude: excludePatterns, - cluster_name: clusterName, tls, migrations: { dir: resolveValue(migrationDir, env, variables), - table_name: tableName, - replication_path: replicationPath, + table: { + name: tableName, + is_replicated: isReplicated, + cluster_name: clusterName, + replication_path: replicationPath, + }, vars: resolvedVars, }, }; diff --git a/packages/config/src/hcl-schema.ts b/packages/config/src/hcl-schema.ts index 56f745b..d107bae 100644 --- a/packages/config/src/hcl-schema.ts +++ b/packages/config/src/hcl-schema.ts @@ -33,11 +33,6 @@ export const clismaSchema = { type: "string", description: "ClickHouse connection string.", }, - cluster_name: { - type: "string", - description: - "Optional cluster name. When set, migrations use replicated tracking and templating.", - }, exclude: { type: "array", items: { @@ -70,14 +65,12 @@ export const clismaSchema = { type: "string", description: "Path to migrations directory.", }, - table_name: { - type: "string", - description: "Custom table name for migration tracking.", - }, - replication_path: { - type: "string", - description: - "Optional replication path for the migrations table in clustered setups.", + table: { + type: "array", + items: { + $ref: "#/$defs/tableBlock", + }, + description: "Migrations tracking table configuration.", }, vars: { type: "object", @@ -88,6 +81,31 @@ export const clismaSchema = { }, }, }, + tableBlock: { + type: "object", + additionalProperties: false, + properties: { + name: { + type: "string", + description: "Custom table name for migration tracking.", + }, + is_replicated: { + type: "boolean", + description: + "Whether tracking table should use ReplicatedReplacingMergeTree and ON CLUSTER.", + }, + cluster_name: { + type: "string", + description: + "Optional cluster name override for ON CLUSTER when replicated mode is enabled.", + }, + replication_path: { + type: "string", + description: + "Optional replication path for the migrations table in replicated setups.", + }, + }, + }, tlsBlock: { type: "object", additionalProperties: false, diff --git a/packages/config/src/tests/parse-config.test.ts b/packages/config/src/tests/parse-config.test.ts index acc23da..d7ff7f4 100644 --- a/packages/config/src/tests/parse-config.test.ts +++ b/packages/config/src/tests/parse-config.test.ts @@ -39,6 +39,50 @@ env "prod" { } `; +const SAMPLE_HCL_WITH_TABLE = ` +env "prod" { + url = "http://default:password@localhost:8123/default" + + migrations { + dir = "migrations" + + table { + name = "custom_migrations" + is_replicated = true + replication_path = "/clickhouse/some/path" + } + } +} +`; + +const SAMPLE_HCL_WITH_ONLY_REPLICATION_PATH = ` +env "prod" { + url = "http://default:password@localhost:8123/default" + + migrations { + dir = "migrations" + + table { + replication_path = "/clickhouse/some/path" + } + } +} +`; + +const SAMPLE_HCL_WITH_CLUSTER_NAME = ` +env "prod" { + url = "http://default:password@localhost:8123/default" + + migrations { + dir = "migrations" + + table { + cluster_name = "prod-cluster" + } + } +} +`; + test("parseConfig resolves env() and var.* values", async () => { const config = await parseConfig( SAMPLE_HCL, @@ -71,6 +115,39 @@ test("parseConfig resolves TLS block and validates mTLS fields", async () => { assert.equal(config.tls?.key_file, "certs/client.key"); }); +test("parseConfig parses migrations.table block", async () => { + const config = await parseConfig(SAMPLE_HCL_WITH_TABLE, {}, {}, "prod"); + + assert.equal(config.migrations.table?.name, "custom_migrations"); + assert.equal(config.migrations.table?.is_replicated, true); + assert.equal( + config.migrations.table?.replication_path, + "/clickhouse/some/path", + ); +}); + +test("parseConfig enables replication when replication_path is set", async () => { + const config = await parseConfig( + SAMPLE_HCL_WITH_ONLY_REPLICATION_PATH, + {}, + {}, + "prod", + ); + + assert.equal(config.migrations.table?.is_replicated, true); + assert.equal( + config.migrations.table?.replication_path, + "/clickhouse/some/path", + ); +}); + +test("parseConfig enables replication when cluster_name is set", async () => { + const config = await parseConfig(SAMPLE_HCL_WITH_CLUSTER_NAME, {}, {}, "prod"); + + assert.equal(config.migrations.table?.is_replicated, true); + assert.equal(config.migrations.table?.cluster_name, "prod-cluster"); +}); + test("parseConfig throws when only one mTLS file is provided", async () => { await assert.rejects( () => diff --git a/packages/core/package.json b/packages/core/package.json index 0837078..f227fcb 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@clisma/core", - "version": "0.2.1", + "version": "0.3.0", "description": "Core ClickHouse migration logic for clisma", "type": "module", "main": "dist/index.js", @@ -13,7 +13,7 @@ "url": "git+https://github.com/StopMakingThatBigFace/clisma.git" }, "dependencies": { - "@clisma/config": "0.2.1", + "@clisma/config": "0.3.0", "@clickhouse/client": "^1.11.2", "handlebars": "^4.7.8", "kleur": "^4.1.5", diff --git a/packages/core/src/migrations/repository.ts b/packages/core/src/migrations/repository.ts index 7c22ad8..168146b 100644 --- a/packages/core/src/migrations/repository.ts +++ b/packages/core/src/migrations/repository.ts @@ -39,57 +39,54 @@ export class MigrationRepository { return this.#ctx; } - async initialize(clusterName?: string): Promise { + async initialize( + isReplicated = false, + clusterName?: string, + ): Promise { if (this.#ctx) { return this.#ctx.cluster || null; } - const clusters = await this.#listClusters(); - const clusterNames = Array.from( - new Set(clusters.map((cluster) => cluster.cluster)), - ); - - const hasNonDefaultCluster = clusterNames.some( - (name) => name !== "default", - ); - - const defaultClusterRows = clusters.filter( - (cluster) => cluster.cluster === "default", - ); - - const defaultHasReplicasOrShards = defaultClusterRows.some( - (row) => row.replica_num > 1 || row.shard_num > 1, - ); - - if (!clusterName) { - if (hasNonDefaultCluster || defaultHasReplicasOrShards) { - const available = clusterNames.length - ? clusterNames.join(", ") - : "none"; - + if (isReplicated) { + const clusters = await this.#listClusters(); + const clusterNames = Array.from( + new Set(clusters.map((cluster) => cluster.cluster)), + ); + const nonDefault = clusterNames.filter((name) => name !== "default"); + + let selectedCluster = clusterName; + if (selectedCluster) { + if (!clusterNames.includes(selectedCluster)) { + throw new Error( + `Cluster "${selectedCluster}" not found. Available clusters: ${clusterNames.join(", ")}`, + ); + } + } else if (nonDefault.length === 1) { + selectedCluster = nonDefault[0]; + } else if (nonDefault.length > 1) { throw new Error( - `Cluster detected but no cluster_name provided. ` + - `Set env.cluster_name in config. Available clusters: ${available}`, + `Multiple non-default clusters found: ${nonDefault.join(", ")}. ` + + "Set migrations.table.cluster_name to choose one.", ); + } else if (clusterNames.includes("default")) { + selectedCluster = "default"; } - } else if (!clusterNames.includes(clusterName)) { - const available = clusterNames.length ? clusterNames.join(", ") : "none"; - throw new Error( - `Cluster "${clusterName}" not found. Available clusters: ${available}`, - ); - } + if (!selectedCluster) { + throw new Error( + "Replicated migrations table requested, but no cluster found in system.clusters.", + ); + } - if (clusterName) { - const safeClusterName = clusterName.replace(/"/g, '\\"'); + const safeClusterName = selectedCluster.replace(/"/g, '\\"'); this.#ctx = { is_replicated: true, create_table_options: `ON CLUSTER "${safeClusterName}"`, - cluster: clusterName, + cluster: selectedCluster, }; - return clusterName; + return selectedCluster; } this.#ctx = { @@ -103,6 +100,7 @@ export class MigrationRepository { async ensureMigrationsTable(): Promise { const ctx = this.getContext(); + const existedBefore = await this.#migrationsTableExists(); const replicationPath = this.#replicationPath ? this.#replicationPath : `/clickhouse/tables/cluster-{cluster}/shard-{shard}/{database}/${this.#tableName}`; @@ -127,6 +125,15 @@ export class MigrationRepository { ORDER BY version `, }); + + if (!existedBefore) { + const mode = ctx.is_replicated ? "replicated" : "non-replicated"; + console.log( + kleur.green( + `Created migrations table ${kleur.bold(this.#tableName)} in ${mode} mode`, + ), + ); + } } async getAppliedMigrations(): Promise> { @@ -205,4 +212,20 @@ export class MigrationRepository { replica_num: number; }>(); } + + async #migrationsTableExists(): Promise { + const safeTableName = this.#tableName.replace(/'/g, "\\'"); + const result = await this.#client.query({ + query: ` + SELECT count() AS count + FROM system.tables + WHERE database = currentDatabase() AND name = '${safeTableName}' + `, + format: "JSONEachRow", + }); + + const rows = await result.json<{ count: number }>(); + + return (rows[0]?.count || 0) > 0; + } } diff --git a/packages/core/src/migrations/runner.ts b/packages/core/src/migrations/runner.ts index 32b7ba2..2e836bf 100644 --- a/packages/core/src/migrations/runner.ts +++ b/packages/core/src/migrations/runner.ts @@ -23,6 +23,7 @@ export class MigrationRunner { #client: ClickHouseClient; #repository: MigrationRepository; #tableName: string; + #isReplicated: boolean; #clusterName?: string; #replicationPath?: string; #tls?: MigrationRunnerOptions["tls"]; @@ -45,6 +46,7 @@ export class MigrationRunner { } this.#tableName = options.tableName || "schema_migrations"; + this.#isReplicated = options.isReplicated || false; this.#clusterName = options.clusterName; this.#replicationPath = options.replicationPath; this.#tls = options.tls; @@ -88,17 +90,21 @@ export class MigrationRunner { } const spinner = ora("Detecting cluster configuration...").start(); + try { - const clusterName = await this.#repository.initialize(this.#clusterName); + const clusterName = await this.#repository.initialize( + this.#isReplicated, + this.#clusterName, + ); if (clusterName) { - const message = this.#clusterName - ? `Using cluster from config: ${kleur.bold(clusterName)}` - : `Detected cluster: ${kleur.bold(clusterName)}`; - - spinner.succeed(kleur.green(message)); + spinner.succeed( + kleur.green( + `Replicated mode enabled on cluster: ${kleur.bold(clusterName)}`, + ), + ); } else { - spinner.info("No cluster detected, using non-replicated mode"); + spinner.info("Using non-replicated mode"); } this.#initialized = true; diff --git a/packages/core/src/migrations/types.ts b/packages/core/src/migrations/types.ts index 9ba1382..50f07e8 100644 --- a/packages/core/src/migrations/types.ts +++ b/packages/core/src/migrations/types.ts @@ -7,6 +7,7 @@ export type MigrationRunnerTLSOptions = { export type MigrationRunnerOptions = { migrationsDir: string; connectionString: string; + isReplicated?: boolean; clusterName?: string; tableName?: string; replicationPath?: string; From f4d32090c9f1937654e7fd4abc673637fdd8fd9d Mon Sep 17 00:00:00 2001 From: ruby Date: Fri, 13 Feb 2026 13:35:23 +0400 Subject: [PATCH 2/2] feat(config,migrations): replace env.cluster_name with migrations.table; add replicated cluster override --- packages/cli/src/cli.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 74890aa..a836a3d 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -167,7 +167,6 @@ const runCommand = async ( templateVars = { ...(envConfig.migrations.vars || {}), - ...(clusterName ? { cluster_name: clusterName } : {}), }; console.log(