diff --git a/src/commands/register.ts b/src/commands/register.ts index 772569f9a..3bc488fef 100644 --- a/src/commands/register.ts +++ b/src/commands/register.ts @@ -174,7 +174,7 @@ export default class ComponentRegister extends BaseCommand { throw new Error('Component Config must have a name'); } - validateInterpolation(component_spec); + validateInterpolation(component_spec, true); const { component_name } = ComponentSlugUtils.parse(component_spec.name); @@ -197,7 +197,8 @@ export default class ComponentRegister extends BaseCommand { const dependency_manager = new LocalDependencyManager(this.app.api, selected_account.name); dependency_manager.environment = 'production'; - const graph = await dependency_manager.getGraph([instanceToInstance(component_spec)], undefined, { interpolate: false, validate: false }); + const graph = await dependency_manager.getGraph([instanceToInstance(component_spec)], undefined, { interpolate: true, validate: true, relax_validation: true }); + // Tmp fix to register host overrides for (const node of graph.nodes.filter(n => n instanceof ServiceNode) as ServiceNode[]) { for (const interface_config of Object.values(node.interfaces)) { diff --git a/src/dependency-manager/graph/type.ts b/src/dependency-manager/graph/type.ts index 930f5698b..72009088d 100644 --- a/src/dependency-manager/graph/type.ts +++ b/src/dependency-manager/graph/type.ts @@ -1,4 +1,5 @@ export interface GraphOptions { interpolate?: boolean; validate?: boolean; + relax_validation?: boolean, } diff --git a/src/dependency-manager/manager.ts b/src/dependency-manager/manager.ts index e5e476dc5..0f5c0f4a8 100644 --- a/src/dependency-manager/manager.ts +++ b/src/dependency-manager/manager.ts @@ -1,6 +1,6 @@ import { instanceToPlain, plainToInstance, serialize } from 'class-transformer'; import { buildNodeRef, ComponentConfig } from './config/component-config'; -import { ArchitectContext, ComponentContext, DatabaseContext } from './config/component-context'; +import { ArchitectContext, ComponentContext, DatabaseContext, ServiceContext } from './config/component-context'; import { DeprecatedInterfacesSpec } from './deprecated-spec/interfaces'; import { DependencyGraph, DependencyGraphMutable } from './graph'; import { IngressEdge } from './graph/edge/ingress'; @@ -21,6 +21,8 @@ import { validateOrRejectSpec } from './spec/utils/spec-validator'; import { Dictionary, transformDictionary } from './utils/dictionary'; import { ArchitectError } from './utils/errors'; import { interpolateObjectLoose, interpolateObjectOrReject, replaceInterpolationBrackets } from './utils/interpolation'; +import { IngressConfig, ServiceConfig, ServiceInterfaceConfig } from './config/service-config'; +import { SecretDefinitionSpec, SecretSpecValue } from './spec/secret-spec'; export default abstract class DependencyManager { account?: string; @@ -395,6 +397,7 @@ export default abstract class DependencyManager { options = { interpolate: true, validate: true, + relax_validation: false, ...options, }; @@ -412,6 +415,14 @@ export default abstract class DependencyManager { const evaluated_component_specs: ComponentSpec[] = []; for (const raw_component_spec of component_specs) { + if (options.relax_validation && raw_component_spec.secrets) { + // Assign dummy value to unset secrets + for (const [key, secret] of Object.entries(raw_component_spec.secrets as Dictionary)) { + if (!secret) { + raw_component_spec.secrets[key] = { default: '-999' }; + } + } + } const { component_spec, context } = await this.getComponentSpecContext(graph, raw_component_spec, secrets, options); if (options.interpolate) { @@ -486,12 +497,67 @@ export default abstract class DependencyManager { context.dependencies = {}; for (const dep_name of Object.keys(component_spec.dependencies || {})) { const dependency_context = context_map[dep_name]; - if (!dependency_context) continue; + if (!dependency_context && !options.relax_validation) { + continue; + } + + if (options.relax_validation) { + // Mock dependency node for validation + const mock_service_ingress_config: IngressConfig = { + enabled: false, + subdomain: '', + path: '', + ip_whitelist: [], + sticky: '', + private: false, + consumers: [], + dns_zone: '', + host: '', + port: '', + protocol: '', + username: '', + password: '', + url: '', + }; + const mock_service_interface_config: ServiceInterfaceConfig = { + host: '', + port: '', + protocol: '', + username: '', + password: '', + url: '', + sticky: '', + path: '', + ingress: mock_service_ingress_config, + }; + const mock_dependency_node = { + __type: '', + config: {} as ServiceConfig, + ref: `${dep_name}--ø`, + component_ref: dep_name, + service_name: 'ø', + interfaces: { 'ø': mock_service_interface_config }, + ingresses: { 'ø': mock_service_ingress_config }, + ports: [], + is_external: false, + instance_id: '', + }; + graph.addNode(mock_dependency_node); - context.dependencies[dep_name] = { - services: dependency_context.services || {}, - outputs: dependency_context.outputs || {}, - }; + const service_context: ServiceContext = { + environment: {}, + interfaces: { 'ø': mock_service_interface_config }, + }; + context.dependencies[dep_name] = { + services: { 'ø': service_context }, + outputs: {}, + }; + } else { + context.dependencies[dep_name] = { + services: dependency_context.services || {}, + outputs: dependency_context.outputs || {}, + }; + } } } @@ -506,7 +572,7 @@ export default abstract class DependencyManager { const context = context_map[component_spec.metadata.ref]; if (options.interpolate) { - component_spec = interpolateObject(component_spec, context, { keys: false, values: true, file: component_spec.metadata.file }); + component_spec = interpolateObject(component_spec, context, { keys: false, values: true, file: component_spec.metadata.file, relax_validation: options.relax_validation }); component_spec.metadata.interpolated = true; } @@ -529,7 +595,6 @@ export default abstract class DependencyManager { this.validateGraph(graph); graph.validated = true; } - return Object.freeze(graph); } } diff --git a/src/dependency-manager/spec/utils/interpolation.ts b/src/dependency-manager/spec/utils/interpolation.ts index cf4e0bfdb..255225829 100644 --- a/src/dependency-manager/spec/utils/interpolation.ts +++ b/src/dependency-manager/spec/utils/interpolation.ts @@ -1,2 +1,6 @@ export const EXPRESSION_REGEX = new RegExp(`\\\${{\\s*(.*?)\\s*}}`, 'g'); export const IF_EXPRESSION_REGEX = new RegExp(`^\\\${{\\s*if(.*?)\\s*}}$`); +export const DEPENDENCY_EXPRESSION_REGEX = new RegExp('^\\${{\\s*dependencies\\.[^}]*}}$'); +export const DEPRECATED_DEPENDENCY_EXPRESSION_REGEX = new RegExp('\\$\\{\\{ dependencies\\.[^.]*\\.(ingresses|interfaces)\\.[^.]*\\.url }}'); +export const ARCHITECT_EXPRESSION_REGEX = new RegExp('^\\${{\\s*architect\\.[^}]*}}'); +export const ENVIRONMENT_EXPRESSION_REGEX = new RegExp('^\\${{\\s*environment\\.[^}]*}}'); diff --git a/src/dependency-manager/spec/utils/spec-validator.ts b/src/dependency-manager/spec/utils/spec-validator.ts index 9f31e4c25..dd95bdae7 100644 --- a/src/dependency-manager/spec/utils/spec-validator.ts +++ b/src/dependency-manager/spec/utils/spec-validator.ts @@ -16,6 +16,8 @@ import { ComponentInstanceMetadata, ComponentSpec } from '../component-spec'; import { ServiceInterfaceSpec } from '../service-spec'; import { findDefinition, getArchitectJSONSchema } from './json-schema'; import { Slugs } from './slugs'; +import { DependencyGraphMutable } from '../../graph'; +import { ServiceNode } from '../../graph/node/service'; export type AjvError = ErrorObject[] | null | undefined; @@ -354,15 +356,31 @@ export const validateOrRejectSpec = (parsed_yml: ParsedYaml, metadata?: Componen return component_spec; }; -export const validateInterpolation = (component_spec: ComponentSpec): void => { +const checkInterpolationPath = (queue: any, child: any): boolean => { + let key = queue.shift(); + const keys = Object.keys(child); + const star = keys.includes('*'); + if (star || child[key]) { + key = star ? '*' : key; + if (queue.length > 0 && typeof child[key] === 'object') { + return checkInterpolationPath(queue, child[key]); + } else { + return true; + } + } else { + return keys.includes(key); + } +}; + +export const validateInterpolation = (component_spec: ComponentSpec, relax_validation?: boolean): void => { const { errors } = interpolateObject(component_spec, {}, { keys: true, values: true, file: component_spec.metadata.file, + relax_validation: relax_validation, }); const filtered_errors = errors.filter(error => !error.message.startsWith(RequiredInterpolationRule.PREFIX)); - if (filtered_errors.length > 0) { throw new ValidationErrors(filtered_errors, component_spec.metadata.file); } diff --git a/src/dependency-manager/utils/interpolation.ts b/src/dependency-manager/utils/interpolation.ts index 3dea52a7d..3b264a41d 100644 --- a/src/dependency-manager/utils/interpolation.ts +++ b/src/dependency-manager/utils/interpolation.ts @@ -1,11 +1,14 @@ import { instanceToInstance } from 'class-transformer'; import deepmerge from 'deepmerge'; -import { EXPRESSION_REGEX, IF_EXPRESSION_REGEX } from '../spec/utils/interpolation'; +import { ARCHITECT_EXPRESSION_REGEX, DEPENDENCY_EXPRESSION_REGEX, DEPRECATED_DEPENDENCY_EXPRESSION_REGEX, ENVIRONMENT_EXPRESSION_REGEX, EXPRESSION_REGEX, IF_EXPRESSION_REGEX } from '../spec/utils/interpolation'; import { Dictionary } from './dictionary'; import { ValidationError, ValidationErrors } from './errors'; import { ArchitectParser } from './parser'; import { matches } from './regex'; import { CONTEXT_KEY_DELIMITER } from './rules'; +import { ComponentSpec } from '../spec/component-spec'; +import { ArchitectContext, DependencyContext, ServiceContext } from '../config/component-context'; +import { IngressConfig, ServiceInterfaceConfig } from '../config/service-config'; export const replaceBrackets = (value: string): string => { return value.replace(/\[/g, '.').replace(/["'\\\]|]/g, ''); @@ -23,8 +26,52 @@ export const replaceInterpolationBrackets = (value: string): string => { return res; }; -export const buildContextMap = (context: any): any => { +/* + Create mock dependencies for dependencies..services.*.interfaces.*. +*/ +const createMockDependencies = (component_spec: ComponentSpec) => { + const dependencies: Dictionary = {}; + for (const dep_name of Object.keys(component_spec.dependencies || {})) { + const mock_service_interface_config: ServiceInterfaceConfig = { + host: '', + port: '', + protocol: '', + username: '', + password: '', + url: '', + sticky: '', + path: '', + ingress: { private: false } as IngressConfig, + }; + const mock_service_context: ServiceContext = { + interfaces: { '*': mock_service_interface_config }, + environment: {}, + }; + + const dependency_context = { + name: dep_name, + dependencies: {}, + secrets: {}, + outputs: {}, + databases: {}, + services: { '*': mock_service_context }, + tasks: {}, + architect: {} as ArchitectContext, + }; + dependencies[dep_name] = { + services: dependency_context.services || {}, + outputs: dependency_context.outputs || {}, + }; + } + return dependencies; +}; + +export const buildContextMap = (context: any, use_mock_dependencies?: boolean): any => { const context_map: Dictionary = {}; + if (use_mock_dependencies) { + context.dependencies = createMockDependencies(context); + } + const queue = [['', context]]; while (queue.length > 0) { const [prefix, c] = queue.shift()!; @@ -47,6 +94,7 @@ export interface InterpolateObjectOptions { keys?: boolean; values?: boolean; file?: { path: string, contents: string }; + relax_validation?: boolean; } const overwriteMerge = (destinationArray: any[], sourceArray: any[], options: deepmerge.Options) => sourceArray; @@ -63,6 +111,7 @@ export const interpolateObject = (obj: T, context: any, _options?: Interpolat const options = { keys: false, values: true, + relax_validation: false, ..._options, }; @@ -99,8 +148,18 @@ export const interpolateObject = (obj: T, context: any, _options?: Interpolat error.invalid_key = true; } } else if (options.values && typeof value === 'string') { - const parsed_value = parser.parseString(value, context_map); - el[key] = parsed_value; + let updated_value = value; + const special_case = DEPRECATED_DEPENDENCY_EXPRESSION_REGEX.test(value) || ENVIRONMENT_EXPRESSION_REGEX.test(value) || ARCHITECT_EXPRESSION_REGEX.test(value); + if (options.relax_validation && special_case) { + el[key] = updated_value; + } else if (options.relax_validation && DEPENDENCY_EXPRESSION_REGEX.test(value)) { + updated_value = value.replace(/(services\.)[^.]+(\.interfaces\.)([^.]+)/g, '$1ø$2ø'); + const parsed_value = parser.parseString(updated_value, context_map); + el[key] = parsed_value; + } else { + const parsed_value = parser.parseString(value, context_map); + el[key] = parsed_value; + } } else { el[key] = value; if (value instanceof Object) { diff --git a/test/dependency-manager/interpolation-validation.test.ts b/test/dependency-manager/interpolation-validation.test.ts index f61a249da..003fe4ced 100644 --- a/test/dependency-manager/interpolation-validation.test.ts +++ b/test/dependency-manager/interpolation-validation.test.ts @@ -2,9 +2,6 @@ import { expect } from 'chai'; import { buildSpecFromYml, validateInterpolation, ValidationErrors } from '../../src'; describe('interpolation-validation', () => { - - const context = {} - describe('validate build block', () => { it('cannot use secret in build block', () => { const component_config = ` @@ -21,7 +18,7 @@ describe('interpolation-validation', () => { const component_spec = buildSpecFromYml(component_config) expect(() => { - validateInterpolation(component_spec) + validateInterpolation(component_spec); }).to.be.throws(ValidationErrors); }); @@ -39,7 +36,7 @@ describe('interpolation-validation', () => { const component_spec = buildSpecFromYml(component_config) expect(() => { - validateInterpolation(component_spec) + validateInterpolation(component_spec); }).to.be.throws(ValidationErrors); }); @@ -57,7 +54,7 @@ describe('interpolation-validation', () => { const component_spec = buildSpecFromYml(component_config) expect(() => { - validateInterpolation(component_spec) + validateInterpolation(component_spec); }).to.be.throws(ValidationErrors); }); @@ -75,7 +72,7 @@ describe('interpolation-validation', () => { const component_spec = buildSpecFromYml(component_config) expect(() => { - validateInterpolation(component_spec) + validateInterpolation(component_spec); }).to.be.throws(ValidationErrors); }); @@ -89,10 +86,10 @@ describe('interpolation-validation', () => { args: \${{ if architect.environment == 'local' }}: ENV: local - ` + `; - const component_spec = buildSpecFromYml(component_config) - validateInterpolation(component_spec) + const component_spec = buildSpecFromYml(component_config); + validateInterpolation(component_spec); }); it('can use conditional around build block if local', () => { @@ -104,10 +101,10 @@ describe('interpolation-validation', () => { build: args: ENV: local - ` + `; - const component_spec = buildSpecFromYml(component_config) - validateInterpolation(component_spec) + const component_spec = buildSpecFromYml(component_config); + validateInterpolation(component_spec); }); it('can use conditional around service block with build block if local', () => { @@ -119,10 +116,10 @@ describe('interpolation-validation', () => { build: args: ENV: local - ` + `; - const component_spec = buildSpecFromYml(component_config) - validateInterpolation(component_spec) + const component_spec = buildSpecFromYml(component_config); + validateInterpolation(component_spec); }); }); @@ -135,11 +132,11 @@ describe('interpolation-validation', () => { args: \${{ if architect.build.tag == 'latest' }}: ENV: prod - ` + `; - const component_spec = buildSpecFromYml(component_config) + const component_spec = buildSpecFromYml(component_config); expect(() => { - validateInterpolation(component_spec) + validateInterpolation(component_spec); }).to.be.throws(ValidationErrors); }); @@ -151,11 +148,11 @@ describe('interpolation-validation', () => { build: args: TAG: \${{ architect.build.tag }} - ` + `; - const component_spec = buildSpecFromYml(component_config) + const component_spec = buildSpecFromYml(component_config); expect(() => { - validateInterpolation(component_spec) + validateInterpolation(component_spec); }).to.be.throws(ValidationErrors); }); @@ -169,10 +166,10 @@ describe('interpolation-validation', () => { api: environment: TEST: \${{ secrets.test }} - ` + `; - const component_spec = buildSpecFromYml(component_config) - validateInterpolation(component_spec) + const component_spec = buildSpecFromYml(component_config); + validateInterpolation(component_spec); }); it('can still use conditional without build block', () => { @@ -183,10 +180,10 @@ describe('interpolation-validation', () => { \${{ if architect.environment == 'local' }}: environment: TEST: test - ` + `; - const component_spec = buildSpecFromYml(component_config) - validateInterpolation(component_spec) + const component_spec = buildSpecFromYml(component_config); + validateInterpolation(component_spec); }); }); }); diff --git a/test/mocks/superset/architect.yml b/test/mocks/superset/architect.yml index 4edc51b38..7ec9bbf7c 100644 --- a/test/mocks/superset/architect.yml +++ b/test/mocks/superset/architect.yml @@ -161,7 +161,7 @@ services: interfaces: http: 8080 environment: - BACKUP_DB_ADDR: ${{ databases.api-db.connection_string }} + BACKUP_DB_ADDR: ${{ databases.api-db2.connection_string }} DB_ADDR: ${{ services.api-db.interfaces.postgres.url }}/${{ secrets.db_name }} DB_USER: ${{ secrets.db_user }} DB_PASS: ${{ secrets.db_pass }}