From f59a067adb4def6bb3e0764e99a63c77ce5625cb Mon Sep 17 00:00:00 2001 From: cmidgley Date: Tue, 15 Oct 2024 13:38:33 -0400 Subject: [PATCH 1/5] Implement manifest 'uninclude' --- tools/mcmanifest.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/mcmanifest.js b/tools/mcmanifest.js index 51e0979900..73af9ab70c 100644 --- a/tools/mcmanifest.js +++ b/tools/mcmanifest.js @@ -1905,6 +1905,8 @@ export class Tool extends TOOL { includeManifest(it) { var currentDirectory = this.currentDirectory; if ("string" == typeof it) { + if (this.uninclude?.includes(it)) + return; this.includeManifestPath(this.resolveVariable(it)); } else if (this.buildTarget != "clean") { @@ -2184,6 +2186,7 @@ export class Tool extends TOOL { } parseManifest(path, manifest) { let platformInclude; + let platformUninclude; if (!manifest) { var buffer = this.readFileString(path); try { @@ -2206,6 +2209,7 @@ export class Tool extends TOOL { if (platform) { this.parseBuild(platform); platformInclude = platform.include; + platformUninclude = platform.uninclude; if (platformInclude) { if (!("include" in manifest)) manifest.include = platformInclude; @@ -2215,6 +2219,13 @@ export class Tool extends TOOL { manifest.include = manifest.include.concat(platformInclude); } } + if (platformUninclude) { + if (!this.uninclude) + this.uninclude = []; + if ("string" === typeof manifest.uninclude) + this.uninclude.push(manifest.uninclude); + this.uninclude = this.uninclude.concat(platformUninclude); + } if (platform.dependency && ("esp32" == this.platform)) { manifest.dependencies = []; for (let i=0; i Date: Tue, 15 Oct 2024 14:05:21 -0400 Subject: [PATCH 2/5] Document uninclude --- documentation/tools/manifest.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/documentation/tools/manifest.md b/documentation/tools/manifest.md index bc7149a9be..a3aa0ab096 100644 --- a/documentation/tools/manifest.md +++ b/documentation/tools/manifest.md @@ -12,6 +12,7 @@ A manifest is a JSON file that describes the modules and resources necessary to * [`build`](#build) * [`include`](#include) * [Including git repositories](#include-git) + * [`uninclude`](#uninclude) * [`creation`](#creation) * [`defines`](#defines) * [`config`](#config) @@ -219,6 +220,22 @@ Each example application in the Moddable SDK includes at least one of the manife Several touch, display, and sensor [drivers](../../modules/drivers) and some [networking modules](../../modules/network) also have manifests to make it easier to incorporate them into your projects. + +### `uninclude` + +The `uninclude` array lists manifests that should not be included. Names must exactly match an `include` in a manifest not yet processed, which are all manifest files (and their sub-includes) in the current manifest file, as well as the platform/subplatform manifests. + +For example, to change the directory for subplatforms only on `ESP32` the following can be used: + +```json + "platforms": { + "esp32/*": { + "uninclude": "./targets/$(SUBPLATFORM)/manifest.json", + "include": "$(SRC)/targets/$(SUBPLATFORM)/manifest.json" + } + } +``` + #### Including Git Repositories A manifest may directly include git repositories. The repositories are cloned as part of the build process and stored with the project's temporary build files. From f49cea0071c4d0db371504e635c0a5febd8d4d4f Mon Sep 17 00:00:00 2001 From: cmidgley Date: Tue, 15 Oct 2024 14:09:56 -0400 Subject: [PATCH 3/5] Moved position of uninclude in doc --- documentation/tools/manifest.md | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/documentation/tools/manifest.md b/documentation/tools/manifest.md index a3aa0ab096..31c45bb424 100644 --- a/documentation/tools/manifest.md +++ b/documentation/tools/manifest.md @@ -220,22 +220,6 @@ Each example application in the Moddable SDK includes at least one of the manife Several touch, display, and sensor [drivers](../../modules/drivers) and some [networking modules](../../modules/network) also have manifests to make it easier to incorporate them into your projects. - -### `uninclude` - -The `uninclude` array lists manifests that should not be included. Names must exactly match an `include` in a manifest not yet processed, which are all manifest files (and their sub-includes) in the current manifest file, as well as the platform/subplatform manifests. - -For example, to change the directory for subplatforms only on `ESP32` the following can be used: - -```json - "platforms": { - "esp32/*": { - "uninclude": "./targets/$(SUBPLATFORM)/manifest.json", - "include": "$(SRC)/targets/$(SUBPLATFORM)/manifest.json" - } - } -``` - #### Including Git Repositories A manifest may directly include git repositories. The repositories are cloned as part of the build process and stored with the project's temporary build files. @@ -299,6 +283,24 @@ The hostname, pathname. branch, and tag are included in the path where the clone *** + +### `uninclude` + +The `uninclude` array lists manifests that should not be included. Names must exactly match an `include` in a manifest not yet processed, which are all manifest files (and their sub-includes) in the current manifest file, as well as the platform/subplatform manifests. + +For example, to change the directory for subplatforms only on `ESP32` the following can be used: + +```json + "platforms": { + "esp32/*": { + "uninclude": "./targets/$(SUBPLATFORM)/manifest.json", + "include": "$(SRC)/targets/$(SUBPLATFORM)/manifest.json" + } + } +``` + +*** + ### `creation` From 4268236f514e973e162a7ce5ebab2894e7120281 Mon Sep 17 00:00:00 2001 From: cmidgley Date: Fri, 18 Oct 2024 17:12:00 -0400 Subject: [PATCH 4/5] Rework for `redirect` with `from` and `to`, across many properties, with `null` wildcard support. --- documentation/tools/manifest.md | 105 ++++++++++++++++++++++++++------ tools/mcmanifest.js | 93 +++++++++++++++++++++++++--- 2 files changed, 169 insertions(+), 29 deletions(-) diff --git a/documentation/tools/manifest.md b/documentation/tools/manifest.md index 31c45bb424..6ad96d0ff8 100644 --- a/documentation/tools/manifest.md +++ b/documentation/tools/manifest.md @@ -12,7 +12,6 @@ A manifest is a JSON file that describes the modules and resources necessary to * [`build`](#build) * [`include`](#include) * [Including git repositories](#include-git) - * [`uninclude`](#uninclude) * [`creation`](#creation) * [`defines`](#defines) * [`config`](#config) @@ -24,6 +23,7 @@ A manifest is a JSON file that describes the modules and resources necessary to * [`platforms`](#platforms) * [`subplatforms`](#subplatforms) * [`bundle`](#bundle) + * [`redirect`](#redirect) * [How manifests are processed](#process) @@ -283,24 +283,6 @@ The hostname, pathname. branch, and tag are included in the path where the clone *** - -### `uninclude` - -The `uninclude` array lists manifests that should not be included. Names must exactly match an `include` in a manifest not yet processed, which are all manifest files (and their sub-includes) in the current manifest file, as well as the platform/subplatform manifests. - -For example, to change the directory for subplatforms only on `ESP32` the following can be used: - -```json - "platforms": { - "esp32/*": { - "uninclude": "./targets/$(SUBPLATFORM)/manifest.json", - "include": "$(SRC)/targets/$(SUBPLATFORM)/manifest.json" - } - } -``` - -*** - ### `creation` @@ -725,6 +707,91 @@ The `bundle` object is used by the [`mcbundle` command line tool](./tools.md#mcb "icon": "./store/icon.png" } ``` +*** + + +### `redirect` + +The `redirect` array can be used to redirect (or change) manifest properties that are defined in other manifest files. This is useful when including system manifests where properties need to be altered without editing the Moddable core files. + +For example, to use a private directory for sub-modules, you can use an `include` redirection like this: + +```json +"redirect": { + "include": { + "from": "./targets/$(SUBPLATFORM)/manifest.json", + "to": "$(SRC)/targets/$(PLATFORM)/$(SUBPLATFORM)/manifest.json" + } +} +``` + +The properties `include`, `strip`, and `preload` (which are string arrays in the manifest) expect `from` and `to` for each property to redirect. The redirection rule can be a single object, or an array of objects: + +```json +"redirect": { + "include": [ + { + "from": "some-include-path", + "to": "some-redirect-path" + }, + { + "from": "another-include-path", + "to": "new-path" + } + ] +} +``` + +The `modules`, `resources`, `data`, `build`, and `config` properties (which are objects in the manifest) expect a similar format, but with an object key qualifier: + +```json +"redirect": { + "modules": { + "embedded:provider/builtin": { + "from": "./targets/$(SUBPLATFORM)/manifest.json", + "to": "$(SRC)/targets/$(SUBPLATFORM)/manifest.json" + } + } +} +``` + +The value of `null` can be used as a wildcard on `from` to match all. For example, to disable all preloads and replace them with a specific list of modules: + +```json +"redirect": { + "preload": { + "from": null, + "to": ["engine", "unit-test"] + } +} +``` + +`to` can also use `null` to indicate the item should be deleted (if `to` is not provided, it is assumed to be a delete). For example, to remove all preloads: + +```json +"redirect": { + "preload": { + "from": null + } +} +``` + +You can qualify `redirect` to apply only for specific platforms by placing it inside the `platforms` property. The following will replace the `config.rotation` value only for the `esp32/m5stick_cplus` platform: + +```json +"platforms": { + "esp32/m5stick_cplus": { + "redirect": { + "config": { + "rotation": { + "from": "270", + "to": "90" + } + } + } + } +} +``` *** diff --git a/tools/mcmanifest.js b/tools/mcmanifest.js index 73af9ab70c..c69983fc99 100644 --- a/tools/mcmanifest.js +++ b/tools/mcmanifest.js @@ -1629,6 +1629,7 @@ export class Tool extends TOOL { this.windows = this.currentPlatform == "win"; this.slash = this.windows ? "\\" : "/"; this.escapedHash = this.windows ? "^#" : "\\#"; + this.redirectRules = []; this.buildPath = this.moddablePath + this.slash + "build"; this.xsPath = this.moddablePath + this.slash + "xs"; @@ -1905,8 +1906,6 @@ export class Tool extends TOOL { includeManifest(it) { var currentDirectory = this.currentDirectory; if ("string" == typeof it) { - if (this.uninclude?.includes(it)) - return; this.includeManifestPath(this.resolveVariable(it)); } else if (this.buildTarget != "clean") { @@ -2186,7 +2185,7 @@ export class Tool extends TOOL { } parseManifest(path, manifest) { let platformInclude; - let platformUninclude; + if (!manifest) { var buffer = this.readFileString(path); try { @@ -2209,7 +2208,6 @@ export class Tool extends TOOL { if (platform) { this.parseBuild(platform); platformInclude = platform.include; - platformUninclude = platform.uninclude; if (platformInclude) { if (!("include" in manifest)) manifest.include = platformInclude; @@ -2219,12 +2217,8 @@ export class Tool extends TOOL { manifest.include = manifest.include.concat(platformInclude); } } - if (platformUninclude) { - if (!this.uninclude) - this.uninclude = []; - if ("string" === typeof manifest.uninclude) - this.uninclude.push(manifest.uninclude); - this.uninclude = this.uninclude.concat(platformUninclude); + if ("redirect" in platform) { + this.redirectRules = this.redirectRules.concat(platform.redirect); } if (platform.dependency && ("esp32" == this.platform)) { manifest.dependencies = []; @@ -2237,6 +2231,10 @@ export class Tool extends TOOL { } } } + if ("redirect" in manifest) { + this.redirectRules = this.redirectRules.concat(manifest.redirect); + } + this.redirect(manifest); if ("include" in manifest) { if (manifest.include instanceof Array) manifest.include.forEach(include => this.includeManifest(include)); @@ -2246,6 +2244,80 @@ export class Tool extends TOOL { this.manifests.push(manifest); return manifest; } + redirect(manifest) { + let platform; + if ("platforms" in manifest) + platform = this.matchPlatform(manifest.platforms, this.fullplatform, false); + for (const redirectRule of this.redirectRules) { + for (const manifestProperty of Object.keys(redirectRule)) { + if (!Array.isArray(redirectRule[manifestProperty])) + redirectRule[manifestProperty] = [redirectRule[manifestProperty]]; + if (["include", "strip", "preload"].includes(manifestProperty)) { + for (const singleRule of redirectRule[manifestProperty]) { + if (singleRule === null || !("from" in singleRule)) + throw new Error(`Missing "from" in redirect rule for property "${manifestProperty}"`); + this.redirectArray(manifest, manifestProperty, singleRule.from , singleRule.to); + if (platform) + this.redirectArray(platform, manifestProperty, singleRule.from, singleRule.to); + } + } else if (["modules", "resources", "data", "build", "config"].includes(manifestProperty)) { + for (const keyedRule of redirectRule[manifestProperty]) { + if (keyedRule === null || "object" !== typeof keyedRule) + throw new Error(`Invalid redirect rule "${manifestProperty}" (must be object, not ${typeof keyedRule})`); + for (const singleRuleKey of Object.keys(keyedRule)) { + const singleRule = keyedRule[singleRuleKey]; + if (singleRule === null || "object" !== typeof singleRule) + throw new Error(`Invalid redirect rule "${manifestProperty}.${singleRuleKey}" (must be object, not ${typeof singleRule})`); + if (!("from" in singleRule)) + throw new Error(`Missing "from" in redirect rule for property "${manifestProperty}"`); + this.redirectObject(manifest, manifestProperty, singleRuleKey, keyedRule[singleRuleKey].from, keyedRule[singleRuleKey].to); + if (platform) + this.redirectObject(platform, manifestProperty, singleRuleKey, keyedRule[singleRuleKey].from, keyedRule[singleRuleKey].to); + } + } + } + else + throw new Error(`Redirect not supported for property "${manifestProperty}"`); + } + } + } + redirectArray(manifest, property, from, to) { + if (!from) { + if (property in manifest) { + manifest[property] = to ?? []; + } + } else if ("string" == typeof from) { + if (property in manifest) { + let array = Array.isArray(manifest[property]) ? manifest[property] : [manifest[property]]; + if (array.includes(from)) { + if (to) { + if (Array.isArray(manifest[property]) && from) + manifest[property] = manifest[property].map(item => item == from ? to : item); + else + manifest[property] = to; + } else { + if (Array.isArray(manifest[property]) && manifest[property].length > 1) + manifest[property] = manifest[property].filter(item => item != from); + else + delete manifest[property]; + } + } + } + } else + throw new Error(`Invalid redirect rule "${property}.from" (must be string or null, not ${typeof from})`); + } + redirectObject(manifest, property, key, from, to) { + if (!from) { + if (property in manifest) { + manifest[property] = to ?? {}; + } + } else if ("string" === typeof from) { + if (property in manifest) { + this.redirectArray(manifest[property], key, from, to); + } + } else + throw new Error(`Invalid redirect rule "${property}.${key}" (must be object or null, not ${typeof from})`); + } resolvePrefix(value) { const colon = value.indexOf(":"); if (colon > 0) { @@ -2311,6 +2383,7 @@ export class Tool extends TOOL { this.manifests.forEach(manifest => this.mergeManifest(this.manifest, manifest)); this.mergeDependencies(this.manifests); + this.redirect(this.manifest); if (this.manifest.errors.length) { this.manifest.errors.forEach(error => { this.reportError(null, 0, error); }); From e1fc0ea71e9df435ed4d49f93eeafe13100f5dca Mon Sep 17 00:00:00 2001 From: cmidgley Date: Sat, 19 Oct 2024 08:20:11 -0400 Subject: [PATCH 5/5] Allow arrays for 'to' replacement --- tools/mcmanifest.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/mcmanifest.js b/tools/mcmanifest.js index c69983fc99..653a6dda50 100644 --- a/tools/mcmanifest.js +++ b/tools/mcmanifest.js @@ -2291,10 +2291,9 @@ export class Tool extends TOOL { let array = Array.isArray(manifest[property]) ? manifest[property] : [manifest[property]]; if (array.includes(from)) { if (to) { - if (Array.isArray(manifest[property]) && from) - manifest[property] = manifest[property].map(item => item == from ? to : item); - else - manifest[property] = to; + if (!Array.isArray(manifest[property])) + manifest[property] = [manifest[property]]; + manifest[property] = manifest[property].filter(item => item != from).concat(to); } else { if (Array.isArray(manifest[property]) && manifest[property].length > 1) manifest[property] = manifest[property].filter(item => item != from);