diff --git a/docs/Configuration.md b/docs/Configuration.md index 19e4009b..988c17b0 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -753,3 +753,85 @@ use_legacy_include_asm: True #### Default `False` + +# Project Organization Configuration + +## Inheritance + +For projects with several similar overlays, splat supports configuration inheritance. Shared options and segments can be +defined in a configuration file and then extended in another configuration file using the top level `parent` key. +Parents are resolved recursively, allowing projects to be organized using a flexible configuration tree. + +Using the parent key is similar to specifying multiple configuration files at the command line when running `split`, but +can be modeled as configuration rather than in the build system. + +#### Usage +psx.yaml: +```yaml +options: + platform: psx + base_path: .. + compiler: GCC + symbol_addrs_path: + - config/symbols.txt + asm_jtbl_label_macro: jlabel + extensions_path: tools/splat_ext + section_order: + - ".data" + - ".rodata" + - ".text" + - ".bss" + - ".sbss" +``` +overlay.yaml: +```yaml +parent: psx.yaml +options: + basename: overlay + target_path: disks/OVERLAY.BIN + asm_path: asm/overlay + asset_path: assets/overlay + src_path: src/overlay + ld_script_path: build/overlay.ld + symbol_addrs_path: + - config/symbols.overlay.txt +segments: + - name: overlay + type: code + start: 0x00000000 + vram: 0x80000000 + align: 4 + subalign: 4 +``` + +This produces a merged config equivalent to: +```yaml +options: + platform: psx + base_path: .. + compiler: GCC + symbol_addrs_path: + - config/symbols.txt + - config/symbols.overlay.txt + asm_jtbl_label_macro: jlabel + extensions_path: tools/splat_ext + section_order: + - ".data" + - ".rodata" + - ".text" + - ".bss" + - ".sbss" + basename: overlay + target_path: disks/OVERLAY.BIN + asm_path: asm/overlay + asset_path: assets/overlay + src_path: src/overlay + ld_script_path: build/overlay.ld +segments: + - name: overlay + type: code + start: 0x00000000 + vram: 0x80000000 + align: 4 + subalign: 4 +``` diff --git a/src/splat/scripts/split.py b/src/splat/scripts/split.py index 33f7388c..9825fde3 100644 --- a/src/splat/scripts/split.py +++ b/src/splat/scripts/split.py @@ -8,11 +8,7 @@ from .. import __package_name__, __version__ from ..disassembler import disassembler_instance -from ..util import cache_handler, progress_bar, vram_classes, statistics - -# This unused import makes the yaml library faster. don't remove -import pylibyaml # pyright: ignore -import yaml +from ..util import cache_handler, conf, progress_bar, vram_classes, statistics from colorama import Fore, Style from intervaltree import Interval, IntervalTree @@ -142,34 +138,6 @@ def assign_symbols_to_segments(): seg.add_symbol(symbol) -def merge_configs(main_config, additional_config): - # Merge rules are simple - # For each key in the dictionary - # - If list then append to list - # - If a dictionary then repeat merge on sub dictionary entries - # - Else assume string or number and replace entry - - for curkey in additional_config: - if curkey not in main_config: - main_config[curkey] = additional_config[curkey] - elif type(main_config[curkey]) != type(additional_config[curkey]): - log.error(f"Type for key {curkey} in configs does not match") - else: - # keys exist and match, see if a list to append - if type(main_config[curkey]) == list: - main_config[curkey] += additional_config[curkey] - elif type(main_config[curkey]) == dict: - # need to merge sub areas - main_config[curkey] = merge_configs( - main_config[curkey], additional_config[curkey] - ) - else: - # not a list or dictionary, must be a number or string, overwrite - main_config[curkey] = additional_config[curkey] - - return main_config - - def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str: s = seg.name.strip() if len(s) > limit: @@ -203,25 +171,6 @@ def calc_segment_dependences( return vram_class_to_follows_segments -def initialize_config( - config_path: List[str], - modes: Optional[List[str]], - verbose: bool, - disassemble_all: bool = False, -) -> Dict[str, Any]: - config: Dict[str, Any] = {} - for entry in config_path: - with open(entry) as f: - additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader) - config = merge_configs(config, additional_config) - - vram_classes.initialize(config.get("vram_classes")) - - options.initialize(config, config_path, modes, verbose, disassemble_all) - - return config - - def read_target_binary() -> bytes: rom_bytes = options.opts.target_path.read_bytes() @@ -500,13 +449,14 @@ def main( skip_version_check: bool = False, stdout_only: bool = False, disassemble_all: bool = False, + include_path: List[Path] = [], ): if stdout_only: progress_bar.out_file = sys.stdout # Load config global config - config = initialize_config(config_path, modes, verbose, disassemble_all) + config = conf.initialize(config_path, include_path, modes, verbose, disassemble_all) disassembler_instance.create_disassembler_instance(skip_version_check, __version__) @@ -585,6 +535,13 @@ def add_arguments_to_parser(parser: argparse.ArgumentParser): help="Disasemble matched functions and migrated data", action="store_true", ) + parser.add_argument( + "-I", + "--include-directory", + help="Add the directory to the list of search directories when including other config", + action="append", + type=Path, + ) def process_arguments(args: argparse.Namespace): @@ -596,6 +553,7 @@ def process_arguments(args: argparse.Namespace): args.skip_version_check, args.stdout_only, args.disassemble_all, + args.include_directory, ) diff --git a/src/splat/segtypes/segment.py b/src/splat/segtypes/segment.py index 1c9e407c..cc923312 100644 --- a/src/splat/segtypes/segment.py +++ b/src/splat/segtypes/segment.py @@ -255,7 +255,7 @@ def parse_ld_align_segment_start(yaml: Union[dict, list]) -> Optional[int]: @staticmethod def parse_suggestion_rodata_section_start( - yaml: Union[dict, list] + yaml: Union[dict, list], ) -> Optional[bool]: if isinstance(yaml, dict): suggestion_rodata_section_start = yaml.get( diff --git a/src/splat/util/conf.py b/src/splat/util/conf.py new file mode 100644 index 00000000..fedb5371 --- /dev/null +++ b/src/splat/util/conf.py @@ -0,0 +1,110 @@ +""" +This module is used to load splat configuration from a YAML file. + +A config dict can be loaded using `initialize`. + + config = conf.initialize("path/to/splat.yaml") +""" + +from typing import Any, Dict, List, Optional, Set, Tuple, Union +from pathlib import Path + +# This unused import makes the yaml library faster. don't remove +import pylibyaml # pyright: ignore +import yaml + +import sys + +from . import log, options, vram_classes + + +def _merge_configs(main_config, additional_config): + # Merge rules are simple + # For each key in the dictionary + # - If list then append to list + # - If a dictionary then repeat merge on sub dictionary entries + # - Else assume string or number and replace entry + + for curkey in additional_config: + if curkey not in main_config: + main_config[curkey] = additional_config[curkey] + elif type(main_config[curkey]) != type(additional_config[curkey]): + log.error(f"Type for key {curkey} in configs does not match") + else: + # keys exist and match, see if a list to append + if type(main_config[curkey]) == list: + main_config[curkey] += additional_config[curkey] + elif type(main_config[curkey]) == dict: + # need to merge sub areas + main_config[curkey] = _merge_configs( + main_config[curkey], additional_config[curkey] + ) + else: + # not a list or dictionary, must be a number or string, overwrite + main_config[curkey] = additional_config[curkey] + + return main_config + + +def _resolve_path(base: Path, rel: Path, include_paths: List[Path]) -> Path: + if (base / rel).exists(): + return base / rel + + for path in include_paths: + candidate = path / rel + if candidate.exists(): + return candidate + log.error(f'"{rel}" not found') + + +def _load_config(config_path: Path, include_path: List[Path]) -> Dict[str, Any]: + base_path = Path(config_path).parent + with open(config_path) as f: + config = yaml.load(f.read(), Loader=yaml.SafeLoader) + if "parent" in config: + parent_path = _resolve_path(base_path, Path(config["parent"]), include_path) + parent = _load_config(parent_path, include_path) + config = _merge_configs(parent, config) + del config["parent"] + + return config + + +def initialize( + config_path: List[str], + include_path: List[Path] = [], + modes: Optional[List[str]] = None, + verbose: bool = False, + disassemble_all: bool = False, +) -> Dict[str, Any]: + """ + Returns a `dict` with resolved splat config. + + Multiple configuration files can be passed in ``config_path`` with each + subsequent file merged into the previous. `parent` keys are resolved + prior to merging multiple files. + + `include_path` can include any additional paths which should be searched for relative parent config files. Paths are + relative to the file being evaluated (i.e. a child config file). + + `modes` specifies which modes are active (all, code, img, gfx, vtx, etc.). The default is all. + + `verbose` may be used to determine whether or not to display additional output. + + `disassemble_all` determines whether functions which are already compiled will be disassembled. + + After being merged, static validation is done on the configuration. + + The returned `dict` represents the merged and validated YAML. + """ + + config: Dict[str, Any] = {} + for entry in config_path: + additional_config = _load_config(Path(entry), include_path) + config = _merge_configs(config, additional_config) + + vram_classes.initialize(config.get("vram_classes")) + + options.initialize(config, config_path, modes, verbose, disassemble_all) + + return config diff --git a/test/basic_app/expected/.splache b/test/basic_app/expected/.splache index 4bf620c1..c5fdf079 100644 Binary files a/test/basic_app/expected/.splache and b/test/basic_app/expected/.splache differ diff --git a/test/basic_app/n64.yaml b/test/basic_app/n64.yaml new file mode 100644 index 00000000..3f45ef36 --- /dev/null +++ b/test/basic_app/n64.yaml @@ -0,0 +1,17 @@ +options: + platform: n64 + compiler: GCC + base_path: . + build_path: build + asm_path: split/asm + src_path: split/src + cache_path: split/.splache + asset_path: split/assets + compiler: GCC + symbol_addrs_path: + - config/symbols.txt + o_as_suffix: True +segments: + - name: header + type: header + start: 0x00 diff --git a/test/basic_app/splat.yaml b/test/basic_app/splat.yaml index dcb164dd..131f0c8d 100644 --- a/test/basic_app/splat.yaml +++ b/test/basic_app/splat.yaml @@ -1,24 +1,14 @@ +parent: n64.yaml options: - platform: n64 - compiler: GCC basename: basic_app - base_path: . - build_path: build target_path: build/basic_app.bin - asm_path: split/asm - src_path: split/src ld_script_path: split/basic_app.ld - cache_path: split/.splache symbol_addrs_path: split/generated.symbols.txt undefined_funcs_auto_path: split/undefined_funcs_auto.txt undefined_syms_auto_path: split/undefined_syms_auto.txt - asset_path: split/assets - compiler: GCC - o_as_suffix: True + symbol_addrs_path: + - config/symbols.splat.txt segments: - - name: header - type: header - start: 0x00 - name: dummy_ipl3 type: code start: 0x40