From ea0488645faebc653fbde2516ff6049d21a57dda Mon Sep 17 00:00:00 2001 From: Gynt Date: Wed, 14 May 2025 23:38:45 +0200 Subject: [PATCH 01/12] add cv2 as requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2849054..8547836 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ pymem dclimplode numpy build +opencv-python \ No newline at end of file From b6dca429f3214299166e8e2fca8dc8566f7ab2ee Mon Sep 17 00:00:00 2001 From: Gynt Date: Thu, 15 May 2025 00:06:08 +0200 Subject: [PATCH 02/12] feat(memory): terrain and height from image POC --- sourcehold/__main__.py | 3 + sourcehold/tool/argparsers/services.py | 19 ++++++ sourcehold/tool/memory/__init__.py | 0 sourcehold/tool/memory/map/__init__.py | 23 +++++++ sourcehold/tool/memory/map/common.py | 19 ++++++ sourcehold/tool/memory/map/height/__init__.py | 67 +++++++++++++++++++ .../tool/memory/map/terrain/__init__.py | 50 ++++++++++++++ sourcehold/tool/memory/map/terrain/colors.py | 18 +++++ sourcehold/tool/memory/map/terrain/logics.py | 18 +++++ 9 files changed, 217 insertions(+) create mode 100644 sourcehold/tool/memory/__init__.py create mode 100644 sourcehold/tool/memory/map/__init__.py create mode 100644 sourcehold/tool/memory/map/common.py create mode 100644 sourcehold/tool/memory/map/height/__init__.py create mode 100644 sourcehold/tool/memory/map/terrain/__init__.py create mode 100644 sourcehold/tool/memory/map/terrain/colors.py create mode 100644 sourcehold/tool/memory/map/terrain/logics.py diff --git a/sourcehold/__main__.py b/sourcehold/__main__.py index b2c5419..9af963f 100644 --- a/sourcehold/__main__.py +++ b/sourcehold/__main__.py @@ -10,6 +10,7 @@ from sourcehold.tool.argparsers.common import main_parser from sourcehold.tool.argparsers.services import services_parser, convert_parser +from sourcehold.tool.memory.map import memory_map from sourcehold.tool.modify.map import modify_map file_input_output = argparse.ArgumentParser(add_help=False) @@ -82,6 +83,8 @@ def main(): return if modify_map(args): return + if memory_map(args): + return if args.service == "aiv": if args.method == "file": diff --git a/sourcehold/tool/argparsers/services.py b/sourcehold/tool/argparsers/services.py index dc28726..3e6a508 100644 --- a/sourcehold/tool/argparsers/services.py +++ b/sourcehold/tool/argparsers/services.py @@ -1,3 +1,4 @@ +import argparse from .common import main_parser, file_input_file_output services_parser = main_parser.add_subparsers(title="service", dest="service", required=True) @@ -11,6 +12,24 @@ convert_aiv_parser.add_argument('--to-format', required=False, default='') memory_parser = services_parser.add_parser('memory') +memory_parser.add_argument('--game', choices=['SHC1.41-latin', "SHCE1.41-latin"], default="SHC1.41-latin") +memory_subparsers = memory_parser.add_subparsers(dest='type', required=True, title='type') + +memory_map_parser = memory_subparsers.add_parser('map') +memory_map_subparsers = memory_map_parser.add_subparsers(dest='action', required=True) + +memory_common = argparse.ArgumentParser(add_help=False) +memory_common.add_argument('what', choices=['terrain', 'height']) + +memory_map_get_parser = memory_map_subparsers.add_parser('get', parents=[memory_common]) +memory_map_get_parser.add_argument('--output', default='') +memory_map_get_parser.add_argument('--output-format', default='', choices=['png']) + +memory_map_set_parser = memory_map_subparsers.add_parser('set', parents=[memory_common]) +memory_map_set_parser.add_argument('--input', default='-') +memory_map_set_parser.add_argument('--input-format', default='', choices=['png']) + + modify_parser = services_parser.add_parser('modify') modify_subparser = modify_parser.add_subparsers(dest='type', required=True, title='type') diff --git a/sourcehold/tool/memory/__init__.py b/sourcehold/tool/memory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sourcehold/tool/memory/map/__init__.py b/sourcehold/tool/memory/map/__init__.py new file mode 100644 index 0000000..52aef44 --- /dev/null +++ b/sourcehold/tool/memory/map/__init__.py @@ -0,0 +1,23 @@ +import pathlib, sys + +from sourcehold.tool.convert.aiv.exports import to_json +from sourcehold.tool.convert.aiv.imports import from_json +from sourcehold.tool.memory.map.height import set_height +from sourcehold.tool.memory.map.terrain import set_terrain + + +def memory_map(args): + #' returns None in case of non applicable + if args.service != "memory": + return None + + if args.type != "map": + return None + + if set_height(args): + return True + + if set_terrain(args): + return True + + return True \ No newline at end of file diff --git a/sourcehold/tool/memory/map/common.py b/sourcehold/tool/memory/map/common.py new file mode 100644 index 0000000..3694605 --- /dev/null +++ b/sourcehold/tool/memory/map/common.py @@ -0,0 +1,19 @@ +import pathlib +from sourcehold.debugtools.memory.access import SHC, SHCE + +def get_process_handle(version): + if version == "SHC1.41-latin": + return SHC() + # if version == "SHCE1.41-latin": + # return SHCE() + raise NotImplementedError(f"process not implemented: {version}") + + +def validate_path(img_path): + if not img_path: + raise Exception(f"no input file specified") + if img_path == "-": + raise NotImplementedError(f"stdin input not yet implemented for this action. Specify a file path using --input") + + if not pathlib.Path(img_path).exists(): + raise Exception(f"file does not exist: {img_path}") \ No newline at end of file diff --git a/sourcehold/tool/memory/map/height/__init__.py b/sourcehold/tool/memory/map/height/__init__.py new file mode 100644 index 0000000..1655ee3 --- /dev/null +++ b/sourcehold/tool/memory/map/height/__init__.py @@ -0,0 +1,67 @@ + + + +# python -m pip install Pillow +import pathlib +import struct +from sourcehold.tool.memory.map.common import get_process_handle, validate_path +from sourcehold.world import create_selection_matrix +import cv2 as cv # type: ignore + +def get_image_data_grayscale(img_path): + img = cv.imread(img_path) + img = cv.cvtColor(img, cv.COLOR_BGR2GRAY) + return img + +selection = create_selection_matrix() + +# (Little endian) unsigned bytes +def get_raw_height(process): + return struct.unpack("<80400B", process.read_section('1045')) + +def set_raw_height(process, data): + bytes_data = struct.pack("<80400B", *data) + # ChangedLayer + process.write_bytes(0x01c5ad88, b'\x02' * 80400) # TODO: fix + # Logical terrain height layer: DefaultHeightLayer + process.write_section('1045', bytes_data) + # Visual height layer, I think also includes walls and towers: HeightLayer + process.write_section('1005', bytes_data) + # LogicLayer + process.write_section('1003', struct.pack("<80400I", *((v & 0xffffff7f) for v in struct.unpack("<80400I", process.read_section('1003'))))) + # Logic2Layer + process.write_section('1037', b'\x04' * 80400) + +# def post_process_raw_height(): +# # MiscDisplayLayer +# process.write_section('1007', struct.pack("<80400H", *(((v & 0xffdf) & 0xf83f) for v in struct.unpack("<80400H", process.read_section('1007'))))) +# # LogicLayer, what a hot mess, probably not all required +# process.write_section('1003', struct.pack("<80400I", *(((((v & 0x5f81c436) & 0xffffff7f) & 0xbfffbfff) | 32768) for v in struct.unpack("<80400I", process.read_section('1003'))))) +# # Logic2Layer +# process.write_section('1037', b'\x04' * 80400) +# # TODO: wall owner layer, and special logic2layer set to 4 or 8 depending on plateau +# # ChangedLayer +# process.write_bytes(0x01c5ad88, b'\x02' * 80400) + + +# post_process_raw_height() + + +def set_height(args): + #' returns None in case of non applicable + if args.what != 'height': + return None + + if args.action != "set": + return None + + img_path = args.input + validate_path(img_path) + + img = get_image_data_grayscale(img_path) + + process = get_process_handle(args.game) + + set_raw_height(process, img[selection].flat) + + return True \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/__init__.py b/sourcehold/tool/memory/map/terrain/__init__.py new file mode 100644 index 0000000..03b1f3c --- /dev/null +++ b/sourcehold/tool/memory/map/terrain/__init__.py @@ -0,0 +1,50 @@ +MAP_SIZE = 400 +TILE_COUNT = MAP_SIZE * ((MAP_SIZE // 2) + 1 ) +# python -m pip install Pillow +import struct, numpy +from sourcehold.tool.memory.map.common import get_process_handle, validate_path +from sourcehold.world import create_selection_matrix +import cv2 as cv # type: ignore + + +def get_image_data(img_path): + img = cv.imread(img_path) + #img = 255 - cv.cvtColor(img, cv.COLOR_BGR2GRAY) + return img + +selection = create_selection_matrix(size=MAP_SIZE) + +# (Little endian) unsigned bytes +def get_raw_logical(process): + return struct.unpack(f"<{TILE_COUNT}I", process.read_section('1003')) + +def set_raw_logical(process, data): + serialized = struct.pack(f"<{TILE_COUNT}I", *data) + # Logical terrain height layer + process.write_section('1003', serialized) + + +# 16 is used for the inaccessible parts of the map, including the outer border of the 800x800 space, and 32 is used for a border just within that + +def set_terrain(args): + #' returns None in case of non applicable + if args.what != 'terrain': + return None + + if args.action != "set": + return None + + img_path = args.input + validate_path(img_path) + img = get_image_data(img_path) + + process = get_process_handle(args.game) + + matrix = numpy.zeros((400, 400), dtype='uint32') + matrix[selection] = get_raw_logical(process) + + # matrix[img > 255//128] |= 0x20000 # boulder flag? + + set_raw_logical(process, matrix[selection].flat) + + return True \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/colors.py b/sourcehold/tool/memory/map/terrain/colors.py new file mode 100644 index 0000000..2586bf9 --- /dev/null +++ b/sourcehold/tool/memory/map/terrain/colors.py @@ -0,0 +1,18 @@ +monsterfish1 = { + 'swamp': "#475937", + 'pitch': '#314235', + 'stone': '#c3bdb4', + 'gravel': '#978f80', + 'rocks': '#675335', + 'iron': '#9e4f00', + 'ford': '#567c71', + 'river': '#427068', + 'ocean': '#1e4a44', + 'oasis': '#47540b', + 'thick_scrub': '#6a692b', + 'light_scrub': '#937e44', + 'earth_and_stones': '#7c7059', + 'earth': '#ae9467', + 'dunes': '#b79453', + 'beach': '#deb977', +} \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/logics.py b/sourcehold/tool/memory/map/terrain/logics.py new file mode 100644 index 0000000..56808aa --- /dev/null +++ b/sourcehold/tool/memory/map/terrain/logics.py @@ -0,0 +1,18 @@ +logic1 = { + 'swamp': 1 << 1, # TODO + 'pitch': '#314235', + 'stone': '#c3bdb4', + 'gravel': '#978f80', + 'rocks': '#675335', + 'iron': '#9e4f00', + 'ford': '#567c71', + 'river': '#427068', + 'ocean': '#1e4a44', + 'oasis': '#47540b', + 'thick_scrub': '#6a692b', + 'light_scrub': '#937e44', + 'earth_and_stones': '#7c7059', + 'earth': '#ae9467', + 'dunes': '#b79453', + 'beach': '#deb977', +} \ No newline at end of file From f3eb259bd372d960b8da2ebffbbc22da0aaf189b Mon Sep 17 00:00:00 2001 From: Gynt Date: Thu, 15 May 2025 00:10:50 +0200 Subject: [PATCH 03/12] CI: sync PR build and test strategy --- .github/workflows/python-pr.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-pr.yml b/.github/workflows/python-pr.yml index 64883c6..1a49d19 100644 --- a/.github/workflows/python-pr.yml +++ b/.github/workflows/python-pr.yml @@ -67,7 +67,7 @@ jobs: run: | python -m pip install pyinstaller pyinstaller --console --onefile ./sourcehold/__main__.py --name sourcehold - + - name: Archive packages uses: actions/upload-artifact@v4 with: @@ -83,9 +83,9 @@ jobs: matrix: # os: [ubuntu-latest, windows-latest, macOS-latest] os: [windows-latest] - platform: [x86, x64] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] - python-platform: [x86, x64] + platform: [x86] + python-version: ['3.8'] + python-platform: [x86] steps: - name: Download package uses: actions/download-artifact@v4.1.7 From 677efbf12e857a8c0dbae23bcac6176a634f938d Mon Sep 17 00:00:00 2001 From: Gynt Date: Thu, 15 May 2025 09:05:50 +0200 Subject: [PATCH 04/12] add cv2 as dependency to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6486704..631a27a 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.8', - install_requires=["pymem", "Pillow", "dclimplode", "numpy"], + install_requires=["pymem", "Pillow", "dclimplode", "numpy", "opencv-python"], test_suite="tests", entry_points={ 'console_scripts': ['sourcehold=sourcehold:entry_point'] From add751828ee9f2e89cead32d5bf8b5a3def189ce Mon Sep 17 00:00:00 2001 From: Gynt Date: Thu, 15 May 2025 09:18:18 +0200 Subject: [PATCH 05/12] add logic info --- sourcehold/tool/memory/map/terrain/logics.py | 45 +++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/sourcehold/tool/memory/map/terrain/logics.py b/sourcehold/tool/memory/map/terrain/logics.py index 56808aa..a513267 100644 --- a/sourcehold/tool/memory/map/terrain/logics.py +++ b/sourcehold/tool/memory/map/terrain/logics.py @@ -15,4 +15,47 @@ 'earth': '#ae9467', 'dunes': '#b79453', 'beach': '#deb977', -} \ No newline at end of file +} + + # NONE=0, + # SEA=1, + # STOCKPILE?=2, + # PLAIN1=4, + # PLAIN2=8 /* also for pitch ditch */, + # WALK_BORDER_RELATED1=16, + # WALK_BORDER_RELATED2=32, + # ROCKY=128, + # WALL_OR_GATEHOUSE=256, + # CRENEL=512, + # BUILDING=1024, + # STAIRS=2048, + # TREE=4096, + # TREE_VARIATION=8192, + # MOAT_DUG_OR_PLANNED=16384, + # MAX_HEIGHT_RELATED=32768, + # UNKNOWN_WALL_RELATED=65536, + # BOULDERS=131072, + # PEBBLES=262144, + # IRON=524288, + # RIVER_FOAM_RIPPLE=1048576, + # FORD=2097152, + # CRENEL_VARIATION?=4194304, + # FARM_WHEAT=16777216, + # FARM_HOP=33554432, + # FARM_APPLE=67108864, + # FARM_DAIRY=134217728, + # KEEP_NON_MANOR_HOUSE=268435456, + # MARSH=536870912, + # MOAT=1073741824, + # OIL=2147483648 + + # NONE=0, + # SCRUB=1, + # DIRT=2, + # MOAT_UNDUG=3, + # PLATEAU_MEDIUM=4, + # PLATEAU_HIGH=8, + # GRASS=16, + # BEACH=32, + # STONES_OR_DRIVEN_SAND?=64, + # THICK_SCRUB=128 \ No newline at end of file From 0ee70ec1f658143cdfeab276b2743c690b5ecd4b Mon Sep 17 00:00:00 2001 From: Gynt Date: Fri, 16 May 2025 15:05:41 +0200 Subject: [PATCH 06/12] unfinished logic values --- sourcehold/tool/memory/map/terrain/logics.py | 61 +++++++++++++++----- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/sourcehold/tool/memory/map/terrain/logics.py b/sourcehold/tool/memory/map/terrain/logics.py index a513267..1e84630 100644 --- a/sourcehold/tool/memory/map/terrain/logics.py +++ b/sourcehold/tool/memory/map/terrain/logics.py @@ -1,20 +1,49 @@ logic1 = { - 'swamp': 1 << 1, # TODO - 'pitch': '#314235', - 'stone': '#c3bdb4', - 'gravel': '#978f80', - 'rocks': '#675335', - 'iron': '#9e4f00', - 'ford': '#567c71', - 'river': '#427068', - 'ocean': '#1e4a44', - 'oasis': '#47540b', - 'thick_scrub': '#6a692b', - 'light_scrub': '#937e44', - 'earth_and_stones': '#7c7059', - 'earth': '#ae9467', - 'dunes': '#b79453', - 'beach': '#deb977', + 'none': 0, + 'ocean': 0x1, + 'plain1': 0x4, # 0x8000 + 'plain2': 0x8, # 0x8000 + 'swamp': 0x20000000, + 'moat_dug': 0x4000, + 'moat': 0x40000000, + 'pitch': 0x80000000, + # max_height_related = 0x8000, #TODO: not right + 'stone': 0x20000, + 'gravel': 0x40000, # TODO: what is this even? + 'rocks': 0x80, + 'iron': 0x80000, + 'ford': 0x200000, + 'river': 0x100000, + 'oasis': 0x8000, # special + 'thick_scrub': 0x8000, # special + 'light_scrub': 0x8000, # special + 'earth_and_stones': 0, # '#7c7059', + 'earth': 0, #'#ae9467', + 'dunes': 0x8000 # special '#b79453', + 'beach': 0x8000 # special '#deb977', +} + +logic2 = { + 'none': 0, + 'thick_scrub': 0x80, + 'stones_or_driven_sand': 0x40, # don't know + 'beach': 0x20, + 'grass': 0x10, + 'plateau_high': 0x8, + 'plateau_low': 0x4, + 'moat_undug': 0x3, + 'scrub': 0x1, + '' + # NONE=0, + # SCRUB=1, + # DIRT=2, + # MOAT_UNDUG=3, + # PLATEAU_MEDIUM=4, + # PLATEAU_HIGH=8, + # GRASS=16, + # BEACH=32, + # STONES_OR_DRIVEN_SAND?=64, + # THICK_SCRUB=128 } # NONE=0, From e8b2b5f87e394cd336eef085be8e04a03e7cd85e Mon Sep 17 00:00:00 2001 From: Gynt Date: Fri, 23 May 2025 22:29:14 +0200 Subject: [PATCH 07/12] feat(get_terrain): working color display --- sourcehold/tool/memory/map/__init__.py | 5 +- sourcehold/tool/memory/map/common.py | 2 +- sourcehold/tool/memory/map/height/__init__.py | 4 +- .../tool/memory/map/terrain/__init__.py | 70 +++++++++++++++++-- sourcehold/tool/memory/map/terrain/colors.py | 48 ++++++++++--- sourcehold/tool/memory/map/terrain/logics.py | 67 ++++++++++++------ 6 files changed, 153 insertions(+), 43 deletions(-) diff --git a/sourcehold/tool/memory/map/__init__.py b/sourcehold/tool/memory/map/__init__.py index 52aef44..a45c841 100644 --- a/sourcehold/tool/memory/map/__init__.py +++ b/sourcehold/tool/memory/map/__init__.py @@ -3,7 +3,7 @@ from sourcehold.tool.convert.aiv.exports import to_json from sourcehold.tool.convert.aiv.imports import from_json from sourcehold.tool.memory.map.height import set_height -from sourcehold.tool.memory.map.terrain import set_terrain +from sourcehold.tool.memory.map.terrain import get_terrain, set_terrain def memory_map(args): @@ -19,5 +19,8 @@ def memory_map(args): if set_terrain(args): return True + + if get_terrain(args): + return True return True \ No newline at end of file diff --git a/sourcehold/tool/memory/map/common.py b/sourcehold/tool/memory/map/common.py index 3694605..dbff0a5 100644 --- a/sourcehold/tool/memory/map/common.py +++ b/sourcehold/tool/memory/map/common.py @@ -9,7 +9,7 @@ def get_process_handle(version): raise NotImplementedError(f"process not implemented: {version}") -def validate_path(img_path): +def validate_input_path(img_path): if not img_path: raise Exception(f"no input file specified") if img_path == "-": diff --git a/sourcehold/tool/memory/map/height/__init__.py b/sourcehold/tool/memory/map/height/__init__.py index 1655ee3..2b66ecf 100644 --- a/sourcehold/tool/memory/map/height/__init__.py +++ b/sourcehold/tool/memory/map/height/__init__.py @@ -4,7 +4,7 @@ # python -m pip install Pillow import pathlib import struct -from sourcehold.tool.memory.map.common import get_process_handle, validate_path +from sourcehold.tool.memory.map.common import get_process_handle, validate_input_path from sourcehold.world import create_selection_matrix import cv2 as cv # type: ignore @@ -56,7 +56,7 @@ def set_height(args): return None img_path = args.input - validate_path(img_path) + validate_input_path(img_path) img = get_image_data_grayscale(img_path) diff --git a/sourcehold/tool/memory/map/terrain/__init__.py b/sourcehold/tool/memory/map/terrain/__init__.py index 03b1f3c..d165a4e 100644 --- a/sourcehold/tool/memory/map/terrain/__init__.py +++ b/sourcehold/tool/memory/map/terrain/__init__.py @@ -2,10 +2,11 @@ TILE_COUNT = MAP_SIZE * ((MAP_SIZE // 2) + 1 ) # python -m pip install Pillow import struct, numpy -from sourcehold.tool.memory.map.common import get_process_handle, validate_path +from sourcehold.tool.memory.map.common import get_process_handle, validate_input_path from sourcehold.world import create_selection_matrix import cv2 as cv # type: ignore - +from .logics import logic1, logic1_vk, logic2, logic2_vk +from .colors import monsterfish1_bgr, bgr_monsterfish1 def get_image_data(img_path): img = cv.imread(img_path) @@ -15,14 +16,22 @@ def get_image_data(img_path): selection = create_selection_matrix(size=MAP_SIZE) # (Little endian) unsigned bytes -def get_raw_logical(process): +def get_raw_logic1(process): return struct.unpack(f"<{TILE_COUNT}I", process.read_section('1003')) -def set_raw_logical(process, data): +def set_raw_logic1(process, data): serialized = struct.pack(f"<{TILE_COUNT}I", *data) # Logical terrain height layer process.write_section('1003', serialized) +def get_raw_logic2(process): + return struct.unpack(f"<{TILE_COUNT}B", process.read_section('1037')) + +def set_raw_logic2(process, data): + serialized = struct.pack(f"<{TILE_COUNT}B", *data) + # Logical terrain height layer + process.write_section('1037', serialized) + # 16 is used for the inaccessible parts of the map, including the outer border of the 800x800 space, and 32 is used for a border just within that @@ -35,16 +44,63 @@ def set_terrain(args): return None img_path = args.input - validate_path(img_path) + validate_input_path(img_path) img = get_image_data(img_path) process = get_process_handle(args.game) matrix = numpy.zeros((400, 400), dtype='uint32') - matrix[selection] = get_raw_logical(process) + matrix[selection] = get_raw_logic1(process) # matrix[img > 255//128] |= 0x20000 # boulder flag? - set_raw_logical(process, matrix[selection].flat) + set_raw_logic1(process, matrix[selection].flat) + + return True + +def get_terrain(args): + #' returns None in case of non applicable + if args.what != 'terrain': + return None + + if args.action != "get": + return None + + img_path = args.output + + process = get_process_handle(args.game) + + logic1matrix = numpy.zeros((400, 400), dtype='uint32') + logic1matrix[selection] = get_raw_logic1(process) + + logic2matrix = numpy.zeros((400, 400), dtype='uint8') + logic2matrix[selection] = get_raw_logic2(process) + + colorlogic1 = numpy.zeros((400,400,3), dtype='uint8') + colorlogic2 = numpy.zeros((400,400,3), dtype='uint8') + + for flag, name in logic1_vk.items(): + color = (0, 0, 0) + if name in monsterfish1_bgr: + color = monsterfish1_bgr[name] + else: + print(f"skipping color for: {name}") + colorlogic1[logic1matrix & flag != 0] = color + print(f"set '{name}' {colorlogic1[logic1matrix & flag != 0].sum()} times to color: {bgr_monsterfish1[color]}") + + for flag, name in logic2_vk.items(): + if name == 'none': + continue + color = (0, 0, 0) + where = logic1matrix == logic1['default_earth_or_texture'] + if name in monsterfish1_bgr: + color = monsterfish1_bgr[name] + else: + print(f"skipping color for: {name}") + colorlogic1[where & (logic2matrix == flag)] = color + print(f"set '{name}' {colorlogic1[logic2matrix == flag].sum()} times to color: {bgr_monsterfish1[color]}") + + cv.imshow('result', colorlogic1) + cv.waitKey(0) return True \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/colors.py b/sourcehold/tool/memory/map/terrain/colors.py index 2586bf9..647375d 100644 --- a/sourcehold/tool/memory/map/terrain/colors.py +++ b/sourcehold/tool/memory/map/terrain/colors.py @@ -1,18 +1,46 @@ -monsterfish1 = { - 'swamp': "#475937", - 'pitch': '#314235', - 'stone': '#c3bdb4', - 'gravel': '#978f80', +monsterfish1_hex = { + 'none': '#000000', + 'default_earth_or_texture': '#ae9467', + 'plateau_medium': '#ae9467', # todo: is this bug prone? + 'plateau_high': '#ae9467', # todo: is this bug prone? + + 'border': '#FF0000', + 'border_edge': '#DD0000', + 'plain1_and_farm': '#cccccc', + 'plain2_and_pitch': '#eeeeee', + 'marsh': "#475937", + 'oil': '#314235', + 'boulders': '#c3bdb4', + 'pebbles': '#978f80', + 'rocks': '#675335', + 'iron': '#9e4f00', 'ford': '#567c71', 'river': '#427068', + + 'moat_undug': '#0000FF', + 'moat_dug': '#0000FF', + 'moat': '#0000FF', 'ocean': '#1e4a44', - 'oasis': '#47540b', + 'oasis_grass': '#47540b', 'thick_scrub': '#6a692b', - 'light_scrub': '#937e44', + 'scrub': '#937e44', 'earth_and_stones': '#7c7059', - 'earth': '#ae9467', - 'dunes': '#b79453', + + 'driven_sand': '#b79453', 'beach': '#deb977', -} \ No newline at end of file + + +} + +def hex_to_bgr(string): + return (int(string[5:7], 16), int(string[3:5], 16), int(string[1:3], 16)) + +def hex_to_rgb(string): + return (int(string[1:3], 16), int(string[3:5], 16), int(string[5:7], 16)) + +monsterfish1_rgb = {key: hex_to_rgb(v) for key, v in monsterfish1_hex.items()} +monsterfish1_bgr = {key: hex_to_bgr(v) for key, v in monsterfish1_hex.items()} +rgb_monsterfish1 = {v: key for key, v in monsterfish1_rgb.items()} +bgr_monsterfish1 = {v: key for key, v in monsterfish1_bgr.items()} \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/logics.py b/sourcehold/tool/memory/map/terrain/logics.py index 1e84630..2bb5b65 100644 --- a/sourcehold/tool/memory/map/terrain/logics.py +++ b/sourcehold/tool/memory/map/terrain/logics.py @@ -1,39 +1,59 @@ logic1 = { 'none': 0, + 'default_earth_or_texture': 0x8000, #'#ae9467', + 'ocean': 0x1, - 'plain1': 0x4, # 0x8000 - 'plain2': 0x8, # 0x8000 - 'swamp': 0x20000000, - 'moat_dug': 0x4000, - 'moat': 0x40000000, - 'pitch': 0x80000000, - # max_height_related = 0x8000, #TODO: not right - 'stone': 0x20000, - 'gravel': 0x40000, # TODO: what is this even? + # 'stockpile': 0x2, + 'plain1_and_farm': 0x4, # 0x8000 + 'plain2_and_pitch': 0x8, # 0x8000 + + 'border': 0x10, + 'border_edge': 0x20, + 'rocks': 0x80, - 'iron': 0x80000, - 'ford': 0x200000, - 'river': 0x100000, - 'oasis': 0x8000, # special + + # 'wall_gatehouse_tower': 0x100, + # 'crenel': 0x200, + # 'building': 0x400, + # 'stairs': 0x800, + # 'tree': 0x1000, + # 'tree_variation': 0x2000, + 'moat_dug': 0x4000, + + 'oasis_grass': 0x8000, # special 'thick_scrub': 0x8000, # special 'light_scrub': 0x8000, # special - 'earth_and_stones': 0, # '#7c7059', - 'earth': 0, #'#ae9467', - 'dunes': 0x8000 # special '#b79453', - 'beach': 0x8000 # special '#deb977', + 'driven_sand': 0x8000, # special '#b79453', + 'beach': 0x8000, # special '#deb977', + # 'unknown_wall_related': 0x10000, + 'boulders': 0x20000, + 'pebbles': 0x40000, # TODO: what is this even? + 'iron': 0x80000, + 'river': 0x100000, + 'ford': 0x200000, + 'crenel_variation': 0x400000, + 'marsh': 0x20000000, + 'moat': 0x40000000, + 'oil': 0x80000000, + # max_height_related = 0x8000, #TODO: not right + } +logic1_vk = {v: k for k, v in logic1.items()} + +logic1_vk[0x8000] = 'default_earth_or_texture' + logic2 = { 'none': 0, 'thick_scrub': 0x80, - 'stones_or_driven_sand': 0x40, # don't know + 'driven_sand': 0x40, # don't know 'beach': 0x20, - 'grass': 0x10, + 'oasis_grass': 0x10, 'plateau_high': 0x8, - 'plateau_low': 0x4, + 'plateau_medium': 0x4, 'moat_undug': 0x3, + 'earth_and_stones': 0x2, 'scrub': 0x1, - '' # NONE=0, # SCRUB=1, # DIRT=2, @@ -46,6 +66,8 @@ # THICK_SCRUB=128 } +logic2_vk = {v: k for k, v in logic2.items()} + # NONE=0, # SEA=1, # STOCKPILE?=2, @@ -87,4 +109,5 @@ # GRASS=16, # BEACH=32, # STONES_OR_DRIVEN_SAND?=64, - # THICK_SCRUB=128 \ No newline at end of file + # THICK_SCRUB=128 + From 4b0bf33fcfaecf2e774cd4d6c33ab36937fcee4c Mon Sep 17 00:00:00 2001 From: Gynt Date: Fri, 23 May 2025 23:45:09 +0200 Subject: [PATCH 08/12] feat(get_height): working height and palette --- sourcehold/tool/argparsers/services.py | 3 +- sourcehold/tool/memory/map/__init__.py | 6 +- sourcehold/tool/memory/map/height/__init__.py | 39 ++++++++-- .../tool/memory/map/terrain/__init__.py | 77 +++++++++++++------ sourcehold/tool/memory/map/terrain/colors.py | 26 ++++++- sourcehold/tool/memory/map/terrain/logics.py | 5 +- 6 files changed, 122 insertions(+), 34 deletions(-) diff --git a/sourcehold/tool/argparsers/services.py b/sourcehold/tool/argparsers/services.py index 3e6a508..a51f7d8 100644 --- a/sourcehold/tool/argparsers/services.py +++ b/sourcehold/tool/argparsers/services.py @@ -20,10 +20,11 @@ memory_common = argparse.ArgumentParser(add_help=False) memory_common.add_argument('what', choices=['terrain', 'height']) +memory_common.add_argument('--palette', default='', required=False) memory_map_get_parser = memory_map_subparsers.add_parser('get', parents=[memory_common]) memory_map_get_parser.add_argument('--output', default='') -memory_map_get_parser.add_argument('--output-format', default='', choices=['png']) +memory_map_get_parser.add_argument('--output-format', default='png', choices=['png']) memory_map_set_parser = memory_map_subparsers.add_parser('set', parents=[memory_common]) memory_map_set_parser.add_argument('--input', default='-') diff --git a/sourcehold/tool/memory/map/__init__.py b/sourcehold/tool/memory/map/__init__.py index a45c841..b897597 100644 --- a/sourcehold/tool/memory/map/__init__.py +++ b/sourcehold/tool/memory/map/__init__.py @@ -2,7 +2,7 @@ from sourcehold.tool.convert.aiv.exports import to_json from sourcehold.tool.convert.aiv.imports import from_json -from sourcehold.tool.memory.map.height import set_height +from sourcehold.tool.memory.map.height import get_height, set_height from sourcehold.tool.memory.map.terrain import get_terrain, set_terrain @@ -17,6 +17,10 @@ def memory_map(args): if set_height(args): return True + + if get_height(args): + return True + if set_terrain(args): return True diff --git a/sourcehold/tool/memory/map/height/__init__.py b/sourcehold/tool/memory/map/height/__init__.py index 2b66ecf..a8f8b96 100644 --- a/sourcehold/tool/memory/map/height/__init__.py +++ b/sourcehold/tool/memory/map/height/__init__.py @@ -4,9 +4,11 @@ # python -m pip install Pillow import pathlib import struct +import numpy from sourcehold.tool.memory.map.common import get_process_handle, validate_input_path from sourcehold.world import create_selection_matrix import cv2 as cv # type: ignore +import sys def get_image_data_grayscale(img_path): img = cv.imread(img_path) @@ -22,15 +24,15 @@ def get_raw_height(process): def set_raw_height(process, data): bytes_data = struct.pack("<80400B", *data) # ChangedLayer - process.write_bytes(0x01c5ad88, b'\x02' * 80400) # TODO: fix + # process.write_bytes(0x01c5ad88, b'\x02' * 80400) # TODO: fix # Logical terrain height layer: DefaultHeightLayer process.write_section('1045', bytes_data) # Visual height layer, I think also includes walls and towers: HeightLayer process.write_section('1005', bytes_data) - # LogicLayer - process.write_section('1003', struct.pack("<80400I", *((v & 0xffffff7f) for v in struct.unpack("<80400I", process.read_section('1003'))))) - # Logic2Layer - process.write_section('1037', b'\x04' * 80400) + # # LogicLayer + # process.write_section('1003', struct.pack("<80400I", *((v & 0xffffff7f) for v in struct.unpack("<80400I", process.read_section('1003'))))) + # # Logic2Layer + # process.write_section('1037', b'\x04' * 80400) # def post_process_raw_height(): # # MiscDisplayLayer @@ -64,4 +66,31 @@ def set_height(args): set_raw_height(process, img[selection].flat) + return True + + + +def get_height(args): + #' returns None in case of non applicable + if args.what != 'height': + return None + + if args.action != "get": + return None + + img = numpy.zeros((400,400), dtype='uint8') + + process = get_process_handle(args.game) + + height = numpy.zeros((400, 400), dtype='uint8') + height[selection] = get_raw_height(process) + + img[selection] = height[selection] + + if not args.output: + print(args.output_format) + sys.stdout.buffer.write(cv.imencode(f".{args.output_format}", img)[1].tobytes()) + else: + cv.imwrite(args.output, img=img) + return True \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/__init__.py b/sourcehold/tool/memory/map/terrain/__init__.py index d165a4e..93c4c65 100644 --- a/sourcehold/tool/memory/map/terrain/__init__.py +++ b/sourcehold/tool/memory/map/terrain/__init__.py @@ -6,11 +6,11 @@ from sourcehold.world import create_selection_matrix import cv2 as cv # type: ignore from .logics import logic1, logic1_vk, logic2, logic2_vk -from .colors import monsterfish1_bgr, bgr_monsterfish1 +from .colors import DEFAULT_PALETTE, monsterfish1_bgr, bgr_monsterfish1 +import sys def get_image_data(img_path): img = cv.imread(img_path) - #img = 255 - cv.cvtColor(img, cv.COLOR_BGR2GRAY) return img selection = create_selection_matrix(size=MAP_SIZE) @@ -42,6 +42,11 @@ def set_terrain(args): if args.action != "set": return None + + if args.palette: + palette = colors.Palette(args.palette) + else: + palette = DEFAULT_PALETTE img_path = args.input validate_input_path(img_path) @@ -49,12 +54,20 @@ def set_terrain(args): process = get_process_handle(args.game) - matrix = numpy.zeros((400, 400), dtype='uint32') - matrix[selection] = get_raw_logic1(process) + logic1matrix = numpy.zeros((400, 400), dtype='uint32') + # logic1matrix[selection] = get_raw_logic1(process) + + logic2matrix = numpy.zeros((400, 400), dtype='uint8') + # logic2matrix[selection] = get_raw_logic2(process) - # matrix[img > 255//128] |= 0x20000 # boulder flag? + for color, name in palette.bgr_palette.items(): + where = (img == color).all(2) + logic1matrix[where] |= logic1[name] + if name in logic2: + logic2matrix[where] = logic2[name] - set_raw_logic1(process, matrix[selection].flat) + set_raw_logic1(process, logic1matrix[selection].flat) + set_raw_logic2(process, logic2matrix[selection].flat) return True @@ -66,8 +79,11 @@ def get_terrain(args): if args.action != "get": return None - img_path = args.output - + if args.palette: + palette = colors.Palette(args.palette) + else: + palette = DEFAULT_PALETTE + process = get_process_handle(args.game) logic1matrix = numpy.zeros((400, 400), dtype='uint32') @@ -76,31 +92,42 @@ def get_terrain(args): logic2matrix = numpy.zeros((400, 400), dtype='uint8') logic2matrix[selection] = get_raw_logic2(process) - colorlogic1 = numpy.zeros((400,400,3), dtype='uint8') - colorlogic2 = numpy.zeros((400,400,3), dtype='uint8') + img = numpy.zeros((400,400,3), dtype='uint8') + if args.debug: + print("logic1") for flag, name in logic1_vk.items(): color = (0, 0, 0) - if name in monsterfish1_bgr: - color = monsterfish1_bgr[name] + if name in palette.palette_bgr: + color = palette.palette_bgr[name] else: - print(f"skipping color for: {name}") - colorlogic1[logic1matrix & flag != 0] = color - print(f"set '{name}' {colorlogic1[logic1matrix & flag != 0].sum()} times to color: {bgr_monsterfish1[color]}") - + if args.debug: + print(f"skipping color for: {name}") + img[logic1matrix & flag != 0] = color + if args.debug: + print(f"set '{name}' {img[logic1matrix & flag != 0].sum()} times to color: {palette.bgr_palette[color]}") + + if args.debug: + print("logic2") for flag, name in logic2_vk.items(): if name == 'none': continue color = (0, 0, 0) - where = logic1matrix == logic1['default_earth_or_texture'] - if name in monsterfish1_bgr: - color = monsterfish1_bgr[name] + where = (logic1matrix & logic1['default_earth_or_texture']) != 0 + if name in palette.palette_bgr: + color = palette.palette_bgr[name] else: - print(f"skipping color for: {name}") - colorlogic1[where & (logic2matrix == flag)] = color - print(f"set '{name}' {colorlogic1[logic2matrix == flag].sum()} times to color: {bgr_monsterfish1[color]}") - - cv.imshow('result', colorlogic1) - cv.waitKey(0) + if args.debug: + print(f"skipping color for: {name}") + img[where & (logic2matrix == flag)] = color + if args.debug: + print(f"set '{name}' {img[where & (logic2matrix == flag)].sum()} times to color: {palette.bgr_palette[color]}") + + if not args.output: + if args.debug: + print(args.output_format) + sys.stdout.buffer.write(cv.imencode(f".{args.output_format}", img)[1].tobytes()) + else: + cv.imwrite(args.output, img=img) return True \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/colors.py b/sourcehold/tool/memory/map/terrain/colors.py index 647375d..caaf5db 100644 --- a/sourcehold/tool/memory/map/terrain/colors.py +++ b/sourcehold/tool/memory/map/terrain/colors.py @@ -1,3 +1,24 @@ +import pathlib +import json + +class Palette(object): + + def __init__(self, path: None = None, palette = None): + if not path and not palette: + raise Exception() + self.palette_hex = {} + if path: + data = pathlib.Path(path).read_text() + self.palette_hex = json.loads(data) + elif palette: + self.palette_hex = palette + else: + raise Exception() + self.palette_bgr = {key: hex_to_bgr(value) for key, value in self.palette_hex.items()} + self.bgr_palette = {hex_to_bgr(value): key for key, value in self.palette_hex.items()} + + + monsterfish1_hex = { 'none': '#000000', 'default_earth_or_texture': '#ae9467', @@ -43,4 +64,7 @@ def hex_to_rgb(string): monsterfish1_rgb = {key: hex_to_rgb(v) for key, v in monsterfish1_hex.items()} monsterfish1_bgr = {key: hex_to_bgr(v) for key, v in monsterfish1_hex.items()} rgb_monsterfish1 = {v: key for key, v in monsterfish1_rgb.items()} -bgr_monsterfish1 = {v: key for key, v in monsterfish1_bgr.items()} \ No newline at end of file +bgr_monsterfish1 = {v: key for key, v in monsterfish1_bgr.items()} + +MONSTERFISH1 = Palette(palette=monsterfish1_hex) +DEFAULT_PALETTE = MONSTERFISH1 \ No newline at end of file diff --git a/sourcehold/tool/memory/map/terrain/logics.py b/sourcehold/tool/memory/map/terrain/logics.py index 2bb5b65..68b33ec 100644 --- a/sourcehold/tool/memory/map/terrain/logics.py +++ b/sourcehold/tool/memory/map/terrain/logics.py @@ -22,9 +22,12 @@ 'oasis_grass': 0x8000, # special 'thick_scrub': 0x8000, # special - 'light_scrub': 0x8000, # special + 'scrub': 0x8000, # special 'driven_sand': 0x8000, # special '#b79453', 'beach': 0x8000, # special '#deb977', + 'plateau_high': 0x8000, + 'plateau_medium': 0x8000, + 'earth_and_stones': 0x8000, # 'unknown_wall_related': 0x10000, 'boulders': 0x20000, 'pebbles': 0x40000, # TODO: what is this even? From 4728247fe2bd8c3458b8315c067b16f452cdf486 Mon Sep 17 00:00:00 2001 From: Gynt Date: Fri, 23 May 2025 23:53:20 +0200 Subject: [PATCH 09/12] fix(set_terrain): make sure border edges are retained --- sourcehold/tool/memory/map/terrain/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sourcehold/tool/memory/map/terrain/__init__.py b/sourcehold/tool/memory/map/terrain/__init__.py index 93c4c65..b72e8eb 100644 --- a/sourcehold/tool/memory/map/terrain/__init__.py +++ b/sourcehold/tool/memory/map/terrain/__init__.py @@ -55,7 +55,11 @@ def set_terrain(args): process = get_process_handle(args.game) logic1matrix = numpy.zeros((400, 400), dtype='uint32') - # logic1matrix[selection] = get_raw_logic1(process) + borderlogic1matrix = numpy.zeros((400, 400), dtype='uint32') + borderlogic1matrix[selection] = get_raw_logic1(process) # We load it here to get the map borders only... + logic1matrix[borderlogic1matrix & logic1['border'] != 0] |= logic1['border'] + logic1matrix[borderlogic1matrix & logic1['border_edge'] != 0] |= logic1['border_edge'] + logic1matrix[borderlogic1matrix == 0] = 0 logic2matrix = numpy.zeros((400, 400), dtype='uint8') # logic2matrix[selection] = get_raw_logic2(process) From 0caedf05dd9011a844a447bd196be4c08cf42330 Mon Sep 17 00:00:00 2001 From: Gynt Date: Fri, 23 May 2025 23:55:29 +0200 Subject: [PATCH 10/12] fix(set_terrain): change colors.Palette into Palette --- sourcehold/tool/memory/map/terrain/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sourcehold/tool/memory/map/terrain/__init__.py b/sourcehold/tool/memory/map/terrain/__init__.py index b72e8eb..0c16520 100644 --- a/sourcehold/tool/memory/map/terrain/__init__.py +++ b/sourcehold/tool/memory/map/terrain/__init__.py @@ -6,7 +6,7 @@ from sourcehold.world import create_selection_matrix import cv2 as cv # type: ignore from .logics import logic1, logic1_vk, logic2, logic2_vk -from .colors import DEFAULT_PALETTE, monsterfish1_bgr, bgr_monsterfish1 +from .colors import DEFAULT_PALETTE, Palette import sys def get_image_data(img_path): @@ -44,7 +44,7 @@ def set_terrain(args): return None if args.palette: - palette = colors.Palette(args.palette) + palette = Palette(args.palette) else: palette = DEFAULT_PALETTE @@ -84,7 +84,7 @@ def get_terrain(args): return None if args.palette: - palette = colors.Palette(args.palette) + palette = Palette(args.palette) else: palette = DEFAULT_PALETTE From 83cd13cb9dde94efb9a8981a0a2da10c6901bf26 Mon Sep 17 00:00:00 2001 From: Gynt Date: Sat, 24 May 2025 00:19:40 +0200 Subject: [PATCH 11/12] fix(main): fixed relative import --- sourcehold/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sourcehold/__main__.py b/sourcehold/__main__.py index 9af963f..670736a 100644 --- a/sourcehold/__main__.py +++ b/sourcehold/__main__.py @@ -76,7 +76,7 @@ args = main_parser.parse_args() -from .tool.convert.aiv import convert_aiv +from sourcehold.tool.convert.aiv import convert_aiv def main(): if convert_aiv(args): From f41d03f69da3d9e1f4f31a2fb9ff109128494636 Mon Sep 17 00:00:00 2001 From: Gynt Date: Sat, 24 May 2025 00:21:25 +0200 Subject: [PATCH 12/12] CI: add onedir and onefile --- .github/workflows/python-pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-pr.yml b/.github/workflows/python-pr.yml index 1a49d19..17ee182 100644 --- a/.github/workflows/python-pr.yml +++ b/.github/workflows/python-pr.yml @@ -66,7 +66,8 @@ jobs: shell: bash run: | python -m pip install pyinstaller - pyinstaller --console --onefile ./sourcehold/__main__.py --name sourcehold + pyinstaller --console --onefile ./sourcehold/__main__.py --name sourcehold --distpath dist/sourcehold-onefile + pyinstaller --console --onedir ./sourcehold/__main__.py --name sourcehold --distpath dist/sourcehold-onedir - name: Archive packages uses: actions/upload-artifact@v4