From 78ca86deea5333a9fbe62eb2cce7e3b523d5e6d0 Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 22 Oct 2025 13:32:55 -0700 Subject: [PATCH 01/10] add upload script --- cellpack/bin/upload_to_client.py | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 cellpack/bin/upload_to_client.py diff --git a/cellpack/bin/upload_to_client.py b/cellpack/bin/upload_to_client.py new file mode 100644 index 00000000..93d94f3a --- /dev/null +++ b/cellpack/bin/upload_to_client.py @@ -0,0 +1,51 @@ +import json +import fire + +from cellpack.autopack.FirebaseHandler import FirebaseHandler +from cellpack.autopack.DBRecipeHandler import DBUploader +from cellpack.autopack.loaders.config_loader import ConfigLoader +from cellpack.autopack.loaders.recipe_loader import RecipeLoader +from cellpack.bin.upload import get_recipe_metadata + +def upload_to_client( + recipe: str, + config: str, + fields: str, + name: str +): + """ + Uploads recipe, config, and editable fields, read from specified + JSON files, to the database for client access + """ + db_handler = FirebaseHandler() + recipe_id = "" + config_id = "" + editable_fields_ids = [] + if FirebaseHandler._initialized: + db_handler = DBUploader(db_handler) + if recipe: + recipe_loader = RecipeLoader(recipe) + recipe_full_data = recipe_loader._read(resolve_inheritance=False) + recipe_meta_data = get_recipe_metadata(recipe_loader) + recipe_id = db_handler.upload_recipe(recipe_meta_data, recipe_full_data) + if config: + config_data = ConfigLoader(config).config + config_id = db_handler.upload_config(config_data, config) + if fields: + editable_fields_data = json.load(open(fields, "r")) + for field in editable_fields_data.get("editable_fields", []): + id, _ = db_handler.upload_data("editable_fields", field) + editable_fields_ids.append(id) + recipe_metadata = { + "name": name, + "recipe": recipe_id, + "config": config_id, + "editable_fields": editable_fields_ids, + } + db_handler.upload_data("client_recipes", recipe_metadata) + +def main(): + fire.Fire(upload_to_client) + +if __name__ == "__main__": + main() From 78b9b0a756443d7aa07d359ba63225fdfdb999a6 Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 22 Oct 2025 13:40:00 -0700 Subject: [PATCH 02/10] add example data and more documentation --- cellpack/bin/upload_to_client.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cellpack/bin/upload_to_client.py b/cellpack/bin/upload_to_client.py index 93d94f3a..9536defc 100644 --- a/cellpack/bin/upload_to_client.py +++ b/cellpack/bin/upload_to_client.py @@ -16,6 +16,15 @@ def upload_to_client( """ Uploads recipe, config, and editable fields, read from specified JSON files, to the database for client access + + :param recipe: string argument + path to local recipe file to upload to firebase + :param config: string argument + path to local config file to upload to firebase + :param fields: string argument + path to local editable fields file to upload to firebase + :param name: string argument + display name for recipe in client selection menu """ db_handler = FirebaseHandler() recipe_id = "" From a9b056bc0919bfacd7bfb3c3cb29f4c912fa996e Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 22 Oct 2025 13:42:16 -0700 Subject: [PATCH 03/10] point to correct collection --- cellpack/bin/upload_to_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cellpack/bin/upload_to_client.py b/cellpack/bin/upload_to_client.py index 9536defc..f778aa53 100644 --- a/cellpack/bin/upload_to_client.py +++ b/cellpack/bin/upload_to_client.py @@ -51,7 +51,9 @@ def upload_to_client( "config": config_id, "editable_fields": editable_fields_ids, } - db_handler.upload_data("client_recipes", recipe_metadata) + + # Upload the combined recipe metadata to example_packings collection for client + db_handler.upload_data("example_packings", recipe_metadata) def main(): fire.Fire(upload_to_client) From f5f7a6910a91aea4089ff861264dea6127c24c4c Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 22 Oct 2025 14:29:03 -0700 Subject: [PATCH 04/10] have server accept recipe as json object in body of request --- cellpack/autopack/__init__.py | 8 +++++++- cellpack/autopack/loaders/recipe_loader.py | 5 +++-- cellpack/bin/pack.py | 4 ++-- docker/server.py | 14 +++++++++----- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py index 2219525b..4bfd996e 100755 --- a/cellpack/autopack/__init__.py +++ b/cellpack/autopack/__init__.py @@ -411,8 +411,14 @@ def read_text_file(filename, destination="", cache="collisionTrees", force=None) def load_file( - filename, destination="", cache="geometries", force=None, use_docker=False + filename, destination="", cache="geometries", force=None, use_docker=False, recipe_obj=None ): + if recipe_obj is not None: + composition = DBRecipeLoader.remove_empty( + recipe_obj.get("composition", {}) + ) + recipe_obj["composition"] = composition + return recipe_obj, None, False if is_remote_path(filename): database_name, file_path = convert_db_shortname_to_url(filename) if database_name == DATABASE_IDS.GITHUB: diff --git a/cellpack/autopack/loaders/recipe_loader.py b/cellpack/autopack/loaders/recipe_loader.py index 84cd78ac..4b2930c8 100644 --- a/cellpack/autopack/loaders/recipe_loader.py +++ b/cellpack/autopack/loaders/recipe_loader.py @@ -30,7 +30,7 @@ class RecipeLoader(object): # TODO: add all default values here default_values = default_recipe_values.copy() - def __init__(self, input_file_path, save_converted_recipe=False, use_docker=False): + def __init__(self, input_file_path, save_converted_recipe=False, use_docker=False, recipe_obj=None): _, file_extension = os.path.splitext(input_file_path) self.current_version = CURRENT_VERSION self.file_path = input_file_path @@ -38,6 +38,7 @@ def __init__(self, input_file_path, save_converted_recipe=False, use_docker=Fals self.ingredient_list = [] self.compartment_list = [] self.save_converted_recipe = save_converted_recipe + self.recipe_obj = recipe_obj # set CURRENT_RECIPE_PATH appropriately for remote(firebase) vs local recipes if autopack.is_remote_path(self.file_path): @@ -169,7 +170,7 @@ def _migrate_version(self, old_recipe): def _read(self, resolve_inheritance=True, use_docker=False): new_values, database_name, is_unnested_firebase = autopack.load_file( - self.file_path, cache="recipes", use_docker=use_docker + self.file_path, cache="recipes", use_docker=use_docker, recipe_obj=self.recipe_obj ) if database_name == "firebase": if is_unnested_firebase: diff --git a/cellpack/bin/pack.py b/cellpack/bin/pack.py index 9db53937..fdbfede0 100644 --- a/cellpack/bin/pack.py +++ b/cellpack/bin/pack.py @@ -25,7 +25,7 @@ def pack( - recipe, config_path=None, analysis_config_path=None, docker=False, validate=True + recipe, config_path=None, analysis_config_path=None, docker=False, validate=True, recipe_str=None ): """ Initializes an autopack packing from the command line @@ -40,7 +40,7 @@ def pack( packing_config_data = ConfigLoader(config_path, docker).config recipe_loader = RecipeLoader( - recipe, packing_config_data["save_converted_recipe"], docker + recipe, packing_config_data["save_converted_recipe"], docker, recipe_str ) recipe_data = recipe_loader.recipe_data analysis_config_data = {} diff --git a/docker/server.py b/docker/server.py index 581e0151..6e625314 100644 --- a/docker/server.py +++ b/docker/server.py @@ -12,11 +12,11 @@ class CellpackServer: def __init__(self): self.packing_tasks = set() - async def run_packing(self, recipe, config, job_id): + async def run_packing(self, recipe, config, job_id, body=None): os.environ["AWS_BATCH_JOB_ID"] = job_id self.update_job_status(job_id, "RUNNING") try: - pack(recipe=recipe, config_path=config, docker=True) + pack(recipe=recipe, config_path=config, docker=True, recipe_str=body) except Exception as e: self.update_job_status(job_id, "FAILED", error_message=str(e)) @@ -37,8 +37,12 @@ async def health_check(self, request: web.Request) -> web.Response: return web.Response() async def pack_handler(self, request: web.Request) -> web.Response: - recipe = request.rel_url.query.get("recipe") - if recipe is None: + recipe = request.rel_url.query.get("recipe") or "" + if request.can_read_body: + body = await request.json() + else: + body = None + if not recipe and not body: raise web.HTTPBadRequest( "Pack requests must include recipe as a query param" ) @@ -46,7 +50,7 @@ async def pack_handler(self, request: web.Request) -> web.Response: job_id = str(uuid.uuid4()) # Initiate packing task to run in background - packing_task = asyncio.create_task(self.run_packing(recipe, config, job_id)) + packing_task = asyncio.create_task(self.run_packing(recipe, config, job_id, body)) # Keep track of task references to prevent them from being garbage # collected, then discard after task completion From f87915ac4c8ee4137333ee8ce1d721214446ad23 Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 22 Oct 2025 14:29:49 -0700 Subject: [PATCH 05/10] update documentation --- docker/Dockerfile.ecs | 3 ++- docs/DOCKER.md | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.ecs b/docker/Dockerfile.ecs index 7a303c02..2016c222 100644 --- a/docker/Dockerfile.ecs +++ b/docker/Dockerfile.ecs @@ -4,7 +4,8 @@ WORKDIR /cellpack COPY . /cellpack RUN python -m pip install --upgrade pip --root-user-action=ignore -RUN python -m pip install . -r requirements/linux/requirements.txt --root-user-action=ignore +RUN python -m pip install . +RUN python -m pip install aiohttp mdutils EXPOSE 80 diff --git a/docs/DOCKER.md b/docs/DOCKER.md index a1214a33..48e4b6dd 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -13,6 +13,6 @@ ## AWS ECS Docker Image 1. Build image, running `docker build -f docker/Dockerfile.ecs -t [CONTAINER-NAME] .` 2. Run packings in the container, running: `docker run -v ~/.aws:/root/.aws -p 80:80 [CONTAINER-NAME]` -3. Try hitting the test endpoint on the server, by navigating to `http://0.0.0.0:8443/hello` in your browser. -4. Try running a packing on the server, by hitting the `http://0.0.0.0:80/pack?recipe=firebase:recipes/one_sphere_v_1.0.0` in your browser. +3. Try hitting the test endpoint on the server, by navigating to `http://0.0.0.0:80/hello` in your browser. +4. Try running a packing on the server, by hitting the `http://0.0.0.0:80/start-packing?recipe=firebase:recipes/one_sphere_v_1.0.0` in your browser. 5. Verify that the packing result path was uploaded to the firebase results table, with the job id specified in the response from the request in step 4.The result simularium file can be found at the s3 path specified there. \ No newline at end of file From 1f2d2e345523914d135379c826bdc00e51fbe6c0 Mon Sep 17 00:00:00 2001 From: ascibisz Date: Wed, 22 Oct 2025 16:41:35 -0700 Subject: [PATCH 06/10] remove accidential dockerfile changes --- docker/Dockerfile.ecs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile.ecs b/docker/Dockerfile.ecs index 2016c222..5ed89b7b 100644 --- a/docker/Dockerfile.ecs +++ b/docker/Dockerfile.ecs @@ -4,8 +4,7 @@ WORKDIR /cellpack COPY . /cellpack RUN python -m pip install --upgrade pip --root-user-action=ignore -RUN python -m pip install . -RUN python -m pip install aiohttp mdutils +RUN python -m pip install . --root-user-action=ignore EXPOSE 80 From bd8ec42d33b005ec06316de1a386478eb4cd24f7 Mon Sep 17 00:00:00 2001 From: ascibisz Date: Thu, 23 Oct 2025 13:42:23 -0700 Subject: [PATCH 07/10] rename param json_recipe --- cellpack/autopack/__init__.py | 10 +++++----- cellpack/autopack/loaders/recipe_loader.py | 6 +++--- cellpack/bin/data-manifest.json | 10 ++++++++++ cellpack/bin/pack.py | 4 ++-- docker/server.py | 2 +- 5 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 cellpack/bin/data-manifest.json diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py index 4bfd996e..65d32078 100755 --- a/cellpack/autopack/__init__.py +++ b/cellpack/autopack/__init__.py @@ -411,14 +411,14 @@ def read_text_file(filename, destination="", cache="collisionTrees", force=None) def load_file( - filename, destination="", cache="geometries", force=None, use_docker=False, recipe_obj=None + filename, destination="", cache="geometries", force=None, use_docker=False, json_recipe=None ): - if recipe_obj is not None: + if json_recipe is not None: composition = DBRecipeLoader.remove_empty( - recipe_obj.get("composition", {}) + json_recipe.get("composition", {}) ) - recipe_obj["composition"] = composition - return recipe_obj, None, False + json_recipe["composition"] = composition + return json_recipe, None, False if is_remote_path(filename): database_name, file_path = convert_db_shortname_to_url(filename) if database_name == DATABASE_IDS.GITHUB: diff --git a/cellpack/autopack/loaders/recipe_loader.py b/cellpack/autopack/loaders/recipe_loader.py index 4b2930c8..23044443 100644 --- a/cellpack/autopack/loaders/recipe_loader.py +++ b/cellpack/autopack/loaders/recipe_loader.py @@ -30,7 +30,7 @@ class RecipeLoader(object): # TODO: add all default values here default_values = default_recipe_values.copy() - def __init__(self, input_file_path, save_converted_recipe=False, use_docker=False, recipe_obj=None): + def __init__(self, input_file_path, save_converted_recipe=False, use_docker=False, json_recipe=None): _, file_extension = os.path.splitext(input_file_path) self.current_version = CURRENT_VERSION self.file_path = input_file_path @@ -38,7 +38,7 @@ def __init__(self, input_file_path, save_converted_recipe=False, use_docker=Fals self.ingredient_list = [] self.compartment_list = [] self.save_converted_recipe = save_converted_recipe - self.recipe_obj = recipe_obj + self.json_recipe = json_recipe # set CURRENT_RECIPE_PATH appropriately for remote(firebase) vs local recipes if autopack.is_remote_path(self.file_path): @@ -170,7 +170,7 @@ def _migrate_version(self, old_recipe): def _read(self, resolve_inheritance=True, use_docker=False): new_values, database_name, is_unnested_firebase = autopack.load_file( - self.file_path, cache="recipes", use_docker=use_docker, recipe_obj=self.recipe_obj + self.file_path, cache="recipes", use_docker=use_docker, json_recipe=self.json_recipe ) if database_name == "firebase": if is_unnested_firebase: diff --git a/cellpack/bin/data-manifest.json b/cellpack/bin/data-manifest.json new file mode 100644 index 00000000..0a5e1f2d --- /dev/null +++ b/cellpack/bin/data-manifest.json @@ -0,0 +1,10 @@ +{ + "assay-dev": { + "data_source": "https://s3-us-west-2.amazonaws.com/file-download-service.allencell.org/assay-dev_2018-10-03.csv?versionId=XVdmE.6g1kk77c7jYA2Ge54eehTjY_AP", + "static_files": [] + }, + "test": { + "data_source": "https://cellpack-demo.s3.us-west-2.amazonaws.com/alli-test/test-manifest.csv", + "static_files": [] + } +} \ No newline at end of file diff --git a/cellpack/bin/pack.py b/cellpack/bin/pack.py index fdbfede0..dadf0f31 100644 --- a/cellpack/bin/pack.py +++ b/cellpack/bin/pack.py @@ -25,7 +25,7 @@ def pack( - recipe, config_path=None, analysis_config_path=None, docker=False, validate=True, recipe_str=None + recipe, config_path=None, analysis_config_path=None, docker=False, validate=True, json_recipe=None ): """ Initializes an autopack packing from the command line @@ -40,7 +40,7 @@ def pack( packing_config_data = ConfigLoader(config_path, docker).config recipe_loader = RecipeLoader( - recipe, packing_config_data["save_converted_recipe"], docker, recipe_str + recipe, packing_config_data["save_converted_recipe"], docker, json_recipe ) recipe_data = recipe_loader.recipe_data analysis_config_data = {} diff --git a/docker/server.py b/docker/server.py index 6e625314..ce6da7ee 100644 --- a/docker/server.py +++ b/docker/server.py @@ -16,7 +16,7 @@ async def run_packing(self, recipe, config, job_id, body=None): os.environ["AWS_BATCH_JOB_ID"] = job_id self.update_job_status(job_id, "RUNNING") try: - pack(recipe=recipe, config_path=config, docker=True, recipe_str=body) + pack(recipe=recipe, config_path=config, docker=True, json_recipe=body) except Exception as e: self.update_job_status(job_id, "FAILED", error_message=str(e)) From 358158eaa5aae8641d8c4c14574a03dbe1a3cb65 Mon Sep 17 00:00:00 2001 From: ascibisz Date: Fri, 9 Jan 2026 11:42:35 -0800 Subject: [PATCH 08/10] remove file that shouldn't be in this PR --- cellpack/bin/upload_to_client.py | 62 -------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 cellpack/bin/upload_to_client.py diff --git a/cellpack/bin/upload_to_client.py b/cellpack/bin/upload_to_client.py deleted file mode 100644 index f778aa53..00000000 --- a/cellpack/bin/upload_to_client.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -import fire - -from cellpack.autopack.FirebaseHandler import FirebaseHandler -from cellpack.autopack.DBRecipeHandler import DBUploader -from cellpack.autopack.loaders.config_loader import ConfigLoader -from cellpack.autopack.loaders.recipe_loader import RecipeLoader -from cellpack.bin.upload import get_recipe_metadata - -def upload_to_client( - recipe: str, - config: str, - fields: str, - name: str -): - """ - Uploads recipe, config, and editable fields, read from specified - JSON files, to the database for client access - - :param recipe: string argument - path to local recipe file to upload to firebase - :param config: string argument - path to local config file to upload to firebase - :param fields: string argument - path to local editable fields file to upload to firebase - :param name: string argument - display name for recipe in client selection menu - """ - db_handler = FirebaseHandler() - recipe_id = "" - config_id = "" - editable_fields_ids = [] - if FirebaseHandler._initialized: - db_handler = DBUploader(db_handler) - if recipe: - recipe_loader = RecipeLoader(recipe) - recipe_full_data = recipe_loader._read(resolve_inheritance=False) - recipe_meta_data = get_recipe_metadata(recipe_loader) - recipe_id = db_handler.upload_recipe(recipe_meta_data, recipe_full_data) - if config: - config_data = ConfigLoader(config).config - config_id = db_handler.upload_config(config_data, config) - if fields: - editable_fields_data = json.load(open(fields, "r")) - for field in editable_fields_data.get("editable_fields", []): - id, _ = db_handler.upload_data("editable_fields", field) - editable_fields_ids.append(id) - recipe_metadata = { - "name": name, - "recipe": recipe_id, - "config": config_id, - "editable_fields": editable_fields_ids, - } - - # Upload the combined recipe metadata to example_packings collection for client - db_handler.upload_data("example_packings", recipe_metadata) - -def main(): - fire.Fire(upload_to_client) - -if __name__ == "__main__": - main() From f0beaa1fc28cfb0242bfc428e8d7879716f3d7bd Mon Sep 17 00:00:00 2001 From: ascibisz Date: Fri, 9 Jan 2026 11:43:04 -0800 Subject: [PATCH 09/10] remove accidential file --- cellpack/bin/data-manifest.json | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 cellpack/bin/data-manifest.json diff --git a/cellpack/bin/data-manifest.json b/cellpack/bin/data-manifest.json deleted file mode 100644 index 0a5e1f2d..00000000 --- a/cellpack/bin/data-manifest.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "assay-dev": { - "data_source": "https://s3-us-west-2.amazonaws.com/file-download-service.allencell.org/assay-dev_2018-10-03.csv?versionId=XVdmE.6g1kk77c7jYA2Ge54eehTjY_AP", - "static_files": [] - }, - "test": { - "data_source": "https://cellpack-demo.s3.us-west-2.amazonaws.com/alli-test/test-manifest.csv", - "static_files": [] - } -} \ No newline at end of file From a54ffa1d8393f89d2afb2926864bd1381e7870f2 Mon Sep 17 00:00:00 2001 From: ascibisz Date: Fri, 9 Jan 2026 11:52:19 -0800 Subject: [PATCH 10/10] lint fixes --- cellpack/autopack/__init__.py | 11 +++++++---- cellpack/autopack/loaders/recipe_loader.py | 13 +++++++++++-- cellpack/bin/pack.py | 7 ++++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cellpack/autopack/__init__.py b/cellpack/autopack/__init__.py index 65d32078..ce06494d 100755 --- a/cellpack/autopack/__init__.py +++ b/cellpack/autopack/__init__.py @@ -411,12 +411,15 @@ def read_text_file(filename, destination="", cache="collisionTrees", force=None) def load_file( - filename, destination="", cache="geometries", force=None, use_docker=False, json_recipe=None + filename, + destination="", + cache="geometries", + force=None, + use_docker=False, + json_recipe=None, ): if json_recipe is not None: - composition = DBRecipeLoader.remove_empty( - json_recipe.get("composition", {}) - ) + composition = DBRecipeLoader.remove_empty(json_recipe.get("composition", {})) json_recipe["composition"] = composition return json_recipe, None, False if is_remote_path(filename): diff --git a/cellpack/autopack/loaders/recipe_loader.py b/cellpack/autopack/loaders/recipe_loader.py index 23044443..cf87a10c 100644 --- a/cellpack/autopack/loaders/recipe_loader.py +++ b/cellpack/autopack/loaders/recipe_loader.py @@ -30,7 +30,13 @@ class RecipeLoader(object): # TODO: add all default values here default_values = default_recipe_values.copy() - def __init__(self, input_file_path, save_converted_recipe=False, use_docker=False, json_recipe=None): + def __init__( + self, + input_file_path, + save_converted_recipe=False, + use_docker=False, + json_recipe=None, + ): _, file_extension = os.path.splitext(input_file_path) self.current_version = CURRENT_VERSION self.file_path = input_file_path @@ -170,7 +176,10 @@ def _migrate_version(self, old_recipe): def _read(self, resolve_inheritance=True, use_docker=False): new_values, database_name, is_unnested_firebase = autopack.load_file( - self.file_path, cache="recipes", use_docker=use_docker, json_recipe=self.json_recipe + self.file_path, + cache="recipes", + use_docker=use_docker, + json_recipe=self.json_recipe, ) if database_name == "firebase": if is_unnested_firebase: diff --git a/cellpack/bin/pack.py b/cellpack/bin/pack.py index dadf0f31..99186c9d 100644 --- a/cellpack/bin/pack.py +++ b/cellpack/bin/pack.py @@ -25,7 +25,12 @@ def pack( - recipe, config_path=None, analysis_config_path=None, docker=False, validate=True, json_recipe=None + recipe, + config_path=None, + analysis_config_path=None, + docker=False, + validate=True, + json_recipe=None, ): """ Initializes an autopack packing from the command line