From 40325d6796f17712d3c2dbbdd82deccdd1954120 Mon Sep 17 00:00:00 2001 From: Ethaniel Billon Date: Mon, 19 May 2025 12:30:16 +0200 Subject: [PATCH 1/4] fix download outputs from vip --- src/vip_client/classes/VipSession.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vip_client/classes/VipSession.py b/src/vip_client/classes/VipSession.py index 3c161bc..01da593 100644 --- a/src/vip_client/classes/VipSession.py +++ b/src/vip_client/classes/VipSession.py @@ -974,7 +974,7 @@ def _init_download(self, workflow) -> dict: # Update the file metadata files_to_download[file].update() # Make the parent directory (if needed) - self._mkdirs(local_path.parent, location="local") + self._mkdirs(local_path.parent, location="local", exist_ok=True) # Return the list of files to download return files_to_download # ------------------------------------------------ From 52037eab4cfced82bcf54085c707f0f538f80354 Mon Sep 17 00:00:00 2001 From: Ethaniel Billon Date: Mon, 19 May 2025 12:30:39 +0200 Subject: [PATCH 2/4] update & fix tutorials --- examples/tutorials/demo-vipgirder.ipynb | 256 +++++++++++++++++ examples/tutorials/demo-vipsession.ipynb | 17 +- examples/tutorials/example-VipGirder.ipynb | 210 -------------- examples/tutorials/exemple_VipCI.ipynb | 303 --------------------- 4 files changed, 262 insertions(+), 524 deletions(-) create mode 100644 examples/tutorials/demo-vipgirder.ipynb delete mode 100644 examples/tutorials/example-VipGirder.ipynb delete mode 100644 examples/tutorials/exemple_VipCI.ipynb diff --git a/examples/tutorials/demo-vipgirder.ipynb b/examples/tutorials/demo-vipgirder.ipynb new file mode 100644 index 0000000..5845008 --- /dev/null +++ b/examples/tutorials/demo-vipgirder.ipynb @@ -0,0 +1,256 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e4b2f485-cb8b-4e4b-b21c-c381760fc914", + "metadata": {}, + "outputs": [], + "source": [ + "# for developement\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "id": "096fc6eb", + "metadata": {}, + "source": [ + "## Use VIP with a Girder server\n", + "Before reading this tutorial we recommand you to see **demo-vipsession** to understand some basis. " + ] + }, + { + "cell_type": "markdown", + "id": "eec57a9a", + "metadata": {}, + "source": [ + "### Connection with VIP & Girder" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2b9532d", + "metadata": {}, + "outputs": [], + "source": [ + "# Import the class\n", + "from vip_client import VipGirder\n", + "\n", + "VipGirder.init(\n", + " vip_key=\"VIP_API_KEY\",\n", + " girder_key=\"GIRDER_API_KEY\")" + ] + }, + { + "cell_type": "markdown", + "id": "8d4a4093", + "metadata": {}, + "source": [ + "### Configuration variables & pipeline info" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f72d24fb-f9f4-45b2-b7bc-0533216932ea", + "metadata": {}, + "outputs": [], + "source": [ + "# General configuration\n", + "pipeline_id = \"CQUEST/0.6\"\n", + "session_name = \"test_notebook_girder\"\n", + "\n", + "input_settings = {\n", + " \"zipped_folder\": \"/collection/ReproVIPSpectro/data/quest/basis.zip\",\n", + " \"data_file\": \"/collection/ReproVIPSpectro/data/quest/signals\",\n", + " \"parameter_file\": \"/collection/ReproVIPSpectro/data/quest/parameters/quest_param_117T_B.txt\",\n", + "}\n", + "\n", + "# N.B.: Here \"data_file\" leads to a folder, the other arguments lead to single files." + ] + }, + { + "cell_type": "markdown", + "id": "d06461ab", + "metadata": {}, + "source": [ + "In `input_settings`, the input files are pointed by Girder paths, usually in format: \"/collection/[collection_name]/[path_to_folder]\". \n", + "- if the path leads to a file, the same file be passed to VIP;\n", + "- If the path leads to a Girder item, all files in this item will be passed to VIP;\n", + "- If the path leads to a Girder folder, all files in this folder will be passed to VIP." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8bc5ad57", + "metadata": {}, + "outputs": [], + "source": [ + "VipGirder.show_pipeline(pipeline_id)" + ] + }, + { + "cell_type": "markdown", + "id": "a6df1cdd", + "metadata": {}, + "source": [ + "### Choose output location" + ] + }, + { + "cell_type": "markdown", + "id": "25704920", + "metadata": {}, + "source": [ + "When using girder, you can choose where the outputs will be located. For this tutorial run the cell corresponding to the location that you want!" + ] + }, + { + "cell_type": "markdown", + "id": "97d7cf93-2ab0-4d07-81b2-654b38480f9a", + "metadata": {}, + "source": [ + "##### Output on **girder** (default)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cdaa53b9-eaec-44d7-a342-7b41191fcb4a", + "metadata": {}, + "outputs": [], + "source": [ + "output_dir=\"/collection/ReproVIPSpectro/test/vip_outputs\"\n", + "\n", + "session = VipGirder(\n", + " session_name=session_name, \n", + " pipeline_id=pipeline_id, \n", + " input_settings=input_settings, \n", + " output_dir=output_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "1aec7d5d-05e1-4baf-af30-2e9b42e791f1", + "metadata": {}, + "source": [ + "##### Output on **VIP**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25da2f62-e044-4ffd-855f-74881ad5d770", + "metadata": {}, + "outputs": [], + "source": [ + "session = VipGirder(\n", + " session_name=session_name, \n", + " pipeline_id=pipeline_id, \n", + " input_settings=input_settings, \n", + " output_location=\"vip\")" + ] + }, + { + "cell_type": "markdown", + "id": "7c726378-93b0-443d-9531-89b42735142d", + "metadata": {}, + "source": [ + "##### Output on **local computer**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bff17ffe-bebc-419d-b68d-ed3b0d2f4cf7", + "metadata": {}, + "outputs": [], + "source": [ + "session = VipGirder(\n", + " session_name=session_name, \n", + " pipeline_id=pipeline_id, \n", + " input_settings=input_settings, \n", + " output_location=\"local\")" + ] + }, + { + "cell_type": "markdown", + "id": "2339ec14", + "metadata": {}, + "source": [ + "### Run & Result" + ] + }, + { + "cell_type": "markdown", + "id": "4286c3df", + "metadata": {}, + "source": [ + "Then, you can run the pipeline as usual!" + ] + }, + { + "cell_type": "markdown", + "id": "ef1ec6d7", + "metadata": {}, + "source": [ + "##### Workflow running" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d47ee6da-1178-4646-b9f6-a71711dcac01", + "metadata": {}, + "outputs": [], + "source": [ + "session.launch_pipeline()\n", + "session.monitor_workflows()" + ] + }, + { + "cell_type": "markdown", + "id": "73066fe4", + "metadata": {}, + "source": [ + "##### Downloading & Cleaning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81f8dc9e-22ab-4603-ae3b-28d83f0a2615", + "metadata": {}, + "outputs": [], + "source": [ + "session.download_outputs()\n", + "session.finish()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/demo-vipsession.ipynb b/examples/tutorials/demo-vipsession.ipynb index b2f2c08..cd9aeb8 100644 --- a/examples/tutorials/demo-vipsession.ipynb +++ b/examples/tutorials/demo-vipsession.ipynb @@ -118,7 +118,7 @@ "metadata": {}, "outputs": [], "source": [ - "my_session.upload_inputs(input_dir);" + "my_session.upload_inputs(input_dir)" ] }, { @@ -169,7 +169,7 @@ "outputs": [], "source": [ "# Pipeline identifier = App/Version (/!\\ mind the case)\n", - "pipeline_id = \"CQUEST/0.2-egi\"" + "pipeline_id = \"CQUEST/0.6\"" ] }, { @@ -595,7 +595,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -605,7 +605,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -619,14 +619,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.11.12" }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" - } - } + "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 diff --git a/examples/tutorials/example-VipGirder.ipynb b/examples/tutorials/example-VipGirder.ipynb deleted file mode 100644 index 37d3c30..0000000 --- a/examples/tutorials/example-VipGirder.ipynb +++ /dev/null @@ -1,210 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "e4b2f485-cb8b-4e4b-b21c-c381760fc914", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a2a0f64-107e-47a3-b994-4ba55735bfa8", - "metadata": {}, - "outputs": [], - "source": [ - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f72d24fb-f9f4-45b2-b7bc-0533216932ea", - "metadata": {}, - "outputs": [], - "source": [ - "import vip_client\n", - "import importlib\n", - "from vip_client import VipGirder\n", - "#importlib.reload(client)\n", - "vip_client.__path__" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a60aa4a-ba39-4c9b-b749-a79b8f6d21e7", - "metadata": {}, - "outputs": [], - "source": [ - "import inspect\n", - "inspect.getfile(VipGirder)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fc36a278-550b-43f5-800d-3500612ba12d", - "metadata": {}, - "outputs": [], - "source": [ - "VipGirder.init()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0a7c7ee2-2656-4585-9930-895e68cc90ac", - "metadata": {}, - "outputs": [], - "source": [ - "session.display()" - ] - }, - { - "cell_type": "markdown", - "id": "97d7cf93-2ab0-4d07-81b2-654b38480f9a", - "metadata": {}, - "source": [ - "# Output on girder (default)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cdaa53b9-eaec-44d7-a342-7b41191fcb4a", - "metadata": {}, - "outputs": [], - "source": [ - "pipeline_id=\"BasicGrepLocal/0.2\"\n", - "VipGirder.show_pipeline(pipeline_id)\n", - "session_name=\"test_girder_girder\"\n", - "output_dir=\"/collection/ReproVIPSpectro/test/vip_outputs\"\n", - "input_settings={\n", - " \"file\":\"/collection/ReproVIPSpectro/test/test_for_grep.txt\",\n", - " \"int\":5,\n", - " \"text\":\"grep\"\n", - "}\n", - "session = VipGirder(session_name=session_name, pipeline_id=pipeline_id, input_settings=input_settings, output_dir=output_dir)" - ] - }, - { - "cell_type": "markdown", - "id": "1aec7d5d-05e1-4baf-af30-2e9b42e791f1", - "metadata": {}, - "source": [ - "# Output on VIP" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25da2f62-e044-4ffd-855f-74881ad5d770", - "metadata": {}, - "outputs": [], - "source": [ - "pipeline_id=\"BasicGrepLocal/0.2\"\n", - "VipGirder.show_pipeline(pipeline_id)\n", - "session_name=\"test_girder_vip\"\n", - "input_settings={\n", - " \"file\":\"/collection/ReproVIPSpectro/test/test_for_grep.txt\",\n", - " \"int\":5,\n", - " \"text\":\"grep\"\n", - "}\n", - "session = VipGirder(session_name=session_name, pipeline_id=pipeline_id, input_settings=input_settings, output_location=\"vip\")" - ] - }, - { - "cell_type": "markdown", - "id": "7c726378-93b0-443d-9531-89b42735142d", - "metadata": {}, - "source": [ - "# Output in local" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bff17ffe-bebc-419d-b68d-ed3b0d2f4cf7", - "metadata": {}, - "outputs": [], - "source": [ - "pipeline_id=\"BasicGrepLocal/0.2\"\n", - "VipGirder.show_pipeline(pipeline_id)\n", - "session_name=\"test_girder_local\"\n", - "input_settings={\n", - " \"file\":\"/collection/ReproVIPSpectro/test/test_for_grep.txt\",\n", - " \"int\":5,\n", - " \"text\":\"grep\"\n", - "}\n", - "session = VipGirder(session_name=session_name, pipeline_id=pipeline_id, input_settings=input_settings, output_location=\"local\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d47ee6da-1178-4646-b9f6-a71711dcac01", - "metadata": {}, - "outputs": [], - "source": [ - "session.launch_pipeline()\n", - "session.display()\n", - "session.monitor_workflows()\n", - "session.display()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "81f8dc9e-22ab-4603-ae3b-28d83f0a2615", - "metadata": {}, - "outputs": [], - "source": [ - "session.download_outputs()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d54e5e44-7350-4e79-a8aa-0a8a58c60fac", - "metadata": {}, - "outputs": [], - "source": [ - "session.finish()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39544d26-4881-4562-9d65-209c61836b33", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/tutorials/exemple_VipCI.ipynb b/examples/tutorials/exemple_VipCI.ipynb deleted file mode 100644 index 06798bc..0000000 --- a/examples/tutorials/exemple_VipCI.ipynb +++ /dev/null @@ -1,303 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Main Parameters\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import the class\n", - "from vip_client.classes import VipGirder\n", - "import time\n", - "\n", - "# Pipeline identifier\n", - "my_pipeline=\"CQUEST/0.1.1\"\n", - "\n", - "# Input Settings\n", - "my_settings = {\n", - " \"zipped_folder\": \"/collection/ReproVIPSpectro/data/quest/basis.zip\",\n", - " \"data_file\": \"/collection/ReproVIPSpectro/data/quest/signals\",\n", - " \"parameter_file\": \"/collection/ReproVIPSpectro/data/quest/parameters/quest_param_117T_B.txt\",\n", - "}\n", - "# N.B.: Here \"data_file\" leads to a folder, the other arguments lead to single files.\n", - "\n", - "# Random output folder\n", - "my_output_dir = \"/collection/ReproVIPSpectro/results/test_%s\" \\\n", - " % time.strftime('%Y%m%d-%H%M%S', time.localtime())" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__N.B.__: In `my_settings`, the input files are pointed by Girder paths, usually in format: \"/collection/[collection_name]/[path_to_folder]\". \n", - "- if the path leads to a file, the same file be passed to VIP;\n", - "- If the path leads to a Girder item, all files in this item will be passed to VIP;\n", - "- If the path leads to a Girder folder, all files in this folder will be passed to VIP." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "Connect to VIP & Girder and run two VIP executions for CQUEST\n", - "---" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__N.B.__: You may kill the jobs after 3-4 minutes to shorten the test." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "---------------------------------------------\n", - "| You are communicating with VIP and Girder |\n", - "---------------------------------------------\n", - "\n", - "\n", - "=== SESSION 'Test_Girder' ===\n", - "\n", - "=== LAUNCH PIPELINE ===\n", - "\n", - "Pipeline ID: 'CQUEST/0.1.1' --> checked\n", - "Input Settings --> parsed\n", - "\n", - "Parameter checks\n", - "----------------\n", - "Pipeline identifier: OK\n", - "Output directory: Created on GIRDER\n", - "Input settings: OK\n", - "----------------\n", - "\n", - "Launching 2 new execution(s) on VIP\n", - "-------------------------------------\n", - "Execution Name: Test_Girder\n", - "Started Workflows:\n", - "\tworkflow-rbvvKU, workflow-VjczE7, \n", - "-------------------------------------\n", - "Done.\n", - "\n", - ">> Session backed up\n", - "\n", - "=== MONITOR WORKFLOWS ===\n", - "\n", - "Updating worflow inventory ... Done.\n", - "All executions are currently running on VIP.\n", - "\n", - "-------------------------------------------------------------\n", - "The current proccess will wait until all executions are over.\n", - "Their progress can be monitored on VIP portal:\n", - "\thttps://vip.creatis.insa-lyon.fr/\n", - "-------------------------------------------------------------\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/home/vila/code/projects/VIP-python-client/examples/old/VipCI/exemple_VipCI.ipynb Cellule 6\u001b[0m line \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 13\u001b[0m session\u001b[39m.\u001b[39mlaunch_pipeline(\n\u001b[1;32m 14\u001b[0m pipeline_id\u001b[39m=\u001b[39mmy_pipeline, \u001b[39m# Pipeline identifier\u001b[39;00m\n\u001b[1;32m 15\u001b[0m input_settings\u001b[39m=\u001b[39mmy_settings, \u001b[39m# Input Settings\u001b[39;00m\n\u001b[1;32m 16\u001b[0m output_dir\u001b[39m=\u001b[39mmy_output_dir, \u001b[39m# Output directory\u001b[39;00m\n\u001b[1;32m 17\u001b[0m nb_runs\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m \u001b[39m# Run 2 workflows at the same time\u001b[39;00m\n\u001b[1;32m 18\u001b[0m )\n\u001b[1;32m 20\u001b[0m \u001b[39m# Monitor VIP Executions\u001b[39;00m\n\u001b[0;32m---> 21\u001b[0m session\u001b[39m.\u001b[39;49mmonitor_workflows()\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/vip_client/classes/VipCI.py:198\u001b[0m, in \u001b[0;36mVipCI.monitor_workflows\u001b[0;34m(self, refresh_time)\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmonitor_workflows\u001b[39m(\u001b[39mself\u001b[39m, refresh_time\u001b[39m=\u001b[39m\u001b[39m30\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m VipCI:\n\u001b[1;32m 193\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 194\u001b[0m \u001b[39m Updates and displays the status of each execution launched in the current session.\u001b[39;00m\n\u001b[1;32m 195\u001b[0m \u001b[39m - If an execution is still runnig, updates status every `refresh_time` (seconds) until all runs are done.\u001b[39;00m\n\u001b[1;32m 196\u001b[0m \u001b[39m - Displays a full report when all executions are done.\u001b[39;00m\n\u001b[1;32m 197\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 198\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49mmonitor_workflows(refresh_time\u001b[39m=\u001b[39;49mrefresh_time)\n", - "File \u001b[0;32m~/.local/lib/python3.10/site-packages/vip_client/classes/VipLauncher.py:587\u001b[0m, in \u001b[0;36mVipLauncher.monitor_workflows\u001b[0;34m(self, refresh_time)\u001b[0m\n\u001b[1;32m 585\u001b[0m \u001b[39m# Sleep until next itertation\u001b[39;00m\n\u001b[1;32m 586\u001b[0m elapsed_time \u001b[39m=\u001b[39m time\u001b[39m.\u001b[39mtime() \u001b[39m-\u001b[39m start\n\u001b[0;32m--> 587\u001b[0m time\u001b[39m.\u001b[39;49msleep(\u001b[39mmax\u001b[39;49m(refresh_time \u001b[39m-\u001b[39;49m elapsed_time, \u001b[39m0\u001b[39;49m))\n\u001b[1;32m 588\u001b[0m \u001b[39m# Display the end of executions\u001b[39;00m\n\u001b[1;32m 589\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_print(\u001b[39m\"\u001b[39m\u001b[39mAll executions are over.\u001b[39m\u001b[39m\"\u001b[39m)\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "# Connect with Vip & Girder\n", - "VipGirder.init(\n", - " vip_key=\"VIP_API_KEY\", # My environment variable for the VIP API key (also works with litteral string or file name)\n", - " girder_key=\"GIRDER_API_KEY\" # My environment variable for the Girder API key (also works with litteral string or file name)\n", - ")\n", - "\n", - "# Create a Session\n", - "session = VipGirder(\n", - " session_name=\"Test_Girder\", # Session Name\n", - ")\n", - "\n", - "# Launch the Pipeline\n", - "session.launch_pipeline(\n", - " pipeline_id=my_pipeline, # Pipeline identifier\n", - " input_settings=my_settings, # Input Settings\n", - " output_dir=my_output_dir, # Output directory\n", - " nb_runs=2 # Run 2 workflows at the same time\n", - ")\n", - "\n", - "# Monitor VIP Executions\n", - "session.monitor_workflows()\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "**Session metadata are now attached to the output folder on Girder.**\n", - "- Check the results on the Girder portal: https://pilot-warehouse.creatis.insa-lyon.fr/#collection/63b6e0b14d15dd536f0484bc/folder/63b6e29b4d15dd536f0484c2" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "__N.B.__: The last cell could also be written with a single line of code:\n", - "```python\n", - "VipGirder.init(\n", - " vip_key=\"VIP_API_KEY\", # My environment variable for the VIP API key \n", - " girder_key=\"GIRDER_API_KEY\", # My environment variable for the Girder API key\n", - " session_name = \"Test_Girder\", # Session Name\n", - " pipeline_id=my_pipeline, # Pipeline identifier\n", - " input_settings=my_settings, # Input Settings\n", - " output_dir=my_output_dir # Output directory\n", - ").run_session(nb_runs=2) # Run 2 workflows at the same time\n", - "```" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "Resume a previous Session and run 2 additional executions\n", - "---\n", - "*A session can be loaded from its output directory on Girder*" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "<<< SESSION 'session_230315_212006' >>>\n", - "\n", - "An existing session was found.\n", - "Session properties were loaded from:\n", - "\t /collection/ReproVIPSpectro/results/test_20230315-211556\n", - "\n", - "<<< LAUNCH PIPELINE >>>\n", - "\n", - "Checking parameters and data ... Done.\n", - "\n", - "Launching 2 new execution(s) on VIP\n", - "-------------------------------------\n", - "\tSession Name: Test_Girder\n", - "\tPipeline Identifier: CQUEST/0.1-logs\n", - "\tStarted workflows:\n", - "\t\tworkflow-gBUbeh, workflow-6fVQTt, \n", - "-------------------------------------\n", - "Done.\n", - "\n", - "Session properties were saved as metadata under folder:\n", - "\t/collection/ReproVIPSpectro/results/test_20230315-211556\n", - "\t(Girder ID: 641228810386da27476979c8)\n", - "\n", - "\n", - "<<< MONITOR WORKFLOW >>>\n", - "\n", - "Updating worflow inventory ... Done.\n", - "2 execution(s) ended with status: Killed\n", - "\t workflow-I9eSOJ , started on: 2023/03/15 21:16:25\n", - "\t workflow-Oxf6GA , started on: 2023/03/15 21:16:17\n", - "2 execution(s) is/are currently running on VIP:\n", - "\t workflow-gBUbeh , started on: 2023/03/15 21:20:17\n", - "\t workflow-6fVQTt , started on: 2023/03/15 21:20:25\n", - "\n", - "-------------------------------------------------------------\n", - "The current proccess will wait until all executions are over.\n", - "Their progress can be monitored on the VIP portal:\n", - "\thttps://vip.creatis.insa-lyon.fr/\n", - "-------------------------------------------------------------\n", - "All executions are over.\n", - "All executions (4) ended with status: Killed\n", - "\n", - "Session properties were saved.\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "VipGirder(my_output_dir).run_session(nb_runs=2)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "Check the results on the Girder portal: https://pilot-warehouse.creatis.insa-lyon.fr/#collection/63b6e0b14d15dd536f0484bc/folder/63b6e29b4d15dd536f0484c2\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.8" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From b988566c311c5bda4114f1d57932a10183b44c11 Mon Sep 17 00:00:00 2001 From: Ethaniel Billon Date: Wed, 11 Jun 2025 17:12:13 +0200 Subject: [PATCH 3/4] fix _exists and empty dir condition --- src/vip_client/classes/VipSession.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/vip_client/classes/VipSession.py b/src/vip_client/classes/VipSession.py index 01da593..2288882 100644 --- a/src/vip_client/classes/VipSession.py +++ b/src/vip_client/classes/VipSession.py @@ -697,7 +697,7 @@ def _assert_location_value(cls, backup_location, label='backup_location') -> Non # Method to check existence of a distant or local resource. @classmethod - def _exists(cls, path: PurePath, location="local") -> bool: + def _exists(cls, path: PurePath, location="local", ignore_empty_dir=False) -> bool: """ Checks existence of a distant (`location`="vip") or local (`location`="local") resource. `path` can be a string or path-like object. If `location` is "local", empty folders are @@ -705,7 +705,10 @@ def _exists(cls, path: PurePath, location="local") -> bool: """ # Check path existence in `location` if location=="local": - return os.path.exists(path) and os.path.isdir(path) and os.listdir(path) + if not ignore_empty_dir: + return os.path.exists(path) + else: + return os.path.exists(path) and (os.path.isfile(path) or os.listdir(path)) else: return super()._exists(path=path, location=location) # ------------------------------------------------ @@ -812,12 +815,17 @@ def _upload_dir(self, local_path: Path, vip_path: PurePosixPath) -> list: Displays what it does if `self._verbose` is True. Returns a list of files which failed to be uploaded on VIP. """ + files_to_upload = [] # Scan the local directory - assert self._exists(local_path, location='local'), f"{local_path} does not exist." + assert os.path.exists(local_path), f"{local_path} does not exist." + # First display self._print(f"Cloning: {local_path} ", end="... ") + + if not os.listdir(local_path): + self._print("Ignoring empty dir") # Scan the distant directory and look for files to upload - if self._mkdirs(vip_path, location="vip"): + elif self._mkdirs(vip_path, location="vip"): # The distant directory did not exist before call # -> upload all the data (no scan to save time) files_to_upload = [ @@ -974,7 +982,7 @@ def _init_download(self, workflow) -> dict: # Update the file metadata files_to_download[file].update() # Make the parent directory (if needed) - self._mkdirs(local_path.parent, location="local", exist_ok=True) + self._mkdirs(local_path.parent, location="local") # Return the list of files to download return files_to_download # ------------------------------------------------ From 2391226c16fb0f150ab0113f2858dd63caff0e88 Mon Sep 17 00:00:00 2001 From: Axel Bonnet Date: Wed, 11 Jun 2025 18:27:50 +0200 Subject: [PATCH 4/4] correction tests --- src/vip_client/classes/VipSession.py | 4 ++-- tests/test_VipSession.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vip_client/classes/VipSession.py b/src/vip_client/classes/VipSession.py index 2288882..4610d12 100644 --- a/src/vip_client/classes/VipSession.py +++ b/src/vip_client/classes/VipSession.py @@ -817,12 +817,12 @@ def _upload_dir(self, local_path: Path, vip_path: PurePosixPath) -> list: """ files_to_upload = [] # Scan the local directory - assert os.path.exists(local_path), f"{local_path} does not exist." + assert self._exists(local_path, location='local'), f"{local_path} does not exist." # First display self._print(f"Cloning: {local_path} ", end="... ") - if not os.listdir(local_path): + if not self._exists(local_path, location='local', ignore_empty_dir=True): self._print("Ignoring empty dir") # Scan the distant directory and look for files to upload elif self._mkdirs(vip_path, location="vip"): diff --git a/tests/test_VipSession.py b/tests/test_VipSession.py index f1d62b0..164f7af 100644 --- a/tests/test_VipSession.py +++ b/tests/test_VipSession.py @@ -64,7 +64,7 @@ def test_run_and_finish(mocker, nb_runs, pipeline_id): processing = 100 - def fake_exists(cls=None, path=None, location="local"): + def fake_exists(cls=None, path=None, location="local", ignore_empty_dir=False): return True with patch.object(VipSession, '_exists', fake_exists):