From 6eb0e4f739944729e9f1d9c235c9ba43dcf762b2 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 06:57:36 +0200
Subject: [PATCH 01/19] Add cromwell 91

---
 ansible/roles/compiled/tasks/common/main.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ansible/roles/compiled/tasks/common/main.yml b/ansible/roles/compiled/tasks/common/main.yml
index be542c1..a5eb248 100644
--- a/ansible/roles/compiled/tasks/common/main.yml
+++ b/ansible/roles/compiled/tasks/common/main.yml
@@ -1360,6 +1360,13 @@
           womtool_url: "https://github.com/broadinstitute/cromwell/releases/download/86/womtool-86.jar",
           womtool_checksum: "sha512:606a68edb74e91a380f8fdf1408f28802f6f4081d3f4ed2a80ffb5ff7cbb5b11d8a320e02be40eda3d349b41ff7307687c94a6189d5cf8471f3bd850a391a1f2",
       }
+      - {
+        version_number: "91",
+        cromwell_url: "https://github.com/broadinstitute/cromwell/releases/download/91/cromwell-91.jar",
+        cromwell_checksum: "sha256:a1378c7b2df9ba8e86a82adff0e2216d44d4b39a338106e4d39a5034cece4731",
+        womtool_url: "https://github.com/broadinstitute/cromwell/releases/download/91/womtool-91.jar",
+        womtool_checksum: "sha256:96a6a63f3c05dba40d704665024fde88dac8cea2f2575a47c55548d3ba600390",
+      }
   include_tasks:
     file: 'cromwell.yaml'
   loop: "{{ versions }}"

From d57063f7e11cb6694de90d13c5f748b934bf245a Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 07:07:23 +0200
Subject: [PATCH 02/19] Remove -

---
 ansible/roles/containers/tasks/common/RStudio.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ansible/roles/containers/tasks/common/RStudio.yaml b/ansible/roles/containers/tasks/common/RStudio.yaml
index ebf412e..1599a13 100644
--- a/ansible/roles/containers/tasks/common/RStudio.yaml
+++ b/ansible/roles/containers/tasks/common/RStudio.yaml
@@ -1,11 +1,11 @@
 - name: Create RStudio {{ item.version_name }} container
   vars:
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - container_name: "RStudio{{ item.rstudio_version }}-R{{ item.r_version }}"
-    - container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
-    - R_module_dir: "{{ common_modules }}/R"
-    - R_binary_dir: "{{ bio_container_dir }}/RStudio/{{ item.rstudio_version }}-R{{ item.r_version }}"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    container_name: "RStudio{{ item.rstudio_version }}-R{{ item.r_version }}"
+    container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
+    R_module_dir: "{{ common_modules }}/R"
+    R_binary_dir: "{{ bio_container_dir }}/RStudio/{{ item.rstudio_version }}-R{{ item.r_version }}"
   tags:
     - "RStudio{{ item.rstudio_version }}-R{{ item.r_version }}"
     - never

From 8f27c727a63bf453caacfb2daa9944d3d81ea05d Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 07:10:41 +0200
Subject: [PATCH 03/19] updating for newer ansible

---
 ansible/roles/containers/tasks/common/RStudio.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/containers/tasks/common/RStudio.yaml b/ansible/roles/containers/tasks/common/RStudio.yaml
index 1599a13..a3c29fc 100644
--- a/ansible/roles/containers/tasks/common/RStudio.yaml
+++ b/ansible/roles/containers/tasks/common/RStudio.yaml
@@ -1,4 +1,4 @@
-- name: Create RStudio {{ item.version_name }} container
+- name: Create RStudio {{ item.rstudio_version }}-R{{ item.r_version }} container
   vars:
     base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
     container_name: "RStudio{{ item.rstudio_version }}-R{{ item.r_version }}"

From 675002e51fe2d2be2b7ac2f7ea79d9f1d4712ffa Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 07:17:30 +0200
Subject: [PATCH 04/19] Fix `[ERROR]: Vars in a Block must be specified as a
 dictionary.`

---
 ansible/roles/containers/tasks/bio/DIANN.yaml | 16 +++++-----
 .../containers/tasks/bio/PopGen_suite.yaml    | 30 +++++++++----------
 ansible/roles/containers/tasks/bio/beast.yaml | 26 ++++++++--------
 ansible/roles/containers/tasks/bio/vep.yaml   | 12 ++++----
 .../tasks/common/python-ml-base.yaml          |  8 ++---
 .../containers/tasks/common/python-ml.yaml    | 14 ++++-----
 .../roles/containers/tasks/common/ubuntu.yaml | 16 +++++-----
 7 files changed, 61 insertions(+), 61 deletions(-)

diff --git a/ansible/roles/containers/tasks/bio/DIANN.yaml b/ansible/roles/containers/tasks/bio/DIANN.yaml
index e10b1d1..72fc33c 100644
--- a/ansible/roles/containers/tasks/bio/DIANN.yaml
+++ b/ansible/roles/containers/tasks/bio/DIANN.yaml
@@ -1,13 +1,13 @@
 - name: Create DIANN {{ item.version_name }} container
   vars:
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - version_number: "{{ item.version_number }}"
-    - container_name: "DIANN_{{ version_number }}"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
-    - diann_binary_dir: "{{ bio_container_dir }}/DIANN/{{ version_number }}"
-    - diann_deb_url: "{{ item.deb_url }}"
-    - diann_module_dir: "{{ bio_modules }}/DIANN"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    version_number: "{{ item.version_number }}"
+    container_name: "DIANN_{{ version_number }}"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
+    diann_binary_dir: "{{ bio_container_dir }}/DIANN/{{ version_number }}"
+    diann_deb_url: "{{ item.deb_url }}"
+    diann_module_dir: "{{ bio_modules }}/DIANN"
   tags:
     - "diann{{ item.version_number }}"
     - never
diff --git a/ansible/roles/containers/tasks/bio/PopGen_suite.yaml b/ansible/roles/containers/tasks/bio/PopGen_suite.yaml
index 29a0e8b..9a81c1f 100644
--- a/ansible/roles/containers/tasks/bio/PopGen_suite.yaml
+++ b/ansible/roles/containers/tasks/bio/PopGen_suite.yaml
@@ -1,20 +1,20 @@
 - name: Create Population Genetics {{ item.version_number }} container
   vars:
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - version_number: "{{ item.version_number }}"
-    - container_name: "PopGen_{{ item.version_number }}"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
-    - arlecore: "{{ item.arlecore }}"
-    - beast: "{{ item.beast }}"
-    - plink2: "{{ item.plink2 }}"
-    - bcftools: "{{ item.bcftools }}"
-    - admixture: "{{ item.admixture }}"
-    - shapeit: "{{ item.shapeit }}"
-    - eigensoft: "{{ item.eigensoft }}"
-    - finestructure: "{{ item.finestructure }}"
-    - beagle: "{{ item.beagle }}"
-    - popgen_module_dir: "{{ bio_modules }}/popgen"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    version_number: "{{ item.version_number }}"
+    container_name: "PopGen_{{ item.version_number }}"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
+    arlecore: "{{ item.arlecore }}"
+    beast: "{{ item.beast }}"
+    plink2: "{{ item.plink2 }}"
+    bcftools: "{{ item.bcftools }}"
+    admixture: "{{ item.admixture }}"
+    shapeit: "{{ item.shapeit }}"
+    eigensoft: "{{ item.eigensoft }}"
+    finestructure: "{{ item.finestructure }}"
+    beagle: "{{ item.beagle }}"
+    popgen_module_dir: "{{ bio_modules }}/popgen"
   tags:
     - "popgen{{ item.version_number }}"
     - never
diff --git a/ansible/roles/containers/tasks/bio/beast.yaml b/ansible/roles/containers/tasks/bio/beast.yaml
index d70c2d7..6aea6b7 100644
--- a/ansible/roles/containers/tasks/bio/beast.yaml
+++ b/ansible/roles/containers/tasks/bio/beast.yaml
@@ -1,18 +1,18 @@
 - name: Create beast {{ item.version_name }} container
   vars:
-    - beast_version: "{{ item.beast_version }}"
-    - beast_url: "{{ item.beast_url }}"
-    - beast_checksum: "{{ item.beast_checksum }}"
-    - beagle_version: "{{ item.beagle_version }}"
-    - beagle_url: "{{ item.beagle_url }}"
-    - beagle_checksum: "{{ item.beagle_checksum }}"
-    - cuda_install: "{{ item.cuda_install }}"
-    - cuda_version: "{{ item.cuda_version }}"
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - container_name: "beast{{  beast_version }}"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
-    - beast_module_dir: "{{ bio_modules }}/beast"
+    beast_version: "{{ item.beast_version }}"
+    beast_url: "{{ item.beast_url }}"
+    beast_checksum: "{{ item.beast_checksum }}"
+    beagle_version: "{{ item.beagle_version }}"
+    beagle_url: "{{ item.beagle_url }}"
+    beagle_checksum: "{{ item.beagle_checksum }}"
+    cuda_install: "{{ item.cuda_install }}"
+    cuda_version: "{{ item.cuda_version }}"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    container_name: "beast{{  beast_version }}"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
+    beast_module_dir: "{{ bio_modules }}/beast"
   tags:
     - "beast{{ item.beast_version }}"
     - never
diff --git a/ansible/roles/containers/tasks/bio/vep.yaml b/ansible/roles/containers/tasks/bio/vep.yaml
index ccd6df1..3fee47d 100644
--- a/ansible/roles/containers/tasks/bio/vep.yaml
+++ b/ansible/roles/containers/tasks/bio/vep.yaml
@@ -1,11 +1,11 @@
 - name: Create base ubuntu {{ item.version_name }} container
   vars:
-    - base_container_name: "vep"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ base_container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ base_container_name }}.sif"
-    - data_dir: "{{ bio_data_dir }}/vep"
-    - cache_dir: "/opt/vep/.vep"
-    - vep_module_dir: "{{ bio_modules }}/vep"
+    base_container_name: "vep"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ base_container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ base_container_name }}.sif"
+    data_dir: "{{ bio_data_dir }}/vep"
+    cache_dir: "/opt/vep/.vep"
+    vep_module_dir: "{{ bio_modules }}/vep"
   tags:
     - "vep"
     - never
diff --git a/ansible/roles/containers/tasks/common/python-ml-base.yaml b/ansible/roles/containers/tasks/common/python-ml-base.yaml
index 52e9571..eedaa8b 100644
--- a/ansible/roles/containers/tasks/common/python-ml-base.yaml
+++ b/ansible/roles/containers/tasks/common/python-ml-base.yaml
@@ -1,9 +1,9 @@
 - name: Create python-ml-base {{ item.version_number }} container
   vars:
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - container_name: "python-ml-base-{{ item.version_number }}"
-    - container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    container_name: "python-ml-base-{{ item.version_number }}"
+    container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
   tags:
     - "python-ml-base{{ item.version_number }}"
     - never
diff --git a/ansible/roles/containers/tasks/common/python-ml.yaml b/ansible/roles/containers/tasks/common/python-ml.yaml
index 09d23ea..7dd4f2e 100644
--- a/ansible/roles/containers/tasks/common/python-ml.yaml
+++ b/ansible/roles/containers/tasks/common/python-ml.yaml
@@ -1,12 +1,12 @@
 - name: Create python-ml {{ item.version_number }} container
   vars:
-    - base_container_image: "{{ common_container_dir }}/python-ml-base-{{ item.base_version }}.sif"
-    - container_name: "python-ml-{{ item.version_number }}"
-    - container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
-    - python_ml_module_dir: "{{ common_modules }}/python-ml"
-    - python_ml_binary_dir: "{{ common_container_dir }}/python-ml/{{ item.version_number }}"
-    - python_ml_kernel_dir: "{{ common_container_dir }}/python-ml/{{ item.version_number }}/kernels"
+    base_container_image: "{{ common_container_dir }}/python-ml-base-{{ item.base_version }}.sif"
+    container_name: "python-ml-{{ item.version_number }}"
+    container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
+    python_ml_module_dir: "{{ common_modules }}/python-ml"
+    python_ml_binary_dir: "{{ common_container_dir }}/python-ml/{{ item.version_number }}"
+    python_ml_kernel_dir: "{{ common_container_dir }}/python-ml/{{ item.version_number }}/kernels"
   tags:
     - "python-ml{{ item.version_number }}"
     - never
diff --git a/ansible/roles/containers/tasks/common/ubuntu.yaml b/ansible/roles/containers/tasks/common/ubuntu.yaml
index 4125b30..8efdf96 100644
--- a/ansible/roles/containers/tasks/common/ubuntu.yaml
+++ b/ansible/roles/containers/tasks/common/ubuntu.yaml
@@ -1,8 +1,8 @@
 - name: Create base ubuntu {{ item.version_name }} container
   vars:
-    - base_container_name: "ubuntu-{{ item.version_name }}"
-    - base_container_recipe: "{{ common_container_recipe_dir }}/{{ base_container_name }}.def"
-    - base_container_image: "{{ common_container_dir }}/{{ base_container_name }}.sif"
+    base_container_name: "ubuntu-{{ item.version_name }}"
+    base_container_recipe: "{{ common_container_recipe_dir }}/{{ base_container_name }}.def"
+    base_container_image: "{{ common_container_dir }}/{{ base_container_name }}.sif"
   tags:
     - never
     - "ubuntu-{{ item.version_name }}"
@@ -21,11 +21,11 @@
 
 - name: Create ubuntu {{ item.version_name }} compile container
   vars:
-    - base_container_name: "ubuntu-{{ item.version_name }}"
-    - base_container_image: "{{ common_container_dir }}/{{ base_container_name }}.sif"
-    - container_name: "ubuntu-{{ item.version_name }}-compile"
-    - container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
+    base_container_name: "ubuntu-{{ item.version_name }}"
+    base_container_image: "{{ common_container_dir }}/{{ base_container_name }}.sif"
+    container_name: "ubuntu-{{ item.version_name }}-compile"
+    container_recipe: "{{ common_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ common_container_dir }}/{{ container_name }}.sif"
   tags:
     - never
     - "ubuntu-{{ item.version_name }}-compile"

From 4395e40431ae419ceeed27cab9d17e7ca839c6fd Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 08:06:46 +0200
Subject: [PATCH 05/19] Remove conmon from 4.3.0+

---
 ansible/roles/compiled/tasks/common/singularity.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/compiled/tasks/common/singularity.yaml b/ansible/roles/compiled/tasks/common/singularity.yaml
index f177878..c73be85 100644
--- a/ansible/roles/compiled/tasks/common/singularity.yaml
+++ b/ansible/roles/compiled/tasks/common/singularity.yaml
@@ -37,7 +37,7 @@
                 state: directory
                 mode: o=rwx,g=rx,o=rx
             - name: Configure singularity
-              command: "./mconfig --prefix={{ install_dir }} --localstatedir=/usr/local/var --sysconfdir=/usr/local/etc{% if version_number is version('3.10', '>=') %} --without-seccomp --without-conmon{% endif %}"
+              command: "./mconfig --prefix={{ install_dir }} --localstatedir=/usr/local/var --sysconfdir=/usr/local/etc{% if version_number is version('3.10', '>=') %} --without-seccomp{% endif %}{% if version_number is version('3.10', '>=') and version_number is version('4.3.0', '<') %} --without-conmon{% endif %}"
               args:
                 chdir: "{{ unzip_dir }}"
                 creates: "{{ unzip_dir }}/builddir/Makefile"

From caa5dc7032cdea97bfe568f95e29a5a978095bbc Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 09:46:36 +0200
Subject: [PATCH 06/19] Add block and tag so that gpu build can be skipper.

---
 ansible/roles/compiled/tasks/common/openmpi.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ansible/roles/compiled/tasks/common/openmpi.yaml b/ansible/roles/compiled/tasks/common/openmpi.yaml
index 8f1d93a..1e1098f 100644
--- a/ansible/roles/compiled/tasks/common/openmpi.yaml
+++ b/ansible/roles/compiled/tasks/common/openmpi.yaml
@@ -43,8 +43,10 @@
               args:
                 chdir: "{{ build_dir }}"
                 creates: "{{ build_dir }}/Makefile"
-            - name: Run OpenMPI {{ version_number }} configure ( 4.1.5 and above )
+            - name: Run OpenMPI {{ version_number }} configure ( 4.1.5 and above ) - GPU build
               when: item.version_number >= "4.1.5"
+              tags:
+                - openmpi_gpu
               block:
                 - name: Configure OpenMPI {{ version_number }} for gpu
                   command: "../configure --prefix={{ install_dir_gpu }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir }} --with-cuda={{ cuda_root }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib"
@@ -66,6 +68,9 @@
                   file:
                     path: "{{ build_dir }}/Makefile"
                     state: absent
+            - name: Run OpenMPI {{ version_number }} configure ( 4.1.5 and above ) - non-GPU build
+              when: item.version_number >= "4.1.5"
+              block:
                 - name: Configure OpenMPI {{ version_number }} for no gpu
                   command: "../configure --prefix={{ install_dir }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib"
                   args:

From 68ddb0f2b54d1c38a3106afed1fb63182ff720b5 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 09:49:49 +0200
Subject: [PATCH 07/19] Add tags to skip cuda

---
 ansible/roles/compiled/tasks/common/ucx.yaml | 48 ++++++++++++++------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/ansible/roles/compiled/tasks/common/ucx.yaml b/ansible/roles/compiled/tasks/common/ucx.yaml
index 7e5cc01..09d2b2a 100644
--- a/ansible/roles/compiled/tasks/common/ucx.yaml
+++ b/ansible/roles/compiled/tasks/common/ucx.yaml
@@ -32,20 +32,40 @@
               args:
                 chdir: "{{ unzipdir }}"
                 creates: "{{ unzipdir }}/configure"
-            - name: Run UCX {{ version_number }} configure
-              command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations --with-cuda={{ cuda_root }}"
-              args:
-                chdir: "{{ build_dir }}"
-                creates: "{{ build_dir }}/Makefile"
-            - name: Build UCX {{ version_number }}
-              make:
-                chdir: "{{ build_dir }}"
-              environment:
-                MAKEFLAGS: "-j {{ ansible_processor_vcpus }}"
-            - name: Install UCX {{ version_number }} to {{ install_dir }}
-              make:
-                chdir: "{{ build_dir }}"
-                target: install
+            - name: Run UCX {{ version_number }} configure - CUDA build
+              tags:
+                - ucx_cuda
+              block:
+                - name: Run UCX {{ version_number }} configure with CUDA
+                  command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations --with-cuda={{ cuda_root }}"
+                  args:
+                    chdir: "{{ build_dir }}"
+                    creates: "{{ build_dir }}/Makefile"
+                - name: Build UCX {{ version_number }} with CUDA
+                  make:
+                    chdir: "{{ build_dir }}"
+                  environment:
+                    MAKEFLAGS: "-j {{ ansible_processor_vcpus }}"
+                - name: Install UCX {{ version_number }} with CUDA to {{ install_dir }}
+                  make:
+                    chdir: "{{ build_dir }}"
+                    target: install
+            - name: Run UCX {{ version_number }} configure - non-CUDA build
+              block:
+                - name: Run UCX {{ version_number }} configure without CUDA
+                  command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations"
+                  args:
+                    chdir: "{{ build_dir }}"
+                    creates: "{{ build_dir }}/Makefile"
+                - name: Build UCX {{ version_number }} without CUDA
+                  make:
+                    chdir: "{{ build_dir }}"
+                  environment:
+                    MAKEFLAGS: "-j {{ ansible_processor_vcpus }}"
+                - name: Install UCX {{ version_number }} without CUDA to {{ install_dir }}
+                  make:
+                    chdir: "{{ build_dir }}"
+                    target: install
 
 #    - name: Install UCX {{ version_number }} module
 #      block:

From 67bee4f11cf636a59e409b09571cd877e48d95b3 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 12:12:35 +0200
Subject: [PATCH 08/19] Add libfreetype-dev

---
 ansible/roles/containers/tasks/common/RStudio.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ansible/roles/containers/tasks/common/RStudio.yaml b/ansible/roles/containers/tasks/common/RStudio.yaml
index a3c29fc..4a77c90 100644
--- a/ansible/roles/containers/tasks/common/RStudio.yaml
+++ b/ansible/roles/containers/tasks/common/RStudio.yaml
@@ -18,6 +18,12 @@
       fail:
         msg: "The container {{ base_container_image }} is missing."
       when: not base_container_exists.stat.exists
+    - name: Install some dependencies on host for RStudio Singularity container
+      apt:
+        name:
+          - libfreetype-dev
+        state: present
+      become: true
     - name: Create RStudio {{ item.rstudio_version }} container
       when: base_container_exists.stat.exists
       block:

From 56cd32141c72e27cea79ad6f9547ba79390bd3c7 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 12:14:10 +0200
Subject: [PATCH 09/19] Add libfreetype-dev

---
 ansible/roles/containers/tasks/common/RStudio.yaml    | 6 ------
 ansible/roles/containers/templates/common/RStudio.def | 1 +
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/ansible/roles/containers/tasks/common/RStudio.yaml b/ansible/roles/containers/tasks/common/RStudio.yaml
index 4a77c90..a3c29fc 100644
--- a/ansible/roles/containers/tasks/common/RStudio.yaml
+++ b/ansible/roles/containers/tasks/common/RStudio.yaml
@@ -18,12 +18,6 @@
       fail:
         msg: "The container {{ base_container_image }} is missing."
       when: not base_container_exists.stat.exists
-    - name: Install some dependencies on host for RStudio Singularity container
-      apt:
-        name:
-          - libfreetype-dev
-        state: present
-      become: true
     - name: Create RStudio {{ item.rstudio_version }} container
       when: base_container_exists.stat.exists
       block:
diff --git a/ansible/roles/containers/templates/common/RStudio.def b/ansible/roles/containers/templates/common/RStudio.def
index 2e501bf..73a129d 100644
--- a/ansible/roles/containers/templates/common/RStudio.def
+++ b/ansible/roles/containers/templates/common/RStudio.def
@@ -70,6 +70,7 @@ From: {{ base_container_image }}
     dirmngr \
     git \
     imagemagick \
+    libfreetype-dev \
     libfribidi-dev \
     libharfbuzz-dev \
     pandoc \

From 51725eda748d47063ea8a4a9673437d740ed0311 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 13:53:29 +0200
Subject: [PATCH 10/19] Moar libfreetype

---
 ansible/roles/containers/templates/common/RStudio.def | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ansible/roles/containers/templates/common/RStudio.def b/ansible/roles/containers/templates/common/RStudio.def
index 73a129d..251a9ba 100644
--- a/ansible/roles/containers/templates/common/RStudio.def
+++ b/ansible/roles/containers/templates/common/RStudio.def
@@ -105,6 +105,8 @@ From: {{ base_container_image }}
     libcairo2-dev \
     libcurl4-openssl-dev \
     libfontconfig-dev \
+    libfreetype-dev \
+    libfreetype6 \
     libfreetype6-dev \
     libfribidi-bin \
     libfribidi-dev \

From 7506d03dcdbd5f72af8a408a609fb5e1cc5ffa58 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 18 Nov 2025 13:56:24 +0200
Subject: [PATCH 11/19] Add libwebp-dev

---
 ansible/roles/containers/templates/common/RStudio.def | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ansible/roles/containers/templates/common/RStudio.def b/ansible/roles/containers/templates/common/RStudio.def
index 251a9ba..36cb7a9 100644
--- a/ansible/roles/containers/templates/common/RStudio.def
+++ b/ansible/roles/containers/templates/common/RStudio.def
@@ -120,6 +120,7 @@ From: {{ base_container_image }}
     libpng-dev \
     libssl-dev \
     libtiff5-dev \
+    libwebp-dev \
     libxml2-dev \
     libxt-dev \
     netcdf-bin \
@@ -133,6 +134,8 @@ From: {{ base_container_image }}
     systemd \
     wget \
 
+   # libfreetype6-dev libpng-dev libtiff5-dev libjpeg-dev libwebp-dev
+
   apt-mark hold r-base*
 
   # Add a default CRAN mirror

From 31d6a471ad7049a0484b78a56819c2db656c6f61 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Fri, 21 Nov 2025 07:13:14 +0200
Subject: [PATCH 12/19] a

---
 ansible/roles/compiled/tasks/common/main.yml  |  7 +-
 .../roles/compiled/tasks/common/nextflow.yaml |  8 +++
 .../roles/compiled/templates/nextflow.config  | 65 +++++++++++++++++++
 ansible/roles/compiled/templates/nextflow.lua |  1 +
 4 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100644 ansible/roles/compiled/templates/nextflow.config

diff --git a/ansible/roles/compiled/tasks/common/main.yml b/ansible/roles/compiled/tasks/common/main.yml
index a5eb248..a2f6b50 100644
--- a/ansible/roles/compiled/tasks/common/main.yml
+++ b/ansible/roles/compiled/tasks/common/main.yml
@@ -1245,7 +1245,7 @@
 
 - name: Install NextFlow
   vars:
-    java_module: "java/openjdk-18.0.2"
+    java_module: "java/openjdk-22.0.1"
     versions:
       - {
          version_number: "22.10.7",
@@ -1267,6 +1267,11 @@
         url: "https://github.com/nextflow-io/nextflow/releases/download/v25.04.6/nextflow",
         checksum: "sha256:a94f8bd1db9c0271ad58ec40b9c71f812d081a66f782396928b9b1f740f0be5f"
       }
+      - {
+        version_number: "25.10.0",
+        url: "https://github.com/nextflow-io/nextflow/releases/download/v25.10.0/nextflow",
+        checksum: "sha256:2a398d1dbf3a7258218ae8991429369ac4fdd86cb99b8c6c8f6c922202d9d524"
+      }
   include_tasks:
     file: 'nextflow.yaml'
   loop: "{{ versions }}"
diff --git a/ansible/roles/compiled/tasks/common/nextflow.yaml b/ansible/roles/compiled/tasks/common/nextflow.yaml
index 1041a19..5a9d215 100644
--- a/ansible/roles/compiled/tasks/common/nextflow.yaml
+++ b/ansible/roles/compiled/tasks/common/nextflow.yaml
@@ -26,6 +26,14 @@
               checksum: "{{ checksum }}"
               mode: u=rwx,g=rx,o=rx
 
+    - name: Deploy NextFlow configuration
+      block:
+        - name: Copy NextFlow config
+          copy:
+            src: nextflow.config
+            dest: "{{ install_dir }}/nextflow.config"
+            mode: u=rw,g=r,o=r
+
     - name: Install NextFlow {{ version_number }} module
       block:
         - name: Check module dir
diff --git a/ansible/roles/compiled/templates/nextflow.config b/ansible/roles/compiled/templates/nextflow.config
new file mode 100644
index 0000000..b5d6c3a
--- /dev/null
+++ b/ansible/roles/compiled/templates/nextflow.config
@@ -0,0 +1,65 @@
+// Nextflow configuration file
+
+profiles {
+    standard {
+        process.executor = 'local'
+    }
+
+    main_small {
+        process.executor = 'slurm'
+        process.queue = 'Main'
+        singularity.enabled = true
+        process.cpus = 1
+        process.memory = 4.GB
+    }
+
+    main_medium {
+        process.executor = 'slurm'
+        process.queue = 'Main'
+        singularity.enabled = true
+        process.cpus = 8
+        process.memory = 64.GB
+    }
+
+    main_large {
+        process.executor = 'slurm'
+        process.queue = 'Main'
+        singularity.enabled = true
+        process.cpus = 32
+        process.memory = 232.GB
+    }
+
+    highmem {
+        process.executor = 'slurm'
+        process.queue = 'HighMem'
+        singularity.enabled = true
+        process.cpus = 32
+        process.memory = 500.GB
+    }
+
+    veryhighmem {
+        process.executor = 'slurm'
+        process.queue = 'HighMem'
+        singularity.enabled = true
+        process.cpus = 96
+        process.memory = 1500.GB
+    }
+
+    gpu_1gpu {
+        process.executor = 'slurm'
+        process.queue = 'GPU'
+        singularity.enabled = true
+        process.cpus = 1
+        process.memory = 4.GB
+        process.clusterOptions = '--gres=gpu:1'
+    }
+
+    gpu_2gpu {
+        process.executor = 'slurm'
+        process.queue = 'GPU'
+        singularity.enabled = true
+        process.cpus = 1
+        process.memory = 4.GB
+        process.clusterOptions = '--gres=gpu:2'
+    }
+}
\ No newline at end of file
diff --git a/ansible/roles/compiled/templates/nextflow.lua b/ansible/roles/compiled/templates/nextflow.lua
index da541a1..44c549e 100644
--- a/ansible/roles/compiled/templates/nextflow.lua
+++ b/ansible/roles/compiled/templates/nextflow.lua
@@ -9,3 +9,4 @@ whatis("Description: NextFlow {{ version_number }}")
 load("{{ java_module }}")
 
 prepend_path('PATH', '{{ install_dir }}')
+setenv('NXF_HOME', '{{ install_dir }}')

From bee0553734be86a17aacf670e553f89d5b881f9e Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Fri, 21 Nov 2025 07:47:34 +0200
Subject: [PATCH 13/19] Move config to correct place

---
 ansible/roles/compiled/{templates => files}/nextflow.config | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename ansible/roles/compiled/{templates => files}/nextflow.config (100%)

diff --git a/ansible/roles/compiled/templates/nextflow.config b/ansible/roles/compiled/files/nextflow.config
similarity index 100%
rename from ansible/roles/compiled/templates/nextflow.config
rename to ansible/roles/compiled/files/nextflow.config

From 67ec9e06da8477169c6fa3f1ff588d14baf32cf5 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Fri, 21 Nov 2025 07:55:30 +0200
Subject: [PATCH 14/19] fix config file name

---
 ansible/roles/compiled/tasks/common/nextflow.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/compiled/tasks/common/nextflow.yaml b/ansible/roles/compiled/tasks/common/nextflow.yaml
index 5a9d215..fdf8c0e 100644
--- a/ansible/roles/compiled/tasks/common/nextflow.yaml
+++ b/ansible/roles/compiled/tasks/common/nextflow.yaml
@@ -31,7 +31,7 @@
         - name: Copy NextFlow config
           copy:
             src: nextflow.config
-            dest: "{{ install_dir }}/nextflow.config"
+            dest: "{{ install_dir }}/config"
             mode: u=rw,g=r,o=r
 
     - name: Install NextFlow {{ version_number }} module

From e4215d5b2a0909b297c2a4c0e1b24c96597617ad Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Fri, 21 Nov 2025 08:25:30 +0200
Subject: [PATCH 15/19] Add nf-ilifu wrapper that appropriately configures
 nextflow

---
 ansible/roles/compiled/tasks/common/nextflow.yaml | 7 ++++++-
 ansible/roles/compiled/templates/nextflow.lua     | 1 -
 ansible/roles/compiled/templates/nf-ilifu         | 5 +++++
 3 files changed, 11 insertions(+), 2 deletions(-)
 create mode 100644 ansible/roles/compiled/templates/nf-ilifu

diff --git a/ansible/roles/compiled/tasks/common/nextflow.yaml b/ansible/roles/compiled/tasks/common/nextflow.yaml
index fdf8c0e..326af37 100644
--- a/ansible/roles/compiled/tasks/common/nextflow.yaml
+++ b/ansible/roles/compiled/tasks/common/nextflow.yaml
@@ -26,13 +26,18 @@
               checksum: "{{ checksum }}"
               mode: u=rwx,g=rx,o=rx
 
-    - name: Deploy NextFlow configuration
+    - name: Deploy NextFlow configuration and wrapper
       block:
         - name: Copy NextFlow config
           copy:
             src: nextflow.config
             dest: "{{ install_dir }}/config"
             mode: u=rw,g=r,o=r
+        - name: Deploy NextFlow wrapper script
+          template:
+            src: nf-ilifu
+            dest: "{{ install_dir }}/nf-ilifu"
+            mode: u=rwx,g=rx,o=rx
 
     - name: Install NextFlow {{ version_number }} module
       block:
diff --git a/ansible/roles/compiled/templates/nextflow.lua b/ansible/roles/compiled/templates/nextflow.lua
index 44c549e..da541a1 100644
--- a/ansible/roles/compiled/templates/nextflow.lua
+++ b/ansible/roles/compiled/templates/nextflow.lua
@@ -9,4 +9,3 @@ whatis("Description: NextFlow {{ version_number }}")
 load("{{ java_module }}")
 
 prepend_path('PATH', '{{ install_dir }}')
-setenv('NXF_HOME', '{{ install_dir }}')
diff --git a/ansible/roles/compiled/templates/nf-ilifu b/ansible/roles/compiled/templates/nf-ilifu
new file mode 100644
index 0000000..8fb24cb
--- /dev/null
+++ b/ansible/roles/compiled/templates/nf-ilifu
@@ -0,0 +1,5 @@
+#!/bin/bash
+# Nextflow wrapper script for ILIFU
+# Automatically loads the shared cluster configuration
+
+exec {{ install_dir }}/nextflow -c {{ install_dir }}/config "$@"
\ No newline at end of file

From 3084b7789f88c035dd6e87fcc86bca0811d9b6c8 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Fri, 21 Nov 2025 08:27:18 +0200
Subject: [PATCH 16/19] rename to show it's a shell script. It's the small
 things

---
 ansible/roles/compiled/tasks/common/nextflow.yaml          | 2 +-
 ansible/roles/compiled/templates/{nf-ilifu => nf-ilifu.sh} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename ansible/roles/compiled/templates/{nf-ilifu => nf-ilifu.sh} (100%)

diff --git a/ansible/roles/compiled/tasks/common/nextflow.yaml b/ansible/roles/compiled/tasks/common/nextflow.yaml
index 326af37..964a9ef 100644
--- a/ansible/roles/compiled/tasks/common/nextflow.yaml
+++ b/ansible/roles/compiled/tasks/common/nextflow.yaml
@@ -35,7 +35,7 @@
             mode: u=rw,g=r,o=r
         - name: Deploy NextFlow wrapper script
           template:
-            src: nf-ilifu
+            src: nf-ilifu.sh
             dest: "{{ install_dir }}/nf-ilifu"
             mode: u=rwx,g=rx,o=rx
 
diff --git a/ansible/roles/compiled/templates/nf-ilifu b/ansible/roles/compiled/templates/nf-ilifu.sh
similarity index 100%
rename from ansible/roles/compiled/templates/nf-ilifu
rename to ansible/roles/compiled/templates/nf-ilifu.sh

From 1c39d2091626b73b1218ea4dc6ee4d7e5cfbaacf Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Fri, 21 Nov 2025 08:28:46 +0200
Subject: [PATCH 17/19] \n

---
 ansible/roles/compiled/templates/nf-ilifu.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/compiled/templates/nf-ilifu.sh b/ansible/roles/compiled/templates/nf-ilifu.sh
index 8fb24cb..d3e16aa 100644
--- a/ansible/roles/compiled/templates/nf-ilifu.sh
+++ b/ansible/roles/compiled/templates/nf-ilifu.sh
@@ -2,4 +2,4 @@
 # Nextflow wrapper script for ILIFU
 # Automatically loads the shared cluster configuration
 
-exec {{ install_dir }}/nextflow -c {{ install_dir }}/config "$@"
\ No newline at end of file
+exec {{ install_dir }}/nextflow -c {{ install_dir }}/config "$@"

From 808fac0f3297a81a73947edbd4516346fe8b9837 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Fri, 21 Nov 2025 08:31:08 +0200
Subject: [PATCH 18/19] \n

---
 ansible/roles/compiled/files/nextflow.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/compiled/files/nextflow.config b/ansible/roles/compiled/files/nextflow.config
index b5d6c3a..47de7af 100644
--- a/ansible/roles/compiled/files/nextflow.config
+++ b/ansible/roles/compiled/files/nextflow.config
@@ -62,4 +62,4 @@ profiles {
         process.memory = 4.GB
         process.clusterOptions = '--gres=gpu:2'
     }
-}
\ No newline at end of file
+}

From 3366db5655f641250b024d73f1de3ca53d6aab02 Mon Sep 17 00:00:00 2001
From: Dane Kennedy <kennedy.dane@gmail.com>
Date: Tue, 25 Nov 2025 14:56:54 +0200
Subject: [PATCH 19/19] Make cromwell a bit more flexible in terms of
 parameters

---
 .../compiled/templates/cromwell_slurm.conf    | 62 ++++++++++++++++++-
 1 file changed, 60 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/compiled/templates/cromwell_slurm.conf b/ansible/roles/compiled/templates/cromwell_slurm.conf
index 78e68c0..bc3f6df 100644
--- a/ansible/roles/compiled/templates/cromwell_slurm.conf
+++ b/ansible/roles/compiled/templates/cromwell_slurm.conf
@@ -9,17 +9,75 @@ backend {
 	  config {
 		runtime-attributes = """
 		Int runtime_minutes = 600
+		Int cpu = 0
 		Int cpus = 2
+		Int memory = 0
 		Int requested_memory_mb_per_core = 7100
 		String queue = "Main"
 		"""
 
 		submit = """
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ] && [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'cpu' and 'cpus'" >&2
+			  exit 1
+			fi
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ] && [ -n "${requested_memory_mb_per_core}" ] && [ "${requested_memory_mb_per_core}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'memory' and 'requested_memory_mb_per_core'" >&2
+			  exit 1
+			fi
+			CPU_FLAG=""
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpu}"
+			elif [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpus}"
+			fi
+			MEMORY_FLAG=""
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ]; then
+			  MEMORY_FLAG="--mem ${memory}"
+			else
+			  MEMORY_FLAG="--mem-per-cpu ${requested_memory_mb_per_core}"
+			fi
 			sbatch -J ${job_name} -D ${cwd} -o ${out} -e ${err} -t ${runtime_minutes} -p ${queue} \
-			${"-c " + cpus} \
-			--mem-per-cpu ${requested_memory_mb_per_core} \
+			${CPU_FLAG} ${MEMORY_FLAG} \
 			--wrap "/bin/bash ${script}"
 		"""
+
+		submit-docker = """
+			if [ -z $SINGULARITY_CACHEDIR ];
+				then CACHE_DIR=$HOME/.singularity/cache
+			else CACHE_DIR=$SINGULARITY_CACHEDIR
+			fi
+			mkdir -p $CACHE_DIR
+			LOCK_FILE=$CACHE_DIR/singularity_pull_flock
+			flock --verbose --exclusive --timeout 900 $LOCK_FILE \
+			singularity exec --containall docker://${docker} \
+			echo "successfully pulled ${docker}!"
+
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ] && [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'cpu' and 'cpus'" >&2
+			  exit 1
+			fi
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ] && [ -n "${requested_memory_mb_per_core}" ] && [ "${requested_memory_mb_per_core}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'memory' and 'requested_memory_mb_per_core'" >&2
+			  exit 1
+			fi
+			CPU_FLAG=""
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpu}"
+			elif [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpus}"
+			fi
+			MEMORY_FLAG=""
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ]; then
+			  MEMORY_FLAG="--mem ${memory}"
+			else
+			  MEMORY_FLAG="--mem-per-cpu ${requested_memory_mb_per_core}"
+			fi
+			sbatch -J ${job_name} -D ${cwd} -o ${out} -e ${err} -t ${runtime_minutes} -p ${queue} \
+			${CPU_FLAG} ${MEMORY_FLAG} \
+			--wrap "singularity exec --containall --bind ${cwd}:${docker_cwd} docker://${docker} ${job_shell} ${docker_script}"
+		"""
+
 		kill = "scancel ${job_id}"
 		check-alive = "squeue -j ${job_id}"
 		job-id-regex = "Submitted batch job (\\d+).*"