Dane/update/cromwell and other important stuff

ansible/roles/compiled/files/nextflow.config

@@ -0,0 +1,65 @@
+// Nextflow configuration file
+
+profiles {
+    standard {
+        process.executor = 'local'
+    }
+
+    main_small {
+        process.executor = 'slurm'
+        process.queue = 'Main'
+        singularity.enabled = true
+        process.cpus = 1
+        process.memory = 4.GB
+    }
+
+    main_medium {
+        process.executor = 'slurm'
+        process.queue = 'Main'
+        singularity.enabled = true
+        process.cpus = 8
+        process.memory = 64.GB
+    }
+
+    main_large {
+        process.executor = 'slurm'
+        process.queue = 'Main'
+        singularity.enabled = true
+        process.cpus = 32
+        process.memory = 232.GB
+    }
+
+    highmem {
+        process.executor = 'slurm'
+        process.queue = 'HighMem'
+        singularity.enabled = true
+        process.cpus = 32
+        process.memory = 500.GB
+    }
+
+    veryhighmem {
+        process.executor = 'slurm'
+        process.queue = 'HighMem'
+        singularity.enabled = true
+        process.cpus = 96
+        process.memory = 1500.GB
+    }
+
+    gpu_1gpu {
+        process.executor = 'slurm'
+        process.queue = 'GPU'
+        singularity.enabled = true
+        process.cpus = 1
+        process.memory = 4.GB
+        process.clusterOptions = '--gres=gpu:1'
+    }
+
+    gpu_2gpu {
+        process.executor = 'slurm'
+        process.queue = 'GPU'
+        singularity.enabled = true
+        process.cpus = 1
+        process.memory = 4.GB
+        process.clusterOptions = '--gres=gpu:2'
+    }
+}

ansible/roles/compiled/tasks/common/main.yml

@@ -1245,7 +1245,7 @@
 
 - name: Install NextFlow
   vars:
-    java_module: "java/openjdk-18.0.2"
+    java_module: "java/openjdk-22.0.1"
     versions:
       - {
          version_number: "22.10.7",
@@ -1267,6 +1267,11 @@
         url: "https://github.com/nextflow-io/nextflow/releases/download/v25.04.6/nextflow",
         checksum: "sha256:a94f8bd1db9c0271ad58ec40b9c71f812d081a66f782396928b9b1f740f0be5f"
       }
+      - {
+        version_number: "25.10.0",
+        url: "https://github.com/nextflow-io/nextflow/releases/download/v25.10.0/nextflow",
+        checksum: "sha256:2a398d1dbf3a7258218ae8991429369ac4fdd86cb99b8c6c8f6c922202d9d524"
+      }
   include_tasks:
     file: 'nextflow.yaml'
   loop: "{{ versions }}"
@@ -1360,6 +1365,13 @@
           womtool_url: "https://github.com/broadinstitute/cromwell/releases/download/86/womtool-86.jar",
           womtool_checksum: "sha512:606a68edb74e91a380f8fdf1408f28802f6f4081d3f4ed2a80ffb5ff7cbb5b11d8a320e02be40eda3d349b41ff7307687c94a6189d5cf8471f3bd850a391a1f2",
       }
+      - {
+        version_number: "91",
+        cromwell_url: "https://github.com/broadinstitute/cromwell/releases/download/91/cromwell-91.jar",
+        cromwell_checksum: "sha256:a1378c7b2df9ba8e86a82adff0e2216d44d4b39a338106e4d39a5034cece4731",
+        womtool_url: "https://github.com/broadinstitute/cromwell/releases/download/91/womtool-91.jar",
+        womtool_checksum: "sha256:96a6a63f3c05dba40d704665024fde88dac8cea2f2575a47c55548d3ba600390",
+      }
   include_tasks:
     file: 'cromwell.yaml'
   loop: "{{ versions }}"

ansible/roles/compiled/tasks/common/nextflow.yaml

@@ -26,6 +26,19 @@
               checksum: "{{ checksum }}"
               mode: u=rwx,g=rx,o=rx
 
+    - name: Deploy NextFlow configuration and wrapper
+      block:
+        - name: Copy NextFlow config
+          copy:
+            src: nextflow.config
+            dest: "{{ install_dir }}/config"
+            mode: u=rw,g=r,o=r
+        - name: Deploy NextFlow wrapper script
+          template:
+            src: nf-ilifu.sh
+            dest: "{{ install_dir }}/nf-ilifu"
+            mode: u=rwx,g=rx,o=rx
+
     - name: Install NextFlow {{ version_number }} module
       block:
         - name: Check module dir

ansible/roles/compiled/tasks/common/openmpi.yaml

@@ -43,8 +43,10 @@
               args:
                 chdir: "{{ build_dir }}"
                 creates: "{{ build_dir }}/Makefile"
-            - name: Run OpenMPI {{ version_number }} configure ( 4.1.5 and above )
+            - name: Run OpenMPI {{ version_number }} configure ( 4.1.5 and above ) - GPU build
               when: item.version_number >= "4.1.5"
+              tags:
+                - openmpi_gpu
               block:
                 - name: Configure OpenMPI {{ version_number }} for gpu
                   command: "../configure --prefix={{ install_dir_gpu }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir }} --with-cuda={{ cuda_root }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib"
@@ -66,6 +68,9 @@
                   file:
                     path: "{{ build_dir }}/Makefile"
                     state: absent
+            - name: Run OpenMPI {{ version_number }} configure ( 4.1.5 and above ) - non-GPU build
+              when: item.version_number >= "4.1.5"
+              block:
                 - name: Configure OpenMPI {{ version_number }} for no gpu
                   command: "../configure --prefix={{ install_dir }} --enable-static --with-singularity={{ singularity_dir }} --with-ucx={{ ucx_dir }} --with-pmix={{ item.pmix }} --with-pmix-libdir={{ item.pmix }}/lib"
                   args:

ansible/roles/compiled/tasks/common/singularity.yaml

@@ -37,7 +37,7 @@
                 state: directory
                 mode: o=rwx,g=rx,o=rx
             - name: Configure singularity
-              command: "./mconfig --prefix={{ install_dir }} --localstatedir=/usr/local/var --sysconfdir=/usr/local/etc{% if version_number is version('3.10', '>=') %} --without-seccomp --without-conmon{% endif %}"
+              command: "./mconfig --prefix={{ install_dir }} --localstatedir=/usr/local/var --sysconfdir=/usr/local/etc{% if version_number is version('3.10', '>=') %} --without-seccomp{% endif %}{% if version_number is version('3.10', '>=') and version_number is version('4.3.0', '<') %} --without-conmon{% endif %}"
               args:
                 chdir: "{{ unzip_dir }}"
                 creates: "{{ unzip_dir }}/builddir/Makefile"

ansible/roles/compiled/tasks/common/ucx.yaml

@@ -32,20 +32,40 @@
               args:
                 chdir: "{{ unzipdir }}"
                 creates: "{{ unzipdir }}/configure"
-            - name: Run UCX {{ version_number }} configure
-              command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations --with-cuda={{ cuda_root }}"
-              args:
-                chdir: "{{ build_dir }}"
-                creates: "{{ build_dir }}/Makefile"
-            - name: Build UCX {{ version_number }}
-              make:
-                chdir: "{{ build_dir }}"
-              environment:
-                MAKEFLAGS: "-j {{ ansible_processor_vcpus }}"
-            - name: Install UCX {{ version_number }} to {{ install_dir }}
-              make:
-                chdir: "{{ build_dir }}"
-                target: install
+            - name: Run UCX {{ version_number }} configure - CUDA build
+              tags:
+                - ucx_cuda
+              block:
+                - name: Run UCX {{ version_number }} configure with CUDA
+                  command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations --with-cuda={{ cuda_root }}"
+                  args:
+                    chdir: "{{ build_dir }}"
+                    creates: "{{ build_dir }}/Makefile"
+                - name: Build UCX {{ version_number }} with CUDA
+                  make:
+                    chdir: "{{ build_dir }}"
+                  environment:
+                    MAKEFLAGS: "-j {{ ansible_processor_vcpus }}"
+                - name: Install UCX {{ version_number }} with CUDA to {{ install_dir }}
+                  make:
+                    chdir: "{{ build_dir }}"
+                    target: install
+            - name: Run UCX {{ version_number }} configure - non-CUDA build
+              block:
+                - name: Run UCX {{ version_number }} configure without CUDA
+                  command: "../contrib/configure-release --prefix={{ install_dir }} --enable-optimizations"
+                  args:
+                    chdir: "{{ build_dir }}"
+                    creates: "{{ build_dir }}/Makefile"
+                - name: Build UCX {{ version_number }} without CUDA
+                  make:
+                    chdir: "{{ build_dir }}"
+                  environment:
+                    MAKEFLAGS: "-j {{ ansible_processor_vcpus }}"
+                - name: Install UCX {{ version_number }} without CUDA to {{ install_dir }}
+                  make:
+                    chdir: "{{ build_dir }}"
+                    target: install
 
 #    - name: Install UCX {{ version_number }} module
 #      block:

ansible/roles/compiled/templates/cromwell_slurm.conf

@@ -9,17 +9,75 @@ backend {
 	  config {
 		runtime-attributes = """
 		Int runtime_minutes = 600
+		Int cpu = 0
 		Int cpus = 2
+		Int memory = 0
 		Int requested_memory_mb_per_core = 7100
 		String queue = "Main"
 		"""
 
 		submit = """
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ] && [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'cpu' and 'cpus'" >&2
+			  exit 1
+			fi
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ] && [ -n "${requested_memory_mb_per_core}" ] && [ "${requested_memory_mb_per_core}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'memory' and 'requested_memory_mb_per_core'" >&2
+			  exit 1
+			fi
+			CPU_FLAG=""
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpu}"
+			elif [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpus}"
+			fi
+			MEMORY_FLAG=""
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ]; then
+			  MEMORY_FLAG="--mem ${memory}"
+			else
+			  MEMORY_FLAG="--mem-per-cpu ${requested_memory_mb_per_core}"
+			fi
 			sbatch -J ${job_name} -D ${cwd} -o ${out} -e ${err} -t ${runtime_minutes} -p ${queue} \
-			${"-c " + cpus} \
-			--mem-per-cpu ${requested_memory_mb_per_core} \
+			${CPU_FLAG} ${MEMORY_FLAG} \
 			--wrap "/bin/bash ${script}"
 		"""
+
+		submit-docker = """
+			if [ -z $SINGULARITY_CACHEDIR ];
+				then CACHE_DIR=$HOME/.singularity/cache
+			else CACHE_DIR=$SINGULARITY_CACHEDIR
+			fi
+			mkdir -p $CACHE_DIR
+			LOCK_FILE=$CACHE_DIR/singularity_pull_flock
+			flock --verbose --exclusive --timeout 900 $LOCK_FILE \
+			singularity exec --containall docker://${docker} \
+			echo "successfully pulled ${docker}!"
+
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ] && [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'cpu' and 'cpus'" >&2
+			  exit 1
+			fi
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ] && [ -n "${requested_memory_mb_per_core}" ] && [ "${requested_memory_mb_per_core}" -gt 0 ]; then
+			  echo "ERROR: Cannot specify both 'memory' and 'requested_memory_mb_per_core'" >&2
+			  exit 1
+			fi
+			CPU_FLAG=""
+			if [ -n "${cpu}" ] && [ "${cpu}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpu}"
+			elif [ -n "${cpus}" ] && [ "${cpus}" -gt 0 ]; then
+			  CPU_FLAG="-c ${cpus}"
+			fi
+			MEMORY_FLAG=""
+			if [ -n "${memory}" ] && [ "${memory}" -gt 0 ]; then
+			  MEMORY_FLAG="--mem ${memory}"
+			else
+			  MEMORY_FLAG="--mem-per-cpu ${requested_memory_mb_per_core}"
+			fi
+			sbatch -J ${job_name} -D ${cwd} -o ${out} -e ${err} -t ${runtime_minutes} -p ${queue} \
+			${CPU_FLAG} ${MEMORY_FLAG} \
+			--wrap "singularity exec --containall --bind ${cwd}:${docker_cwd} docker://${docker} ${job_shell} ${docker_script}"
+		"""
+
 		kill = "scancel ${job_id}"
 		check-alive = "squeue -j ${job_id}"
 		job-id-regex = "Submitted batch job (\\d+).*"

ansible/roles/compiled/templates/nf-ilifu.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Nextflow wrapper script for ILIFU
+# Automatically loads the shared cluster configuration
+
+exec {{ install_dir }}/nextflow -c {{ install_dir }}/config "$@"

ansible/roles/containers/tasks/bio/DIANN.yaml

@@ -1,13 +1,13 @@
 - name: Create DIANN {{ item.version_name }} container
   vars:
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - version_number: "{{ item.version_number }}"
-    - container_name: "DIANN_{{ version_number }}"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
-    - diann_binary_dir: "{{ bio_container_dir }}/DIANN/{{ version_number }}"
-    - diann_deb_url: "{{ item.deb_url }}"
-    - diann_module_dir: "{{ bio_modules }}/DIANN"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    version_number: "{{ item.version_number }}"
+    container_name: "DIANN_{{ version_number }}"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
+    diann_binary_dir: "{{ bio_container_dir }}/DIANN/{{ version_number }}"
+    diann_deb_url: "{{ item.deb_url }}"
+    diann_module_dir: "{{ bio_modules }}/DIANN"
   tags:
     - "diann{{ item.version_number }}"
     - never

ansible/roles/containers/tasks/bio/PopGen_suite.yaml

@@ -1,20 +1,20 @@
 - name: Create Population Genetics {{ item.version_number }} container
   vars:
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - version_number: "{{ item.version_number }}"
-    - container_name: "PopGen_{{ item.version_number }}"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
-    - arlecore: "{{ item.arlecore }}"
-    - beast: "{{ item.beast }}"
-    - plink2: "{{ item.plink2 }}"
-    - bcftools: "{{ item.bcftools }}"
-    - admixture: "{{ item.admixture }}"
-    - shapeit: "{{ item.shapeit }}"
-    - eigensoft: "{{ item.eigensoft }}"
-    - finestructure: "{{ item.finestructure }}"
-    - beagle: "{{ item.beagle }}"
-    - popgen_module_dir: "{{ bio_modules }}/popgen"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    version_number: "{{ item.version_number }}"
+    container_name: "PopGen_{{ item.version_number }}"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
+    arlecore: "{{ item.arlecore }}"
+    beast: "{{ item.beast }}"
+    plink2: "{{ item.plink2 }}"
+    bcftools: "{{ item.bcftools }}"
+    admixture: "{{ item.admixture }}"
+    shapeit: "{{ item.shapeit }}"
+    eigensoft: "{{ item.eigensoft }}"
+    finestructure: "{{ item.finestructure }}"
+    beagle: "{{ item.beagle }}"
+    popgen_module_dir: "{{ bio_modules }}/popgen"
   tags:
     - "popgen{{ item.version_number }}"
     - never

ansible/roles/containers/tasks/bio/beast.yaml

@@ -1,18 +1,18 @@
 - name: Create beast {{ item.version_name }} container
   vars:
-    - beast_version: "{{ item.beast_version }}"
-    - beast_url: "{{ item.beast_url }}"
-    - beast_checksum: "{{ item.beast_checksum }}"
-    - beagle_version: "{{ item.beagle_version }}"
-    - beagle_url: "{{ item.beagle_url }}"
-    - beagle_checksum: "{{ item.beagle_checksum }}"
-    - cuda_install: "{{ item.cuda_install }}"
-    - cuda_version: "{{ item.cuda_version }}"
-    - base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
-    - container_name: "beast{{  beast_version }}"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
-    - beast_module_dir: "{{ bio_modules }}/beast"
+    beast_version: "{{ item.beast_version }}"
+    beast_url: "{{ item.beast_url }}"
+    beast_checksum: "{{ item.beast_checksum }}"
+    beagle_version: "{{ item.beagle_version }}"
+    beagle_url: "{{ item.beagle_url }}"
+    beagle_checksum: "{{ item.beagle_checksum }}"
+    cuda_install: "{{ item.cuda_install }}"
+    cuda_version: "{{ item.cuda_version }}"
+    base_container_image: "{{ common_container_dir }}/{{ item.base_container_name }}.sif"
+    container_name: "beast{{  beast_version }}"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ container_name }}.sif"
+    beast_module_dir: "{{ bio_modules }}/beast"
   tags:
     - "beast{{ item.beast_version }}"
     - never

ansible/roles/containers/tasks/bio/vep.yaml

@@ -1,11 +1,11 @@
 - name: Create base ubuntu {{ item.version_name }} container
   vars:
-    - base_container_name: "vep"
-    - container_recipe: "{{ bio_container_recipe_dir }}/{{ base_container_name }}.def"
-    - container_image: "{{ bio_container_dir }}/{{ base_container_name }}.sif"
-    - data_dir: "{{ bio_data_dir }}/vep"
-    - cache_dir: "/opt/vep/.vep"
-    - vep_module_dir: "{{ bio_modules }}/vep"
+    base_container_name: "vep"
+    container_recipe: "{{ bio_container_recipe_dir }}/{{ base_container_name }}.def"
+    container_image: "{{ bio_container_dir }}/{{ base_container_name }}.sif"
+    data_dir: "{{ bio_data_dir }}/vep"
+    cache_dir: "/opt/vep/.vep"
+    vep_module_dir: "{{ bio_modules }}/vep"
   tags:
     - "vep"
     - never