From e308292a34225eee152bde8b35148f65496366b9 Mon Sep 17 00:00:00 2001 From: Tony Liang Date: Thu, 13 Aug 2026 21:56:46 +0800 Subject: [PATCH 1/2] fix: move container option to process level config, and remove stale temp var option --- modules/caret_multimodal/predict/main.nf | 5 +---- modules/caret_multimodal/preprocess/main.nf | 7 +----- .../caret_multimodal/select_feature/main.nf | 5 +---- modules/caret_multimodal/train/main.nf | 8 ++----- modules/prepare_data/parse_metadata/main.nf | 7 ++---- modules/prepare_data/prepare_mae_data/main.nf | 6 +---- modules/prepare_data/prepare_mu_data/main.nf | 5 +---- .../prepare_data/uncompress_record/main.nf | 6 ++--- nextflow.config | 22 +++++++++++++++---- 9 files changed, 29 insertions(+), 42 deletions(-) diff --git a/modules/caret_multimodal/predict/main.nf b/modules/caret_multimodal/predict/main.nf index ffada37..e842c78 100755 --- a/modules/caret_multimodal/predict/main.nf +++ b/modules/caret_multimodal/predict/main.nf @@ -3,12 +3,9 @@ include { getPublishPath } from "${modulesDir}/functions" process CARET_MULTIMODAL_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}" debug "${params.debug}" - container "${ onSockeye ? - 'caret_multimodal.sif' : - 'tonyliang19/caret_multimodal:latest' }" + label 'caret_multimodal' publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_name}", diff --git a/modules/caret_multimodal/preprocess/main.nf b/modules/caret_multimodal/preprocess/main.nf index 6ae03c6..e1f7390 100755 --- a/modules/caret_multimodal/preprocess/main.nf +++ b/modules/caret_multimodal/preprocess/main.nf @@ -21,17 +21,12 @@ include { getPublishPath } from "${modulesDir}/functions" // TODO: Replace name here process CARET_MULTIMODAL_PREPROCESS { - // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // debugs true or false by param in MESSI.config tag "${dataset_name}" // identifier of process when ran in parallel // By var before to determine what container to use // Uses apptainer if true otherwise docker - container "${ onSockeye ? - 'caret_multimodal.sif' : - 'tonyliang19/caret_multimodal:latest' }" - + label "caret_multimodal" label "process_low" /* outputs store to outdir for saving and inspecting purpose */ diff --git a/modules/caret_multimodal/select_feature/main.nf b/modules/caret_multimodal/select_feature/main.nf index acd3eb2..85171bb 100755 --- a/modules/caret_multimodal/select_feature/main.nf +++ b/modules/caret_multimodal/select_feature/main.nf @@ -4,13 +4,10 @@ include { getPublishPath } from "${modulesDir}/functions" process CARET_MULTIMODAL_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}" debug true label 'process_high' - container "${ onSockeye ? - 'caret_multimodal.sif' : - 'tonyliang19/caret_multimodal:latest' }" + label 'caret_multimodal' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", diff --git a/modules/caret_multimodal/train/main.nf b/modules/caret_multimodal/train/main.nf index 40fb4eb..d184360 100755 --- a/modules/caret_multimodal/train/main.nf +++ b/modules/caret_multimodal/train/main.nf @@ -17,14 +17,9 @@ include { getPublishPath } from "${modulesDir}/functions" process CARET_MULTIMODAL_TRAIN { - // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') // Identifier for each dataset and fold combination tag "${dataset_name}-${fold_path.name}" - // TODO: rename to the actual image name used - container "${ onSockeye ? - 'caret_multimodal.sif' : - 'tonyliang19/caret_multimodal:latest' }" + // Parse the output directory to migrate results to publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_path.name}", @@ -34,6 +29,7 @@ process CARET_MULTIMODAL_TRAIN { // TODO: add your custom labels to tell if process // consumes large RAM/ROM and gpu access or not label 'process_medium' + label 'caret_multimodal' // TODO: Change arg name to mae_path or mu_path diff --git a/modules/prepare_data/parse_metadata/main.nf b/modules/prepare_data/parse_metadata/main.nf index 05114a5..2222ee1 100644 --- a/modules/prepare_data/parse_metadata/main.nf +++ b/modules/prepare_data/parse_metadata/main.nf @@ -11,16 +11,13 @@ //include { getPublishPath } from "${modulesDir}/functions" // TODO: Replace name here process PARSE_METADATA { - // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // debugs true or false by param in MESSI.config // tag "${dataset_name}" // identifier of process when ran in parallel // By var before to determine what container to use // Uses apptainer if true otherwise docker - container "${ onSockeye ? - 'save_simulate.sif' : - 'tonyliang19/save_simulate:latest'}" + label "generic" + /* outputs store to outdir for saving and inspecting purpose */ publishDir ( diff --git a/modules/prepare_data/prepare_mae_data/main.nf b/modules/prepare_data/prepare_mae_data/main.nf index b97b1cc..3e0b452 100644 --- a/modules/prepare_data/prepare_mae_data/main.nf +++ b/modules/prepare_data/prepare_mae_data/main.nf @@ -8,14 +8,10 @@ */ process PREPARE_MAE_DATA { - def onSockeye = workflow.projectDir.toString().contains('/scratch') /* process metadata and configs */ tag "${dataset_name}" - // TODO: move this containter to somewhere else? label 'process_low' - container "${ onSockeye ? - 'save_simulate.sif' : - 'tonyliang19/save_simulate:latest' }" + label 'generic' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", saveAS: { file }, diff --git a/modules/prepare_data/prepare_mu_data/main.nf b/modules/prepare_data/prepare_mu_data/main.nf index 8b0ebff..47a1c9f 100644 --- a/modules/prepare_data/prepare_mu_data/main.nf +++ b/modules/prepare_data/prepare_mu_data/main.nf @@ -10,10 +10,7 @@ process PREPARE_MU_DATA { /* process metadata and configs */ tag "${dataset_name}" label 'process_low' - // TODO: move this containter to somewhere else? - container "${ onSockeye ? - 'save_simulate.sif' : - 'tonyliang19/save_simulate:latest' }" + label 'generic' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", saveAS: { file }, diff --git a/modules/prepare_data/uncompress_record/main.nf b/modules/prepare_data/uncompress_record/main.nf index de3fbd2..dbb2c43 100644 --- a/modules/prepare_data/uncompress_record/main.nf +++ b/modules/prepare_data/uncompress_record/main.nf @@ -3,10 +3,8 @@ process UNCOMPRESS_RECORD { /* process metadata and configs */ tag "${row_map.dataset_name}" label 'process_low' - // TODO: move this containter to somewhere else? - container "${ onSockeye ? - 'save_simulate.sif' : - 'tonyliang19/save_simulate:latest' }" + label 'generic' + publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${row_map.dataset_name}", saveAS: { file }, diff --git a/nextflow.config b/nextflow.config index bc88ede..3228614 100644 --- a/nextflow.config +++ b/nextflow.config @@ -136,6 +136,9 @@ params { cpu_training_array_size = 1 // Training array size cpu_prediction_array_size = 1 // Prediction array size feature_selection_array_size = 1 // Feature selection array size + + + } // Widely-used environment variables @@ -162,7 +165,7 @@ profiles { docker.enabled = true docker.temp = 'auto' // This is temporal fix - docker.runOptions = "-u \$(id -u):\$(id -g) -v ${projectDir}:/home/rstudio/${projectDir} --entrypoint ''" + docker.runOptions = "-u \$(id -u):\$(id -g) --entrypoint '' -v ${projectDir}:/home/rstudio/${projectDir}" apptainer.enabled = false singularity.enabled = false } @@ -224,9 +227,20 @@ profiles { // Process option cache and their shell option process { - cache = 'lenient' - // Capture exit codes from upstream processes when piping - shell = ['/bin/bash', '-euo', 'pipefail'] + cache = 'lenient' + // Capture exit codes from upstream processes when piping + shell = ['/bin/bash', '-euo', 'pipefail'] + + /* + MODULE CONTAINER CONFIG + */ + + // Assigning module containers here + + withLabel: caret_multimodal { container = 'tonyliang19/caret_multimodal:latest' } + withLabel: generic { container = 'tonyliang19/save_simulate:latest' } + withLabel: mogonet { container = 'tonyliang19/mogonet:latest' } + } // Pipeline information after exectuion From b0c6ba5550e7ceab94a3f63201704f2d01a64ebd Mon Sep 17 00:00:00 2001 From: Tony Liang Date: Tue, 18 Aug 2026 00:21:53 +0800 Subject: [PATCH 2/2] fix: moved all container definition into config --- modules/calculate_metrics/main.nf | 8 ++------ modules/cooperative_learning/predict/main.nf | 5 +---- modules/cooperative_learning/preprocess/main.nf | 5 +---- modules/cooperative_learning/select_feature/main.nf | 7 ++----- modules/cooperative_learning/train/main.nf | 5 +---- modules/diablo/downstream/main.nf | 5 +---- modules/diablo/predict/main.nf | 5 +---- modules/diablo/preprocess/main.nf | 5 +---- modules/diablo/select_feature/main.nf | 5 +---- modules/diablo/train/main.nf | 5 +---- modules/integrao/predict/main.nf | 5 +---- modules/integrao/preprocess/main.nf | 5 +---- modules/integrao/select_feature/main.nf | 5 +---- modules/integrao/train/main.nf | 5 +---- modules/local/samplesheet_check/main.nf | 5 +---- modules/merge_result_table/main.nf | 5 +---- modules/merge_selected_features/main.nf | 5 +---- modules/mofa/predict/main.nf | 5 +---- modules/mofa/preprocess/main.nf | 5 +---- modules/mofa/select_feature/main.nf | 5 +---- modules/mofa/train/main.nf | 5 +---- modules/mogonet/predict/main.nf | 7 ++----- modules/mogonet/preprocess/main.nf | 5 +---- modules/mogonet/select_feature/main.nf | 5 +---- modules/mogonet/train/main.nf | 5 +---- modules/rgcca/predict/main.nf | 9 ++------- modules/rgcca/preprocess/main.nf | 5 +---- modules/rgcca/select_feature/main.nf | 5 +---- modules/rgcca/train/main.nf | 5 +---- modules/sklearn/predict/main.nf | 5 +---- modules/sklearn/preprocess/main.nf | 5 +---- modules/sklearn/select_feature/main.nf | 5 +---- modules/sklearn/train/main.nf | 5 +---- modules/split_train_test/main.nf | 7 ++----- nextflow.config | 7 ++++++- 35 files changed, 45 insertions(+), 145 deletions(-) diff --git a/modules/calculate_metrics/main.nf b/modules/calculate_metrics/main.nf index fb2575e..dfc71c6 100644 --- a/modules/calculate_metrics/main.nf +++ b/modules/calculate_metrics/main.nf @@ -1,11 +1,7 @@ process CALCULATE_METRICS { - def onSockeye = workflow.projectDir.toString().contains('/scratch') debug true label 'process_single' - container "${ onSockeye ? - 'mogonet.sif' : - 'tonyliang19/mogonet:latest' }" - + label 'mogonet' // Should be a generic python container? publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}", mode: 'copy', @@ -28,4 +24,4 @@ process CALCULATE_METRICS { --threshold=${threshold} > \ ${task.process.tokenize(':')[-1].toLowerCase()}.log """ -} \ No newline at end of file +} diff --git a/modules/cooperative_learning/predict/main.nf b/modules/cooperative_learning/predict/main.nf index 674d0f6..dd8724c 100644 --- a/modules/cooperative_learning/predict/main.nf +++ b/modules/cooperative_learning/predict/main.nf @@ -2,13 +2,10 @@ include { getPublishPath } from "${modulesDir}/functions" process COOPERATIVE_LEARNING_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}" debug true label 'process_single' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_name}", diff --git a/modules/cooperative_learning/preprocess/main.nf b/modules/cooperative_learning/preprocess/main.nf index d146759..b405711 100644 --- a/modules/cooperative_learning/preprocess/main.nf +++ b/modules/cooperative_learning/preprocess/main.nf @@ -5,14 +5,11 @@ include { getPublishPath } from "${modulesDir}/functions" process COOPERATIVE_LEARNING_PREPROCESS { // Temp variables - def onSockeye = workflow.projectDir.toString().contains('/scratch') /* Directives for process */ debug "${params.debug}" // default is true tag "${dataset_name}" label 'process_single' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest'}" + label 'codia' // More special publish dir by getting method/method_abc to method/abc publishDir ( diff --git a/modules/cooperative_learning/select_feature/main.nf b/modules/cooperative_learning/select_feature/main.nf index d7836f2..cddec60 100644 --- a/modules/cooperative_learning/select_feature/main.nf +++ b/modules/cooperative_learning/select_feature/main.nf @@ -4,13 +4,10 @@ include { getPublishPath } from "${modulesDir}/functions" process COOPERATIVE_LEARNING_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}" debug true label 'process_high' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", @@ -21,7 +18,7 @@ process COOPERATIVE_LEARNING_SELECT_FEATURE { // Labels label 'low_mem' label 'cpu' - label 'codia' + label 'codia' /* Input and output blocks*/ diff --git a/modules/cooperative_learning/train/main.nf b/modules/cooperative_learning/train/main.nf index bfccb08..6fc9dff 100644 --- a/modules/cooperative_learning/train/main.nf +++ b/modules/cooperative_learning/train/main.nf @@ -7,13 +7,10 @@ include { getPublishPath } from "${modulesDir}/functions" process COOPERATIVE_LEARNING_TRAIN { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_path.name}" debug "${params.debug}" label 'process_high' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' // More special publish dir by getting method/method_abc to method/abc publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_path.name}", diff --git a/modules/diablo/downstream/main.nf b/modules/diablo/downstream/main.nf index 8600457..ce7ddea 100644 --- a/modules/diablo/downstream/main.nf +++ b/modules/diablo/downstream/main.nf @@ -22,16 +22,13 @@ include { getPublishPath } from "${modulesDir}/functions" // TODO: Replace name here process DIABLO_DOWNSTREAM { // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // debugs true or false by param in MESSI.config label 'process_single' tag "${dataset_name}" // identifier of process when ran in parallel // By var before to determine what container to use // Uses apptainer if true otherwise docker - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest'}" + label 'codia' /* outputs store to outdir for saving and inspecting purpose */ publishDir ( diff --git a/modules/diablo/predict/main.nf b/modules/diablo/predict/main.nf index 5e563e7..b11a39d 100644 --- a/modules/diablo/predict/main.nf +++ b/modules/diablo/predict/main.nf @@ -3,13 +3,10 @@ include { getPublishPath } from "${modulesDir}/functions" process DIABLO_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}-${method}" debug true label 'process_single' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' publishDir ( //path: "${params.outdir}/${task.process.tokenize(':')[-1].toLowerCase()}/${dataset_name}/${fold_name}", diff --git a/modules/diablo/preprocess/main.nf b/modules/diablo/preprocess/main.nf index b053580..15b3452 100644 --- a/modules/diablo/preprocess/main.nf +++ b/modules/diablo/preprocess/main.nf @@ -27,16 +27,13 @@ include { getPublishPath } from "${modulesDir}/functions" process DIABLO_PREPROCESS { // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // debugs true or false by param in MESSI.config label 'process_single' tag "${dataset_name}" // identifier of process when ran in parallel // By var before to determine what container to use // Uses apptainer if true otherwise docker - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest'}" + label 'codia' /* outputs store to outdir for saving and inspecting purpose */ publishDir ( diff --git a/modules/diablo/select_feature/main.nf b/modules/diablo/select_feature/main.nf index a465a8d..099a734 100644 --- a/modules/diablo/select_feature/main.nf +++ b/modules/diablo/select_feature/main.nf @@ -4,13 +4,10 @@ include { getPublishPath } from "${modulesDir}/functions" process DIABLO_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-design_${design}-ncomp_${ncomp}" debug true label 'process_high' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", diff --git a/modules/diablo/train/main.nf b/modules/diablo/train/main.nf index 49f5970..e42c039 100644 --- a/modules/diablo/train/main.nf +++ b/modules/diablo/train/main.nf @@ -6,15 +6,12 @@ include { getPublishPath } from "${modulesDir}/functions" process DIABLO_TRAIN { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') def method_name = "diablo" tag "${dataset_name}-${fold_path.name}-design_${design}-ncomp_${ncomp}" //debug "${params.debug}" debug true label 'process_medium' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' // Parse this path publishDir ( //path: "${params.outdir}/${task.process.tokenize(':')[-1].toLowerCase()}/${dataset_name}/${fold_path.name}", diff --git a/modules/integrao/predict/main.nf b/modules/integrao/predict/main.nf index e3ebaeb..c281125 100755 --- a/modules/integrao/predict/main.nf +++ b/modules/integrao/predict/main.nf @@ -3,12 +3,9 @@ include { getPublishPath } from "${modulesDir}/functions" process INTEGRAO_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}" debug "${params.debug}" - container "${ onSockeye ? - 'integrao.sif' : - 'tonyliang19/integrao:latest' }" + label 'integrao' publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_name}", diff --git a/modules/integrao/preprocess/main.nf b/modules/integrao/preprocess/main.nf index a5d4224..671b558 100755 --- a/modules/integrao/preprocess/main.nf +++ b/modules/integrao/preprocess/main.nf @@ -22,15 +22,12 @@ include { getPublishPath } from "${modulesDir}/functions" // TODO: Replace name here process INTEGRAO_PREPROCESS { // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // debugs true or false by param in MESSI.config tag "${dataset_name}" // identifier of process when ran in parallel // By var before to determine what container to use // Uses apptainer if true otherwise docker - container "${ onSockeye ? - 'integrao.sif' : - 'tonyliang19/integrao:latest' }" + label 'integrao' label "process_low" diff --git a/modules/integrao/select_feature/main.nf b/modules/integrao/select_feature/main.nf index 8bf3316..734bb7a 100755 --- a/modules/integrao/select_feature/main.nf +++ b/modules/integrao/select_feature/main.nf @@ -4,14 +4,11 @@ include { getPublishPath } from "${modulesDir}/functions" process INTEGRAO_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}" debug true label 'process_high' label 'gpu' - container "${ onSockeye ? - 'integrao.sif' : - 'tonyliang19/integrao:latest' }" + label 'integrao' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", diff --git a/modules/integrao/train/main.nf b/modules/integrao/train/main.nf index 95ea531..f8c2110 100755 --- a/modules/integrao/train/main.nf +++ b/modules/integrao/train/main.nf @@ -18,13 +18,10 @@ include { getPublishPath } from "${modulesDir}/functions" process INTEGRAO_TRAIN { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') // Identifier for each dataset and fold combination tag "${dataset_name}-${fold_path.name}" // TODO: rename to the actual image name used - container "${ onSockeye ? - 'integrao.sif' : - 'tonyliang19/integrao:latest' }" + label 'integrao' // Parse the output directory to migrate results to publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_path.name}", diff --git a/modules/local/samplesheet_check/main.nf b/modules/local/samplesheet_check/main.nf index c4739b0..4e04f0f 100644 --- a/modules/local/samplesheet_check/main.nf +++ b/modules/local/samplesheet_check/main.nf @@ -2,10 +2,7 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" label 'process_single_low' - def onSockeye = workflow.projectDir.toString().contains('/scratch') - container "${ onSockeye ? - 'mogonet.sif' : - 'tonyliang19/mogonet:latest' }" + label 'mogonet' input: path(samplesheet) diff --git a/modules/merge_result_table/main.nf b/modules/merge_result_table/main.nf index 0227cb8..6794d42 100644 --- a/modules/merge_result_table/main.nf +++ b/modules/merge_result_table/main.nf @@ -1,11 +1,8 @@ process MERGE_RESULT_TABLE { - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${method_name}" debug true label 'process_single' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' publishDir ( //path: "${params.outdir}/${task.process.tokenize(':')[-1].toLowerCase()}/${method_name}", diff --git a/modules/merge_selected_features/main.nf b/modules/merge_selected_features/main.nf index 362fd45..86904fa 100644 --- a/modules/merge_selected_features/main.nf +++ b/modules/merge_selected_features/main.nf @@ -1,11 +1,8 @@ process MERGE_SELECTED_FEATURES { - def onSockeye = workflow.projectDir.toString().contains('/scratch') //tag "${method_name}" debug true label 'process_single' - container "${ onSockeye ? - 'codia.sif' : - 'tonyliang19/codia:latest' }" + label 'codia' publishDir ( //path: "${params.outdir}/${task.process.tokenize(':')[-1].toLowerCase()}/${method_name}", diff --git a/modules/mofa/predict/main.nf b/modules/mofa/predict/main.nf index 0c31e3a..b1d1b26 100644 --- a/modules/mofa/predict/main.nf +++ b/modules/mofa/predict/main.nf @@ -3,13 +3,10 @@ include { getPublishPath } from "${modulesDir}/functions" process MOFA_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}" debug true label 'process_single' - container "${ onSockeye ? - 'mofa.sif' : - 'tonyliang19/mofa:latest' }" + label 'mofa' publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_name}", diff --git a/modules/mofa/preprocess/main.nf b/modules/mofa/preprocess/main.nf index 15c4b80..ff7efbe 100644 --- a/modules/mofa/preprocess/main.nf +++ b/modules/mofa/preprocess/main.nf @@ -27,16 +27,13 @@ include { getPublishPath } from "${modulesDir}/functions" process MOFA_PREPROCESS { // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // debugs true or false by param in MESSI.config label 'process_single' tag "${dataset_name}-num_factor_${num_factors}" // identifier of process when ran in parallel // By var before to determine what container to use // Uses apptainer if true otherwise docker - container "${ onSockeye ? - 'mofa.sif' : - 'tonyliang19/mofa:latest'}" + label 'mofa' /* outputs store to outdir for saving and inspecting purpose */ publishDir ( diff --git a/modules/mofa/select_feature/main.nf b/modules/mofa/select_feature/main.nf index 41054e5..699bab0 100644 --- a/modules/mofa/select_feature/main.nf +++ b/modules/mofa/select_feature/main.nf @@ -4,13 +4,10 @@ include { getPublishPath } from "${modulesDir}/functions" process MOFA_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}" debug true label 'process_high' - container "${ onSockeye ? - 'mofa.sif' : - 'tonyliang19/mofa:latest' }" + label 'mofa' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", diff --git a/modules/mofa/train/main.nf b/modules/mofa/train/main.nf index fbba820..603ddfd 100644 --- a/modules/mofa/train/main.nf +++ b/modules/mofa/train/main.nf @@ -6,14 +6,11 @@ include { getPublishPath } from "${modulesDir}/functions" process MOFA_TRAIN { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_path.name}" //debug "${params.debug}" debug true label 'process_medium' - container "${ onSockeye ? - 'mofa.sif' : - 'tonyliang19/mofa:latest' }" + label 'mofa' // Parse this path publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_path.name}", diff --git a/modules/mogonet/predict/main.nf b/modules/mogonet/predict/main.nf index 6578977..63821f6 100644 --- a/modules/mogonet/predict/main.nf +++ b/modules/mogonet/predict/main.nf @@ -2,13 +2,10 @@ include { getPublishPath } from "${modulesDir}/functions" process MOGONET_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}" debug true label 'process_low' - container "${ onSockeye ? - 'mogonet.sif' : - 'tonyliang19/mogonet:latest' }" + label 'mogonet' publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_name}", @@ -19,7 +16,7 @@ process MOGONET_PREDICT { // Labels label 'low_mem' label 'gpu' - label 'mogonet' + label 'mogonet' input: tuple val(dataset_name), val(fold_name), path(model) diff --git a/modules/mogonet/preprocess/main.nf b/modules/mogonet/preprocess/main.nf index 571212a..8fa0e12 100644 --- a/modules/mogonet/preprocess/main.nf +++ b/modules/mogonet/preprocess/main.nf @@ -31,16 +31,13 @@ include { getPublishPath } from "${modulesDir}/functions" process MOGONET_PREPROCESS { // Temp variables - def onSockeye = workflow.projectDir.toString().contains('/scratch') def output_dir = "fold" // process configurations label 'process_low' debug "${params.debug}" tag "${dataset_name}" // If on sockeye, then use sif file otherwise assuming local (use docker) - container "${ onSockeye ? - 'mogonet.sif' : - 'tonyliang19/save_simulate:latest' }" + label 'mogonet' /* Outputs are stored to outdir (fold), for saving purpose, passing input/output should be done with channels */ publishDir ( diff --git a/modules/mogonet/select_feature/main.nf b/modules/mogonet/select_feature/main.nf index 5833224..3c16934 100644 --- a/modules/mogonet/select_feature/main.nf +++ b/modules/mogonet/select_feature/main.nf @@ -4,13 +4,10 @@ include { getPublishPath } from "${modulesDir}/functions" process MOGONET_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-he${he_base_dim}" debug true label 'process_high' - container "${ onSockeye ? - 'mogonet.sif' : - 'tonyliang19/mogonet:latest' }" + label 'mogonet' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", diff --git a/modules/mogonet/train/main.nf b/modules/mogonet/train/main.nf index 4a8c842..c4d746a 100644 --- a/modules/mogonet/train/main.nf +++ b/modules/mogonet/train/main.nf @@ -11,16 +11,13 @@ include { getPublishPath } from "${modulesDir}/functions" process MOGONET_TRAIN { // This label is defined in nextflow.config // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') // Process configurations tag "${dataset_name}-${fold_path.name}-he${he_base_dim}" label 'python' label 'process_medium' label 'gpu' debug "${params.debug}" - container "${ onSockeye ? - 'mogonet.sif' : - 'tonyliang19/mogonet:latest' }" + label 'mogonet' // Saving outputs publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_path.name}", diff --git a/modules/rgcca/predict/main.nf b/modules/rgcca/predict/main.nf index c6dcf0e..431dd65 100644 --- a/modules/rgcca/predict/main.nf +++ b/modules/rgcca/predict/main.nf @@ -3,14 +3,10 @@ include { getPublishPath } from "${modulesDir}/functions" process RGCCA_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}-${method}" debug "${params.debug}" - label 'process_single' - container "${ onSockeye ? - 'rgcca.sif' : - 'tonyliang19/rgcca:latest' }" - + label 'process_single' + label 'rgcca' publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_name}", mode: 'copy', @@ -23,7 +19,6 @@ process RGCCA_PREDICT { */ label 'low_mem' label 'cpu' - label 'codia' /* Given RGCCA itself has many methods available, so we have an extra input here diff --git a/modules/rgcca/preprocess/main.nf b/modules/rgcca/preprocess/main.nf index 4e5dab9..37126ee 100644 --- a/modules/rgcca/preprocess/main.nf +++ b/modules/rgcca/preprocess/main.nf @@ -22,14 +22,11 @@ include { getPublishPath } from "${modulesDir}/functions" // TODO: Replace name here process RGCCA_PREPROCESS { // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // default is true tag "${dataset_name}" label 'process_single' - container "${ onSockeye ? - 'rgcca.sif' : - 'tonyliang19/rgcca:latest'}" + label 'rgcca' /* outputs store to outdir for saving and inspecting purpose */ publishDir ( diff --git a/modules/rgcca/select_feature/main.nf b/modules/rgcca/select_feature/main.nf index 951d11d..963211b 100644 --- a/modules/rgcca/select_feature/main.nf +++ b/modules/rgcca/select_feature/main.nf @@ -4,13 +4,10 @@ include { getPublishPath } from "${modulesDir}/functions" process RGCCA_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-design_${design}-ncomp_${ncomp}" debug true label 'process_low' - container "${ onSockeye ? - 'rgcca.sif' : - 'tonyliang19/rgcca:latest' }" + label 'rgcca' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", diff --git a/modules/rgcca/train/main.nf b/modules/rgcca/train/main.nf index 306f977..714232d 100644 --- a/modules/rgcca/train/main.nf +++ b/modules/rgcca/train/main.nf @@ -18,14 +18,11 @@ include { getPublishPath } from "${modulesDir}/functions" process RGCCA_TRAIN { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') // Identifier for each dataset and fold combination tag "${dataset_name}-${fold_path.name}-${method}-design_${design}-ncomp_${ncomp}" label 'process_single' // TODO: rename to the actual image name used - container "${ onSockeye ? - 'rgcca.sif' : - 'tonyliang19/rgcca:latest' }" + label 'rgcca' // Parse the output directory to migrate results to publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_path.name}", diff --git a/modules/sklearn/predict/main.nf b/modules/sklearn/predict/main.nf index 44343d8..eb8fe18 100755 --- a/modules/sklearn/predict/main.nf +++ b/modules/sklearn/predict/main.nf @@ -3,12 +3,9 @@ include { getPublishPath } from "${modulesDir}/functions" process SKLEARN_PREDICT { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}-${fold_name}-${model_name}" debug "${params.debug}" - container "${ onSockeye ? - 'sklearn.sif' : - 'tonyliang19/sklearn:latest' }" + label 'sklearn' publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_name}/${model_name}", diff --git a/modules/sklearn/preprocess/main.nf b/modules/sklearn/preprocess/main.nf index f27a4b3..c339ced 100755 --- a/modules/sklearn/preprocess/main.nf +++ b/modules/sklearn/preprocess/main.nf @@ -22,15 +22,12 @@ include { getPublishPath } from "${modulesDir}/functions" // TODO: Replace name here process SKLEARN_PREPROCESS { // temp variables to use - def onSockeye = workflow.projectDir.toString().contains('/scratch') // process level configuration debug "${params.debug}" // debugs true or false by param in MESSI.config tag "${dataset_name}" // identifier of process when ran in parallel // By var before to determine what container to use // Uses apptainer if true otherwise docker - container "${ onSockeye ? - 'sklearn.sif' : - 'tonyliang19/sklearn:latest' }" + label 'sklearn' label "process_low" diff --git a/modules/sklearn/select_feature/main.nf b/modules/sklearn/select_feature/main.nf index 34ecc3f..52a0b0e 100755 --- a/modules/sklearn/select_feature/main.nf +++ b/modules/sklearn/select_feature/main.nf @@ -4,13 +4,10 @@ include { getPublishPath } from "${modulesDir}/functions" process SKLEARN_SELECT_FEATURE { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${model_name}-${dataset_name}" debug true label 'process_high' - container "${ onSockeye ? - 'sklearn.sif' : - 'tonyliang19/sklearn:latest' }" + label 'sklearn' publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}/${dataset_name}", diff --git a/modules/sklearn/train/main.nf b/modules/sklearn/train/main.nf index 0250cb8..e3b3c59 100755 --- a/modules/sklearn/train/main.nf +++ b/modules/sklearn/train/main.nf @@ -18,13 +18,10 @@ include { getPublishPath } from "${modulesDir}/functions" process SKLEARN_TRAIN { // Vars stuff - def onSockeye = workflow.projectDir.toString().contains('/scratch') // Identifier for each dataset and fold combination tag "${dataset_name}-${fold_path.name}-${model_name}" // TODO: rename to the actual image name used - container "${ onSockeye ? - 'sklearn.sif' : - 'tonyliang19/sklearn:latest' }" + label 'sklearn' // Parse the output directory to migrate results to publishDir ( path: "${params.outdir}/${getPublishPath(task.process)}/${dataset_name}/${fold_path.name}", diff --git a/modules/split_train_test/main.nf b/modules/split_train_test/main.nf index 6d74028..658cd37 100644 --- a/modules/split_train_test/main.nf +++ b/modules/split_train_test/main.nf @@ -2,13 +2,10 @@ process SPLIT_TRAIN_TEST { // process metadata and configs // Vars stuff //def isRemote = workflow.containerEngine == 'apptainer' && !workflow.profile == 'standard' - def onSockeye = workflow.projectDir.toString().contains('/scratch') tag "${dataset_name}" label 'process_single' + label 'generic' debug "${params.debug}" - container "${ onSockeye ? - 'save_simulate.sif' : - 'tonyliang19/save_simulate:latest' }" publishDir ( path: "${params.outdir}/${task.process.tokenize(':').join('/').toLowerCase()}", //saveAS: { fn -> fn.endsWith(".log") ? "${id}/$fn" : fn }, @@ -44,4 +41,4 @@ process SPLIT_TRAIN_TEST { touch ${output_dir}/b.txt touch test.log """ -} \ No newline at end of file +} diff --git a/nextflow.config b/nextflow.config index 3228614..7f5b6c9 100644 --- a/nextflow.config +++ b/nextflow.config @@ -240,7 +240,12 @@ process { withLabel: caret_multimodal { container = 'tonyliang19/caret_multimodal:latest' } withLabel: generic { container = 'tonyliang19/save_simulate:latest' } withLabel: mogonet { container = 'tonyliang19/mogonet:latest' } - + withLabel: mofa { container = 'tonyliang19/mofa:latest' } + withLabel: rgcca { container = 'tonyliang19/rgcca:latest' } + withLabel: codia { container = 'tonyliang19/codia:latest' } + withLabel: integrao { container = 'tonyliang19/integrao:latest' } + withLabel: sklearn { container = 'tonyliang19/sklearn:latest' } + } // Pipeline information after exectuion