Skip to content

Implement global output publishing pattern for Nextflow v25.10+ (Issue #41) #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ppreshant wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Implement global output publishing pattern for Nextflow v25.10+ (Issue #41) #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
584 changes: 584 additions & 0 deletions CODE_REVIEW.md
View file
Open in desktop

Large diffs are not rendered by default.

174 changes: 151 additions & 23 deletions main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,111 @@ workflow ORCHESTRATE_SOMATEM {
)

emit:
versions = SOMATEM.out.versions // channel: [ path(versions.yml) ]
clean_reads = SOMATEM.out.clean_reads
key_outputs = SOMATEM.out.key_outputs // channel: [ path(taxonomy_report.tsv) | path(assembly_graph.gfa), path(bandage_image.png) ]
versions = SOMATEM.out.versions // channel: [ path(versions.yml) ]
clean_reads = SOMATEM.out.clean_reads
key_outputs = SOMATEM.out.key_outputs // channel: [ path(taxonomy_report.tsv) | path(assembly_graph.gfa), path(bandage_image.png) ]

// Assembly outputs
mapping = SOMATEM.out.mapping // channel: [ val(meta), path(*.bam) ]
bin_tables = SOMATEM.out.bin_tables // channel: [ path(*.csv) | path(*.tsv) ]
bin_fasta = SOMATEM.out.bin_fasta // channel: [ path(*.fa.gz) ]
assembly = SOMATEM.out.assembly // channel: [ val(meta), path(*.fasta.gz) ]
assembly_gfa = SOMATEM.out.assembly_gfa // channel: [ val(meta), path(*.gfa.gz) ]
assembly_log = SOMATEM.out.assembly_log // channel: [ val(meta), path(*.log) ]
coverage = SOMATEM.out.coverage // channel: [ val(meta), path(*.txt) ]
checkm2_report = SOMATEM.out.checkm2_report // channel: [ val(meta), path(*.tsv) ]

// Annotation outputs
bakta_gff = SOMATEM.out.bakta_gff // channel: [ val(meta), path(*.gff) ]
bakta_tsv = SOMATEM.out.bakta_tsv // channel: [ val(meta), path(*.tsv) ]
bakta_txt = SOMATEM.out.bakta_txt // channel: [ val(meta), path(*.txt) ]

// Taxonomic outputs
singlem_profile = SOMATEM.out.singlem_profile // channel: [ val(meta), path(*.tsv) ]
taxburst_html = SOMATEM.out.taxburst_html // channel: [ val(meta), path(*.html) ]

// Analysis outputs
pigeon_html = SOMATEM.out.pigeon_html // channel: [ val(meta), path(*.html) ]
appraise_summary = SOMATEM.out.appraise_summary // channel: [ val(meta), path(*.tsv) ]

// separate key emits for publishing convenience
mapping = SOMATEM.out.mapping // channel: [ val(meta), path(*.bam) ]
bin_tables = SOMATEM.out.bin_tables // channel: [ path(*.csv) | path(*.tsv) ]
bin_fasta = SOMATEM.out.bin_fasta // channel: [ path(*.fa.gz) ]
}
/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
GLOBAL OUTPUT DEFINITIONS
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/

output {
// Quality Control outputs
clean_reads {
path { meta, _reads -> "preprocessing/${meta.id}" }
}

// Assembly outputs
assembly_fasta {
path { meta, _fasta -> "assembly/${meta.id}" }
}
assembly_gfa {
path { meta, _gfa -> "assembly/${meta.id}" }
}
assembly_logs {
path { meta, _log -> "assembly/${meta.id}/logs" }
}

// Mapping outputs
mapping {
path { meta, _bam -> "mapping/${meta.id}" }

@ppreshant ppreshant Jan 8, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

read again why I chose the other syntax vs this one?

mapping {
        path { "mapping/" }
    }

}
coverage {
path { meta, _cov -> "mapping/${meta.id}/coverage" }
}

// Binning outputs
binning_tables {
path { "binning/tables" }
}
binning_fasta {
path { "binning/fasta" }
}

// Quality assessment
checkm2_reports {
path { meta, _report -> "quality_assessment/${meta.id}" }
}

// Annotation outputs (high-quality bins)
annotation_gff {
path { meta, _gff -> "annotation/${meta.sample_id}/${meta.id}" }
}
annotation_tables {
path { meta, _tsv -> "annotation/${meta.sample_id}/${meta.id}" }
}
annotation_summary {
path { meta, _txt -> "annotation/${meta.sample_id}/${meta.id}" }
}

// Taxonomic profiling outputs
taxonomy_profiles {
path { meta, _profile -> "taxonomy/${meta.id}" }
}
taxonomy_visualizations {
path { meta, _html -> "taxonomy/${meta.id}/visualizations" }
}

// Post-hoc analysis
pigeon_reports {
path { meta, _html -> "post_hoc_analysis/${meta.id}/pigeon" }
}
appraise_reports {
path { meta, _summary -> "post_hoc_analysis/${meta.id}/appraise" }
}

// Pipeline metadata
versions {
path { "pipeline_info" }
}
}

/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
RUN MAIN WORKFLOW
Expand Down Expand Up @@ -99,25 +194,58 @@ workflow {
)

publish:
mapping = ORCHESTRATE_SOMATEM.out.mapping
binning_tables = ORCHESTRATE_SOMATEM.out.bin_tables
binning_fasta = ORCHESTRATE_SOMATEM.out.bin_fasta
}

output {
// Quality control outputs
clean_reads = ORCHESTRATE_SOMATEM.out.clean_reads

mapping {
path { "mapping/" }
}
binning_tables {
path { "binning/tables/" }
}
binning_fasta {
path { "binning/fasta/" }
}
// Assembly outputs
assembly_fasta = ORCHESTRATE_SOMATEM.out.assembly
assembly_gfa = ORCHESTRATE_SOMATEM.out.assembly_gfa
assembly_logs = ORCHESTRATE_SOMATEM.out.assembly_log

// Mapping outputs
mapping = ORCHESTRATE_SOMATEM.out.mapping
coverage = ORCHESTRATE_SOMATEM.out.coverage

// Binning outputs
binning_tables = ORCHESTRATE_SOMATEM.out.bin_tables
binning_fasta = ORCHESTRATE_SOMATEM.out.bin_fasta

// Quality assessment
checkm2_reports = ORCHESTRATE_SOMATEM.out.checkm2_report

// Annotation outputs (high-quality bins)
annotation_gff = ORCHESTRATE_SOMATEM.out.bakta_gff
annotation_tables = ORCHESTRATE_SOMATEM.out.bakta_tsv
annotation_summary = ORCHESTRATE_SOMATEM.out.bakta_txt

// Taxonomic profiling outputs
taxonomy_profiles = ORCHESTRATE_SOMATEM.out.singlem_profile
taxonomy_visualizations = ORCHESTRATE_SOMATEM.out.taxburst_html

// Post-hoc analysis
pigeon_reports = ORCHESTRATE_SOMATEM.out.pigeon_html
appraise_reports = ORCHESTRATE_SOMATEM.out.appraise_summary

// Version tracking
versions = ORCHESTRATE_SOMATEM.out.versions

onComplete:
log.info "=" * 80
log.info "Pipeline execution completed!"
log.info "Status: ${workflow.success ? 'SUCCESS' : 'FAILED'}"
log.info "Completed at: ${workflow.complete}"
log.info "Duration: ${workflow.duration}"
log.info "Results published to: ${params.outdir}"
log.info "=" * 80

onError:
log.error "=" * 80
log.error "Pipeline execution failed!"
log.error "Error message: ${workflow.errorMessage}"
log.error "Failed at: ${workflow.errorReport}"
log.error "=" * 80
}


/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
THE END
Expand Down
5 changes: 2 additions & 3 deletions modules/local/bakta/bakta/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@ process BAKTA_BAKTA {
tag "$meta.id"
label 'process_medium'


// Outputs
publishDir "${params.output_dir}/annotation/${meta.sample_id}/${meta.id}", mode: 'copy', pattern: "*.{embl,faa,ffn,fna,gbff,gff,tsv,txt,json,png,svg}"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

input:
tuple val(meta), path(fasta)
Expand Down
4 changes: 2 additions & 2 deletions modules/local/checkm2/parse/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ process CHECKM2_PARSE {
tag "$meta.id"
label 'process_low'

// Outputs
publishDir "${params.output_dir}/quality/${meta.id}", mode: 'copy', pattern: "*.csv"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

conda "conda-forge::python=3.9 conda-forge::pandas=2.0.3"

Expand Down
6 changes: 2 additions & 4 deletions modules/local/singlem/appraise/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,8 @@ process SINGLEM_APPRAISE {
tag "$meta.id"
label 'process_single'


// Outputs
publishDir "${params.output_dir}/appraise/${meta.id}", mode: 'copy', pattern: "*.tsv"
publishDir "${params.output_dir}/appraise/${meta.id}", mode: 'copy', pattern: "*.svg"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

conda "${moduleDir}/environment.yml"

Expand Down
4 changes: 2 additions & 2 deletions modules/local/singlem/pipe/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ process SINGLEM_PIPE {
tag "${meta.id}_${sample_type}"
label 'process_medium'

// Outputs
publishDir "${params.output_dir}/taxonomy/${meta.id}", mode: 'copy', pattern: "*.csv"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

conda "${moduleDir}/environment.yml"

Expand Down
4 changes: 2 additions & 2 deletions modules/local/taxburst/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ process TAXBURST {
tag "$meta.id"
label 'process_single'

// Outputs
publishDir "${params.output_dir}/taxonomy/${meta.id}", mode: 'copy', pattern: "*.html"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

conda "${moduleDir}/environment.yml"

Expand Down
4 changes: 2 additions & 2 deletions modules/nf-core/checkm2/predict/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ process CHECKM2_PREDICT {
tag "${meta.id}"
label 'process_medium'

// Outputs
publishDir "${params.output_dir}/quality/${meta.id}", mode: 'copy', pattern: "*.tsv"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

conda "${moduleDir}/environment.yml"

Expand Down
6 changes: 2 additions & 4 deletions modules/nf-core/flye/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,8 @@ process FLYE {
tag "$meta.id"
label 'process_high'

// Outputs
publishDir "${params.output_dir}/assembly/${meta.id}", mode: 'copy', pattern: "*.fasta"
publishDir "${params.output_dir}/assembly/${meta.id}", mode: 'copy', pattern: "*.gfa"
publishDir "${params.output_dir}/assembly/${meta.id}", mode: 'copy', pattern: "*.log"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

conda "${moduleDir}/environment.yml"

Expand Down
4 changes: 2 additions & 2 deletions modules/nf-core/samtools/coverage/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ process SAMTOOLS_COVERAGE {
tag "$meta.id"
label 'process_single'

// Outputs
publishDir "${params.output_dir}/mapping/${meta.id}", mode: 'copy', pattern: "*.txt"
// Note: Publishing is now handled globally via output blocks in main.nf
// This provides centralized control and better organization of outputs

conda "${moduleDir}/environment.yml"

Expand Down
26 changes: 22 additions & 4 deletions workflows/somatem.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,28 @@ workflow SOMATEM {
clean_reads = PREPROCESSING.out.clean_reads
key_outputs = ch_key_outputs // channel: [ path(taxonomy_report.tsv) | path(assembly_graph.gfa), path(bandage_image.png) ]

// separate key emits for publishing convenience
mapping = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bam_sorted : channel.empty() // channel: [ val(meta), path(*.bam) ]
bin_tables = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bins_csv.mix(ASSEMBLY_MAGS.out.bins_tsv) : channel.empty() // channel: [ path(*.csv) | path(*.tsv) ]
bin_fasta = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bins : channel.empty() // channel: [ path(*.fa.gz) ]
// Assembly outputs
mapping = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bam_sorted : channel.empty() // channel: [ val(meta), path(*.bam) ]
bin_tables = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bins_csv.mix(ASSEMBLY_MAGS.out.bins_tsv) : channel.empty() // channel: [ path(*.csv) | path(*.tsv) ]
bin_fasta = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bins : channel.empty() // channel: [ path(*.fa.gz) ]
assembly = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.assembly : channel.empty() // channel: [ val(meta), path(*.fasta.gz) ]
assembly_gfa = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.assembly_gfa : channel.empty() // channel: [ val(meta), path(*.gfa.gz) ]
assembly_log = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.assembly_log : channel.empty() // channel: [ val(meta), path(*.log) ]
coverage = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.coverage : channel.empty() // channel: [ val(meta), path(*.txt) ]
checkm2_report = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.checkm2_report : channel.empty() // channel: [ val(meta), path(*.tsv) ]

// Annotation outputs (high-quality bins only)
bakta_gff = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bakta_gff : channel.empty() // channel: [ val(meta), path(*.gff) ]
bakta_tsv = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bakta_tsv : channel.empty() // channel: [ val(meta), path(*.tsv) ]
bakta_txt = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.bakta_txt : channel.empty() // channel: [ val(meta), path(*.txt) ]

// Taxonomic profiling outputs
singlem_profile = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.singlem_profile : channel.empty() // channel: [ val(meta), path(*.tsv) ]

@ppreshant ppreshant Jan 8, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Need to change params.analysis_type to taxonomic-profiling

taxburst_html = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.taxburst_html : channel.empty() // channel: [ val(meta), path(*.html) ]

// Post-hoc analysis
pigeon_html = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.pigeon_html : channel.empty() // channel: [ val(meta), path(*.html) ]
appraise_summary = params.analysis_type == "assembly" ? ASSEMBLY_MAGS.out.appraise_summary : channel.empty() // channel: [ val(meta), path(*.tsv) ]
}

/*
Expand Down