Skip to content

Adding resource estimations for multimouse tasks #58

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SkepticRaven merged 9 commits into from
Sep 3, 2025
Merged

Adding resource estimations for multimouse tasks #58

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
4688a3e
Adding resource estimations for multimouse tasks
SkepticRaven Aug 5, 2025
ca7b236
Merge branch 'main' into YODA-111-resource-tuning-of-multimouse-pose-…
SkepticRaven Aug 26, 2025
ff2ab3c
Adjusting containers to benchmark tracklet generation
SkepticRaven Aug 27, 2025
6d88e57
Dynamic resources referencing wrong var
SkepticRaven Aug 27, 2025
05db4c0
.hr -> .hours
SkepticRaven Aug 27, 2025
cec2feb
Making dynamic resource equations simpler to read.
SkepticRaven Sep 2, 2025
d211027
Reverting pretty format because nextflow seems to not parse the funct…
SkepticRaven Sep 2, 2025
28cf1b6
Max with only ints, not floats or doubles.
SkepticRaven Sep 2, 2025
2a8585e
Adjusting to 16GB start
SkepticRaven Sep 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 120 additions & 21 deletions nextflow/configs/profiles/sumner2.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,15 @@ process {
// gpu_training: 14day limit, 16gpu/user
clusterOptions = '-q gpu_inference --gres gpu:1 --nice=64000000'
containerOptions = "--nv"
resourceLimits = [ cpus: 6, memory: 63.GB, time: 6.h ]
// Fair sharing of memory should be 63GB, but full node has 2TB
resourceLimits = [ cpus: 6, memory: 128.GB, time: 6.h ]
}
withLabel: "gpu_long" {
queue = "gpu_a100_mig"
clusterOptions = '-q gpu_training --gres gpu:1 --nice=64000000'
containerOptions = "--nv"
// Fair sharing of memory should be 63GB, but full node has 2TB
resourceLimits = [ cpus: 6, memory: 128.GB, time: 14.d ]
}
// Alternative GPU profiles available on the cluster
withLabel: "gpu_a100" {
Expand Down Expand Up @@ -183,7 +191,7 @@ process {
* Runtime options
*/
withLabel: "tracking" {
container = "/projects/kumar-lab/multimouse-pipeline/deployment-runtime_2025-03-05.sif"
container = "/projects/kumar-lab/multimouse-pipeline/nextflow-containers/deployment-runtime_2025-08-26.sif"
}
withLabel: "jabs_classify" {
container = "/projects/kumar-lab/multimouse-pipeline/nextflow-containers/JABS-GUI_2025-02-12_v0.18.1.sif"
Expand All @@ -208,7 +216,7 @@ process {
}
withLabel: "rclone" {
// executor.queueSize = 1
container = "/projects/kumar-lab/multimouse-pipeline/rclone.sif"
container = "/projects/kumar-lab/multimouse-pipeline/rclone_2025-08-18.sif"
}

/*
Expand Down Expand Up @@ -238,7 +246,10 @@ process {
cpus = 1
memory = { 1.GB * task.attempt }
// 0.5 * t_min + 10.5
time = { ((0.5 * params.clip_duration / 30 / 60 + 10.5) * 1.5).toInteger() + '.sec' * task.attempt }
time = {
def r_value = Math.max(60, ((0.5 * (params.clip_duration / 30 / 60) + 10.5) * 1.5 * task.attempt).toInteger())
return r_value + '.sec'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
Expand All @@ -247,7 +258,10 @@ process {
cpus = 2
memory = { 4.GB * task.attempt }
// 2.5 * t_min + 5
time = { ((2.5 * params.clip_duration / 30 / 60 + 5) * 1.5).toInteger() + '.sec' * task.attempt }
time = {
def r_value = Math.max(60, ((2.5 * (params.clip_duration / 30 / 60) + 5) * 1.5 * task.attempt).toInteger())
return r_value + '.sec'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
Expand All @@ -257,13 +271,13 @@ process {
cpus = 1
// 0.0048 * t_min + 0.080
memory = {
def base_mb = Math.max(1024, ((0.0048 * params.clip_duration / 30 / 60 + 0.08) * 1.5 * 1024).toInteger())
return (base_mb * task.attempt).MB
def r_value = Math.max(1, ((0.0048 * (params.clip_duration / 30 / 60) + 0.08) * 1.5 * task.attempt).toInteger())
return r_value + '.GB'
}
// 0.6 * t_min + 30
time = {
def base_min = Math.max(1, ((0.6 * params.clip_duration / 30 / 60 + 30) * 1.5 / 60).toInteger())
return (base_min * task.attempt).min
def r_value = Math.max(10, ((0.6 * (params.clip_duration / 30 / 60) + 30) * 1.5 * task.attempt).toInteger())
return r_value + '.min'
}
array = 200
errorStrategy = 'retry'
Expand All @@ -272,7 +286,10 @@ process {
withLabel: "r_rearpaw" {
cpus = 1
// 0.0047 * t_min + 0.0801
memory = { ((0.0047 * params.clip_duration / 30 / 60 + 0.0801) * 1.5).toInteger() + '.GB' * task.attempt }
memory = {
def r_value = Math.max(1, ((0.0047 * (params.clip_duration / 30 / 60) + 0.0801) * 1.5 * task.attempt).toInteger())
return r_value + '.GB'
}
time = { 10.min * task.attempt }
array = 200
errorStrategy = 'retry'
Expand All @@ -282,7 +299,10 @@ process {
withLabel: "r_gait_h5" {
cpus = 1
// 0.0009 * t_min + 0.727
memory = { ((0.0009 * params.clip_duration / 30 / 60 + 0.727) * 1.5).toInteger() + '.GB' * task.attempt }
memory = {
def r_value = Math.max(1, ((0.0009 * (params.clip_duration / 30 / 60) + 0.727) * 1.5 * task.attempt).toInteger())
return r_value + '.GB'
}
time = { 10.min * task.attempt }
array = 200
errorStrategy = 'retry'
Expand All @@ -291,7 +311,10 @@ process {
withLabel: "r_gait_bin" {
cpus = 1
// 0.000005 * t_min + 0.0095
memory = { ((0.000005 * params.clip_duration / 30 / 60 + 0.0095) * 1.5).toInteger() + '.GB' * task.attempt }
memory = {
def r_value = Math.max(1, ((0.000005 * (params.clip_duration / 30 / 60) + 0.0095) * 1.5 * task.attempt).toInteger())
return r_value + '.GB'
}
time = { 5.min * task.attempt }
array = 200
errorStrategy = 'retry'
Expand All @@ -304,9 +327,15 @@ process {
// are more computationally expensive than per-frame features (calculated once per pose file)
cpus = 1
// 0.182 * t_min - 0.273 for 6 windows
memory = { ((0.182 * params.clip_duration / 30 / 60) * 2.5).toInteger() + '.GB' * task.attempt }
memory = {
def r_value = Math.max(1, ((0.182 * (params.clip_duration / 30 / 60) - 0.273) * 2.5 * task.attempt).toInteger())
return r_value + '.GB'
}
// 162 * t_min + 4.6 for 6 windows
time = { ((162 * params.clip_duration / 30 / 60 + 4.6) * 2.5).toInteger() + '.sec' * task.attempt }
time = {
def r_value = Math.max(1, ((162 * (params.clip_duration / 30 / 60) + 4.6) * 2.5 * task.attempt).toInteger())
return r_value + '.sec'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
Expand All @@ -316,9 +345,15 @@ process {
// Adding more classifiers will scale linearly
cpus = 1
// 0.429 * t_min + 0.3 for 10 classifiers
memory = { ((0.429 * params.clip_duration / 30 / 60 + 0.3) * 2.5).toInteger() + '.GB' * task.attempt }
memory = {
def r_value = Math.max(1, ((0.429 * (params.clip_duration / 30 / 60) + 0.3) * 2.5 * task.attempt).toInteger())
return r_value + '.GB'
}
// 30 * t_min + 127 for 10 classifiers
time = { ((30 * params.clip_duration / 30 / 60 + 127) * 2.5).toInteger() + '.sec' * task.attempt }
time = {
def r_value = Math.max(1, ((127 + 30 * (params.clip_duration / 30 / 60)) * 2.5 * task.attempt).toInteger())
return r_value + '.sec'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
Expand All @@ -337,7 +372,10 @@ process {
cpus = 1
memory = { 1.GB * task.attempt }
// 0.111 * t_min + 37 for 6 classifiers
time = { ((0.111 * params.clip_duration / 30 / 60 + 37) * 1.5).toInteger() + '.sec' * task.attempt }
time = {
def r_value = Math.max(1, ((0.111 * (params.clip_duration / 30 / 60) + 37) * 2.5 * task.attempt).toInteger())
return r_value + '.sec'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
Expand All @@ -359,9 +397,15 @@ process {
// because the shape of matrices grows by the prediction data
// Equation provided, but numbers were manually increased
// 0.06 * t_min + 1.86
memory = { ((0.06 * params.clip_duration / 30 / 60 + 2) * 2.5).toInteger() + '.GB' * task.attempt }
memory = {
def r_value = Math.max(1, ((0.06 * (params.clip_duration / 30 / 60) + 1.86) * 2.5 * task.attempt).toInteger())
return r_value + '.GB'
}
// 15 * t_min - 63
time = { ((15 * params.clip_duration / 30 / 60 + 0) * 2.5).toInteger() + '.sec' * task.attempt }
time = {
def r_value = Math.max(1, ((15 * (params.clip_duration / 30 / 60) - 63) * 2.5 * task.attempt).toInteger())
return r_value + '.sec'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
Expand All @@ -388,13 +432,68 @@ process {
withLabel: "r_clip_video" {
cpus = 2
// 0.0000003 * t_min + 0.35
memory = { ((0.0000003 * params.clip_duration / 30 / 60 + 0.35) * 1.5).toInteger() + '.GB' * task.attempt }
memory = {
def r_value = Math.max(1, ((0.0000003 * (params.clip_duration / 30 / 60) + 0.35) * 2.5 * task.attempt).toInteger())
return r_value + '.GB'
}
// 19.8 * t_min + 52
time = { ((19.8 * params.clip_duration / 30 / 60 + 52) * 2.5).toInteger() + '.sec' * task.attempt }
time = {
def r_value = Math.max(1, ((52 + 19.8 * (params.clip_duration / 30 / 60)) * 2.5 * task.attempt).toInteger())
return r_value + '.sec'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
}
// Multi Mouse Resources
// These processes assume the input video is 1hr and scale with number of animals
withLabel: "r_multi_seg" {
cpus = 2
// 8.53 GB * num_mice
memory = {
def r_value = Math.max(1, ((8.53 * params.num_mice) * 2.5 * task.attempt).toInteger())
return r_value + '.GB'
}
time = { 5.hours * task.attempt }
array = 200
errorStrategy = 'retry'
maxRetries = 3
}
withLabel: "r_multi_keypoints" {
cpus = 2
// 2.26 GB * num_mice + 2.47 GB
memory = {
def r_value = Math.max(1, ((2.26 * params.num_mice + 2.47) * 2.5 * task.attempt).toInteger())
return r_value + '.GB'
}
// 6 * num_mice + 3
time = {
def r_value = Math.max(1, ((3 + 6 * params.num_mice) * 2.5 * task.attempt).toInteger())
return r_value + '.hours'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
}
withLabel: "r_multi_identity" {
cpus = 2
memory = { 2.GB * task.attempt }
// 0.1 * num_mice + 0.16
time = {
def r_value = Math.max(1, ((0.1 * params.num_mice + 0.16) * 2.5 * task.attempt).toInteger())
return r_value + '.hours'
}
array = 200
errorStrategy = 'retry'
maxRetries = 3
}
withLabel: "r_multi_tracklets" {
cpus = 1
// TODO: Tune these numbers based on number of mice
// First video suggested ~7GB and ~30min for 3 mice
memory = { 10.GB * task.attempt }
Comment thread
SkepticRaven marked this conversation as resolved.
Outdated
time = { 45.min * task.attempt }
}
// Static Object Resources
// These do not scale with clip duration
withLabel: "r_arena_corners" {
Expand Down
16 changes: 10 additions & 6 deletions nextflow/modules/multi_mouse.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
process PREDICT_MULTI_MOUSE_SEGMENTATION {
label "gpu"
label "gpu_long"
label "tracking"

label "r_multi_seg"

input:
tuple path(video_file), path(in_pose)

Expand All @@ -16,9 +17,10 @@ process PREDICT_MULTI_MOUSE_SEGMENTATION {
}

process PREDICT_MULTI_MOUSE_KEYPOINTS {
label "gpu"
label "gpu_long"
label "tracking"

label "r_multi_keypoints"

input:
tuple path(video_file), path(in_pose)

Expand All @@ -35,7 +37,8 @@ process PREDICT_MULTI_MOUSE_KEYPOINTS {
process PREDICT_MULTI_MOUSE_IDENTITY {
label "gpu"
label "tracking"

label "r_multi_identity"

input:
tuple path(video_file), path(in_pose)

Expand All @@ -52,7 +55,8 @@ process PREDICT_MULTI_MOUSE_IDENTITY {
process GENERATE_MULTI_MOUSE_TRACKLETS {
label "cpu"
label "tracking"

label "r_multi_tracklets"

input:
tuple path(video_file), path(in_pose)
val num_animals
Expand Down