From 4f14dc7e85b873fe5b625d86519c5836a7fe5b07 Mon Sep 17 00:00:00 2001 From: cyfraysse Date: Wed, 1 Jul 2026 18:01:24 +0200 Subject: [PATCH 1/3] feat: native restart support (wall-time stop, resume, append) Add the engine primitives so a simulation can run longer than an HPC walltime by stopping cleanly and resuming later: - run!(; wall_time): stop on a wall-clock budget, return :completed or :need_restart; short-circuit guard for schedulers that end before steps - Simulation gains a t_start field/kwarg: resume at t_start and skip already-completed schedule points - Store{Callbacks,Trajectories,Acceptance} open output files in append mode on restart (t_start > 0); file opening moved from constructors into initialise; store_first no longer re-fires on restart; summary.log appended Tests: test/restart_test.jl (5 testsets, 20 assertions), wired into suite. --- src/algorithms.jl | 20 +++---- src/metropolis.jl | 8 +-- src/simulation.jl | 41 ++++++++++----- test/restart_test.jl | 121 +++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 4 ++ 5 files changed, 162 insertions(+), 32 deletions(-) create mode 100644 test/restart_test.jl diff --git a/src/algorithms.jl b/src/algorithms.jl index 7c856a5..4c0cfdb 100644 --- a/src/algorithms.jl +++ b/src/algorithms.jl @@ -106,11 +106,6 @@ struct StoreCallbacks{V} <: AriannaAlgorithm for c in eachindex(chains) paths[c] = joinpath.(dirs[c], cb_names) files[c] = Vector{IOStream}(undef, length(callbacks)) - try - files[c] = open.(paths[c], "w") - catch e - rethrow(e) - end end return new{V}(callbacks, paths, files, store_first, store_last) @@ -129,9 +124,10 @@ function initialise(algorithm::StoreCallbacks, simulation::Simulation) simulation.verbose && println("Opening callback files...") for c in eachindex(algorithm.files) - algorithm.files[c] .= open.(algorithm.paths[c], "w") + writing_mode = simulation.t_start > 0 ? "a" : "w" + algorithm.files[c] .= open.(algorithm.paths[c], writing_mode) end - algorithm.store_first && make_step!(simulation, algorithm) + (algorithm.store_first && simulation.t_start == 0) && make_step!(simulation, algorithm) return nothing end @@ -210,11 +206,6 @@ struct StoreTrajectories{F<:Format} <: AriannaAlgorithm ext = fmt.extension paths = joinpath.(dirs, "trajectory$(ext)") files = Vector{IOStream}(undef, length(paths)) - try - files = open.(paths, "w") - finally - close.(files) - end return new{typeof(fmt)}(paths, files, fmt, store_first, store_last) end @@ -236,8 +227,9 @@ end function initialise(algorithm::StoreTrajectories, simulation::Simulation) simulation.verbose && println("Opening trajectory files...") - algorithm.files .= open.(algorithm.paths, "w") - algorithm.store_first && make_step!(simulation, algorithm) + writing_mode = simulation.t_start > 0 ? "a" : "w" + algorithm.files .= open.(algorithm.paths, writing_mode) + (algorithm.store_first && simulation.t_start == 0) && make_step!(simulation, algorithm) return nothing end diff --git a/src/metropolis.jl b/src/metropolis.jl index 89ee9e5..4fa4c8d 100644 --- a/src/metropolis.jl +++ b/src/metropolis.jl @@ -524,11 +524,6 @@ struct StoreAcceptance <: AriannaAlgorithm mkpath.(dirs) paths = joinpath.(dirs, "acceptance.dat") files = Vector{IOStream}(undef, length(paths)) - try - files = open.(paths, "w") - finally - close.(files) - end return new(paths, files, ids) end end @@ -545,7 +540,8 @@ end function initialise(algorithm::StoreAcceptance, simulation::Simulation) simulation.verbose && println("Opening acceptance files...") - algorithm.files .= open.(algorithm.paths, "w") + writing_mode = simulation.t_start > 0 ? "a" : "w" + algorithm.files .= open.(algorithm.paths, writing_mode) return nothing end diff --git a/src/simulation.jl b/src/simulation.jl index 170c2f7..3697018 100644 --- a/src/simulation.jl +++ b/src/simulation.jl @@ -8,6 +8,7 @@ A structure representing a Monte Carlo simulation. - `algorithms::A`: List of algorithms. - `steps::Int`: Number of MC sweeps. - `t::Int`: Current time step. +- `t_start::Int`: time step to start the simulation from. - `schedulers::VS`: List of schedulers (one for each algorithm). - `counters::Vector{Int}`: Counters for the schedulers (one for each algorithm). - `path::String`: Simulation path. @@ -18,6 +19,7 @@ mutable struct Simulation{S,A,VS} algorithms::A steps::Int t::Int + t_start::Int schedulers::VS counters::Vector{Int} path::String @@ -31,6 +33,7 @@ mutable struct Simulation{S,A,VS} - `algorithms::A`: List of algorithms. - `schedulers::VS`: List of schedulers (one for each algorithm). - `steps::Int`: Number of MC sweeps. + - `t_start::Int=0`: starting time of the simulation - `path::String="data"`: Simulation path. - `verbose::Bool=false`: Flag for verbose output. """ @@ -39,22 +42,23 @@ mutable struct Simulation{S,A,VS} algorithms::A, schedulers::VS, steps::Int; + t_start::Int=0, path::String="data", verbose::Bool=false ) where {S<:AriannaSystem,A,VS} @assert length(schedulers) == length(algorithms) @assert all(scheduler -> all(x -> 0 ≤ x ≤ steps, scheduler), schedulers) @assert all(scheduler -> issorted(scheduler), schedulers) - t = 0 - counters = [findfirst(x -> x > 0, scheduler) for scheduler in schedulers] + t = t_start # initial time for the simulation 0 by default + counters = [findfirst(x -> x > t_start, scheduler) for scheduler in schedulers] mkpath(path) - return new{S,A,VS}(chains, algorithms, steps, t, schedulers, counters, path, verbose) + return new{S,A,VS}(chains, algorithms, steps, t, t_start, schedulers, counters, path, verbose) end end """ - Simulation(chains, algorithm_list, steps; path="data", verbose=false) + Simulation(chains, algorithm_list, steps; t_start=0 path="data", verbose=false) Create a new `Simulation` instance from a list of algorithm constructors. @@ -62,10 +66,11 @@ Create a new `Simulation` instance from a list of algorithm constructors. - `chains`: Vector of independent Arianna systems. - `algorithm_list`: List of algorithm constructors. - `steps`: Number of MC sweeps. +- `t_start=0`: initial time for the simulation. - `path="data"`: Simulation path. - `verbose=false`: Flag for verbose output. """ -function Simulation(chains, algorithm_list, steps; path="data", verbose=false) +function Simulation(chains, algorithm_list, steps; t_start=0, path="data", verbose=false) schedulers_tmp = [] algorithms_tmp = [] algorithm_names = [] @@ -84,7 +89,7 @@ function Simulation(chains, algorithm_list, steps; path="data", verbose=false) end schedulers = ntuple(k -> schedulers_tmp[k], length(schedulers_tmp)) algorithms = ntuple(k -> algorithms_tmp[k], length(algorithms_tmp)) - return Simulation(chains, algorithms, schedulers, steps; path=path, verbose=verbose) + return Simulation(chains, algorithms, schedulers, steps; t_start=t_start, path=path, verbose=verbose) end """ @@ -160,11 +165,13 @@ function write_system(io, system::AriannaSystem) end function write_summary(simulation) - open(joinpath(simulation.path, "summary.log"), "w") do file + writing_mode = simulation.t_start > 0 ? "a" : "w" # idiomatic code for if t_start > 0 "a" else "w" + open(joinpath(simulation.path, "summary.log"), writing_mode) do file println(file, "SIMULATION SUMMARY") println(file) println(file, "Simulation:") println(file, "\tSteps: $(simulation.steps)") + println(file, "\tInitial time: $(simulation.t_start)") println(file, "\tNumber of chains: $(length(simulation.chains))") println(file, "\tNumber of algorithms: $(length(simulation.algorithms))") println(file, "\tVerbose: $(simulation.verbose)") @@ -203,14 +210,17 @@ function finalise_summary(simulation) end """ - run!(simulation::Simulation) + run!(simulation::Simulation; wall_time::Real=Inf) Run the Monte Carlo simulation. # Arguments - `simulation::Simulation`: The simulation instance to run. +- `wall_time::Real=Inf`: to set a wall_time (in seconds) that trigger a restart by default wall_time is Inf """ -function run!(simulation::Simulation) +function run!(simulation::Simulation; wall_time::Real=Inf) + t0 = time() # initial measure of time in seconds + status = :completed # default try simulation.verbose && println("\n" * "-"^50) simulation.verbose && println("\033[1;32mINITIALISATION\033[0m") @@ -219,13 +229,20 @@ function run!(simulation::Simulation) end write_summary(simulation) simulation.verbose && println("\033[1;32m\nRUNNING SIMULATION...\033[0m") - sim_time = @elapsed for simulation.t in 1:simulation.steps + sim_time = @elapsed for simulation.t in (simulation.t_start+1):simulation.steps for k in eachindex(simulation.algorithms) - if simulation.t == simulation.schedulers[k][simulation.counters[k]] + # Guard against a scheduler that ends before `steps`: once its counter + # runs past the last entry, `schedulers[k][counters[k]]` would be out of + # bounds. `&&` short-circuits, so we only index when the counter is valid. + if simulation.counters[k] ≤ length(simulation.schedulers[k]) && simulation.t == simulation.schedulers[k][simulation.counters[k]] make_step!(simulation, simulation.algorithms[k]) simulation.counters[k] += 1 end end + if time()-t0 ≥ wall_time + status = :need_restart # switch to need_restart if wall_time is hit by the simulation + break # exit the `for simulation.t` loop + end end simulation.verbose && println("\nSimulation completed in $(sim_time) s") update_summary(simulation, sim_time) @@ -238,7 +255,7 @@ function run!(simulation::Simulation) simulation.verbose && println("\033[1;32m\nDONE\033[0m") simulation.verbose && println("-"^50 * "\n") end - return nothing + return status end nothing diff --git a/test/restart_test.jl b/test/restart_test.jl new file mode 100644 index 0000000..ea4fe4f --- /dev/null +++ b/test/restart_test.jl @@ -0,0 +1,121 @@ +using Arianna +using Test +using DelimitedFiles + +# particle_1d.jl defines the `Particle` system, `Displacement`, `StandardGaussian` +# and `energy`, but it calls `potential(x)` without defining it — the includer must. +include("../examples/particle_1d/particle_1d.jl") + +potential(x) = x^2 + +# Helper: build a harmonic-oscillator simulation with `steps` sweeps. +# `t_start` lets us build a *resumed* simulation; `sampletimes` is an explicit +# schedule so we can check which points get skipped on resume. +function make_simulation(steps; path, M=2, seed=42, t_start=0, sampletimes=[steps], store_first=true) + rng = Xoshiro(seed) + β = 2.0 + chains = [System(4rand(rng) - 2, β) for _ in 1:M] + pool = (Move(Displacement(0.0), StandardGaussian(), ComponentArray(σ=0.1), 1.0),) + algorithm_list = ( + (algorithm=Metropolis, pool=pool, seed=seed, parallel=false), + (algorithm=StoreCallbacks, callbacks=(energy,), scheduler=sampletimes, store_first=store_first), + (algorithm=StoreLastFrames, scheduler=[steps]), + ) + return Simulation(chains, algorithm_list, steps; t_start=t_start, path=path, verbose=false) +end + +@testset "run! stops early when the wall_time budget is exceeded" begin + steps = 10^6 # large: a full run would take a while + path = "data/restart_test/early_stop" + simulation = make_simulation(steps; path=path) + + # wall_time = 0.0 makes this deterministic: the check `time() - t0 ≥ wall_time` + # is already true after the very first *complete* step, so it always breaks at t = 1. + # (No sleep, no reliance on how fast the machine is → not a flaky test.) + status = run!(simulation; wall_time=0.0) + + @test status == :need_restart # it reported *why* it stopped + @test 1 ≤ simulation.t < steps # a complete step ran, but not the whole run + + # The `finally` block must still run on the way out → the checkpoint frame exists. + @test isfile(joinpath(path, "chains", "1", "lastframe.dat")) +end + +@testset "run! completes and reports :completed when there is no time limit" begin + steps = 1000 # small: runs to the end quickly + path = "data/restart_test/complete" + simulation = make_simulation(steps; path=path) + + status = run!(simulation) # default wall_time = Inf → never times out + + @test status == :completed + @test simulation.t == steps # the loop reached the last step +end + +@testset "run! resumes from t_start and skips already-done schedule points" begin + steps = 1000 + t_start = 500 + sampletimes = collect(100:100:1000) # [100, 200, …, 1000] + path = "data/restart_test/resume" + simulation = make_simulation(steps; path=path, t_start=t_start, sampletimes=sampletimes) + + # The constructor should have placed us at t_start. + @test simulation.t == t_start + @test simulation.t_start == t_start + + # Each scheduler's counter should point at the FIRST time strictly after + # t_start, i.e. everything ≤ t_start is skipped (already done before restart). + for k in eachindex(simulation.schedulers) + c = simulation.counters[k] + sched = simulation.schedulers[k] + @test sched[c] > t_start # next scheduled point is after t_start + c > 1 && @test sched[c-1] ≤ t_start # and the one before it was ≤ t_start + end + + # A resumed run still finishes cleanly and reaches the end. + status = run!(simulation) + @test status == :completed + @test simulation.t == steps +end + +@testset "restart appends to data files instead of truncating them" begin + path = "data/restart_test/append" + steps_full = 1000 + sampletimes_full = collect(100:100:steps_full) # [100, …, 1000] + t_stop = 500 + energy_path = joinpath(path, "chains", "1", "energy.dat") + + # Segment A — fresh first job, with store_first=true (the DEFAULT). A fresh + # run records the initial config at t=0, then samples the schedule ≤ t_stop. + sampletimes_A = filter(t -> t ≤ t_stop, sampletimes_full) # [100, …, 500] + simA = make_simulation(t_stop; path=path, t_start=0, sampletimes=sampletimes_A) + run!(simA) + times_A = Int.(readdlm(energy_path)[:, 1]) + @test times_A == vcat(0, sampletimes_A) # t=0 initial frame + 100…500 + + # Segment B — the restart job: full schedule, resuming at t_stop, SAME path. + # t_start > 0 ⇒ initialise opens energy.dat in APPEND mode AND must NOT re-fire + # store_first (that would duplicate the seam). (Reloading the configuration is + # the downstream package's job; here we only test Arianna's file behaviour.) + simB = make_simulation(steps_full; path=path, t_start=t_stop, sampletimes=sampletimes_full) + run!(simB) + times_all = Int.(readdlm(energy_path)[:, 1]) + + @test times_all == vcat(0, sampletimes_full) # A survived + B appended, single t=0 + @test issorted(times_all) # increasing across the seam + @test allunique(times_all) # no duplicate (no second t=0, no second t=500) +end + +@testset "run! survives a schedule that ends before steps (no BoundsError)" begin + steps = 500 + # This callback schedule stops at 300, well before steps=500. After t=300 its + # counter runs past the last entry; without the guard in run!, indexing + # schedulers[k][counters[k]] at t=301 would throw a BoundsError. + sampletimes = [100, 200, 300] + path = "data/restart_test/short_schedule" + simulation = make_simulation(steps; path=path, sampletimes=sampletimes) + + status = run!(simulation) # must run to the end without erroring + @test status == :completed + @test simulation.t == steps +end diff --git a/test/runtests.jl b/test/runtests.jl index 8d71681..b782330 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,6 +14,10 @@ end include("ad_backends_test.jl") end +@safetestset "Restart Test" begin + include("restart_test.jl") +end + @safetestset "Quality Assurance" begin using Aqua using Arianna From 22f37091dd9165f9bf339a8c97b16dca8f150899 Mon Sep 17 00:00:00 2001 From: cyfraysse Date: Fri, 2 Oct 2026 16:39:39 +0200 Subject: [PATCH 2/3] fix asked by the reviewers : status to :running by default and edge cases clarification --- src/simulation.jl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/simulation.jl b/src/simulation.jl index 3697018..8da7114 100644 --- a/src/simulation.jl +++ b/src/simulation.jl @@ -50,7 +50,7 @@ mutable struct Simulation{S,A,VS} @assert all(scheduler -> all(x -> 0 ≤ x ≤ steps, scheduler), schedulers) @assert all(scheduler -> issorted(scheduler), schedulers) t = t_start # initial time for the simulation 0 by default - counters = [findfirst(x -> x > t_start, scheduler) for scheduler in schedulers] + counters = [something(findfirst(x -> x > t_start, scheduler),1) for scheduler in schedulers] mkpath(path) return new{S,A,VS}(chains, algorithms, steps, t, t_start, schedulers, counters, path, verbose) end @@ -165,7 +165,7 @@ function write_system(io, system::AriannaSystem) end function write_summary(simulation) - writing_mode = simulation.t_start > 0 ? "a" : "w" # idiomatic code for if t_start > 0 "a" else "w" + writing_mode = simulation.t_start > 0 ? "a" : "w" open(joinpath(simulation.path, "summary.log"), writing_mode) do file println(file, "SIMULATION SUMMARY") println(file) @@ -220,7 +220,7 @@ Run the Monte Carlo simulation. """ function run!(simulation::Simulation; wall_time::Real=Inf) t0 = time() # initial measure of time in seconds - status = :completed # default + status = :running # default try simulation.verbose && println("\n" * "-"^50) simulation.verbose && println("\033[1;32mINITIALISATION\033[0m") @@ -231,12 +231,11 @@ function run!(simulation::Simulation; wall_time::Real=Inf) simulation.verbose && println("\033[1;32m\nRUNNING SIMULATION...\033[0m") sim_time = @elapsed for simulation.t in (simulation.t_start+1):simulation.steps for k in eachindex(simulation.algorithms) - # Guard against a scheduler that ends before `steps`: once its counter - # runs past the last entry, `schedulers[k][counters[k]]` would be out of - # bounds. `&&` short-circuits, so we only index when the counter is valid. - if simulation.counters[k] ≤ length(simulation.schedulers[k]) && simulation.t == simulation.schedulers[k][simulation.counters[k]] + if simulation.t == simulation.schedulers[k][simulation.counters[k]] make_step!(simulation, simulation.algorithms[k]) - simulation.counters[k] += 1 + if simulation.counters[k] < length(simulation.schedulers[k]) # pin the counter at the last entry to prevent out of bound situations + simulation.counters[k] += 1 + end end end if time()-t0 ≥ wall_time @@ -244,7 +243,10 @@ function run!(simulation::Simulation; wall_time::Real=Inf) break # exit the `for simulation.t` loop end end - simulation.verbose && println("\nSimulation completed in $(sim_time) s") + if status == :running + status = :completed + end + simulation.verbose && println("\nSimulation $(status == :completed ? "completed" : "stopped for restart") in $(sim_time) s") update_summary(simulation, sim_time) finally simulation.verbose && println("\033[1;32m\nFINALISATION\033[0m") From 220b7df681882661ebcab41d43da511a9f4aacf9 Mon Sep 17 00:00:00 2001 From: Romain Simon <93144534+romainljsimon@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:05:28 +0200 Subject: [PATCH 3/3] Added a catch if error occurs --- src/simulation.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/simulation.jl b/src/simulation.jl index 8da7114..cc85838 100644 --- a/src/simulation.jl +++ b/src/simulation.jl @@ -220,7 +220,7 @@ Run the Monte Carlo simulation. """ function run!(simulation::Simulation; wall_time::Real=Inf) t0 = time() # initial measure of time in seconds - status = :running # default + status = :running # default try simulation.verbose && println("\n" * "-"^50) simulation.verbose && println("\033[1;32mINITIALISATION\033[0m") @@ -248,6 +248,12 @@ function run!(simulation::Simulation; wall_time::Real=Inf) end simulation.verbose && println("\nSimulation $(status == :completed ? "completed" : "stopped for restart") in $(sim_time) s") update_summary(simulation, sim_time) + + catch err + status = :failed + simulation.verbose && println("\033[1;31m\nSIMULATION FAILED:\033[0m $(sprint(showerror, err))") + rethrow() + finally simulation.verbose && println("\033[1;32m\nFINALISATION\033[0m") for algorithm in simulation.algorithms