diff --git a/AGENTS.md b/AGENTS.md index 506544d7..e0d82d61 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,7 @@ # Adaptive controller conventions Preserve the existing `GPUTsit5` PI-controller behavior. Alternative controllers must be explicit opt-in variants, with documentation explaining their step-size stability and accuracy tradeoffs. Do not change the default based on equal-tolerance timing alone: use work–precision comparisons at matched achieved error, include rejected steps, and test beyond simple nonstiff problems such as Lorenz. + +# Differentiation validation + +Gradient tests must use the package and its loaded extensions without adding differentiation rules in test code. Keep CPU Enzyme coverage alongside CUDA coverage. Exercise all active problem fields when validating transfer rules, including times and captured function values; parameter-only losses cannot detect discarded sensitivities. Keep initialization regressions enabled across backends and fix their owning dependencies rather than gating the tests. diff --git a/Project.toml b/Project.toml index 88076678..32eacc66 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,7 @@ authors = ["Chris Rackauckas", "SciML"] version = "3.21.1" [deps] +EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" @@ -32,6 +33,7 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" [weakdeps] +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" @@ -41,6 +43,7 @@ OpenCL = "08131aa3-fb12-5dee-8b74-c09406e224a2" oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b" [extensions] +EnzymeExt = "Enzyme" AMDGPUExt = ["AMDGPU"] CUDAExt = ["CUDA"] JLArraysExt = ["JLArrays"] @@ -59,6 +62,8 @@ DifferentiationInterface = "0.7.18" DiffEqBase = "7.19" Distributed = "1" DocStringExtensions = "0.9" +Enzyme = "0.13.201" +EnzymeCore = "0.8.21" ForwardDiff = "1.3.3" GPUArraysCore = "0.2" JLArrays = "0.3" @@ -77,7 +82,7 @@ SciMLBase = "3.49" SciMLPublic = "1" Setfield = "1" SimpleDiffEq = "1.11" -SimpleNonlinearSolve = "2.14" +SimpleNonlinearSolve = "2.14.5" StaticArrays = "1.9.14" StaticArraysCore = "1.4.3" TOML = "1" diff --git a/docs/Project.toml b/docs/Project.toml index 91d86dea..7a2f78ff 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -6,7 +6,9 @@ DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" DiffEqGPU = "071ae1c0-96b5-11e9-1965-c90190d839ea" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" @@ -30,7 +32,9 @@ CUDA = "5, 6" DiffEqBase = "7" DiffEqGPU = "3" Documenter = "1" +Enzyme = "0.13.201" ForwardDiff = "1" +KernelAbstractions = "0.9.38" Lux = "1" ModelingToolkit = "11" Optimisers = "0.4" diff --git a/docs/src/examples/ad.md b/docs/src/examples/ad.md index 44147e15..953b9db0 100644 --- a/docs/src/examples/ad.md +++ b/docs/src/examples/ad.md @@ -96,3 +96,54 @@ monteprob = EnsembleProblem(prob; prob_func) saveat = 1.0f0 ) ``` + +## Enzyme reverse mode with `EnsembleGPUKernel` + +For `GPUTsit5`, use Enzyme to differentiate a scalar loss of the ensemble's final +states. Set `cpu_offload` to zero and `save_everystep = false`. Keep the parameters +on the host when constructing the problems; the ensemble solver transfers the +compatible problems to the selected backend. + +```@example enzyme_kernel +using DiffEqGPU, Enzyme, KernelAbstractions, SciMLBase, StaticArrays + +function ensemble_loss(p, backend) + rhs(u, p, t) = p[1] * u + prob = ODEProblem{false}(rhs, SVector(1.0), (0.0, 1.0), SVector(p[1])) + prob_func = (prob, ctx) -> remake(prob; p = SVector(p[ctx.sim_id])) + ensemble = EnsembleProblem(prob; prob_func, safetycopy = false) + sol = solve( + ensemble, GPUTsit5(), EnsembleGPUKernel(backend, 0.0); + trajectories = length(p), adaptive = false, dt = 0.05, + save_everystep = false + ) + return sum(s -> sum(s.u[end]), sol.u) +end + +p = [0.2, -0.3] +backend = CPU() +dp = zero(p) +Enzyme.autodiff(Reverse, ensemble_loss, Active, Duplicated(p, dp), Const(backend)) +@assert dp ≈ exp.(p) +dp +``` + +For NVIDIA GPUs, load `CUDA` and pass `CUDA.CUDABackend()` instead of `CPU()`. +CUDA gradients with released dependencies remain unverified. The +[upstream GPUCompiler compatibility work](https://github.com/EnzymeAD/Enzyme.jl/pull/3512) +and compatible CUDA allocation rules are required for that configuration. +Loading Enzyme activates DiffEqGPU's transfer rules automatically. Reset the shadow +buffer `dp` to zero before each independent reverse-mode call: Enzyme accumulates +into it. Fixed-step gradients differentiate the numerical steps; adaptive gradients +differentiate the executed solver path and are not a record-and-replay adjoint +with a frozen mesh. + +Callbacks and DAE initialization are solver features; their absence from this +example does not imply that they are unsupported. Representative Float64 CPU +checks also validate Enzyme gradients through a fixed-time discrete callback, +nonlinear initialization, the final integration time, and requested `saveat` +times. These checks do not establish differentiation support for every callback +or a complete singular-mass DAE solve. In particular, a continuous callback with +a parameter-dependent event time can produce +[incorrect gradients](https://github.com/SciML/DiffEqGPU.jl/issues/533); event-time +sensitivities need separate validation. diff --git a/ext/EnzymeExt.jl b/ext/EnzymeExt.jl new file mode 100644 index 00000000..129c93d4 --- /dev/null +++ b/ext/EnzymeExt.jl @@ -0,0 +1,36 @@ +module EnzymeExt + +using DiffEqGPU: DiffEqGPU +using Enzyme: Enzyme, EnzymeRules, Const, Duplicated, Reverse + +function EnzymeRules.augmented_primal( + config::EnzymeRules.RevConfig, func::Const{typeof(DiffEqGPU._kernel_transfer)}, + ::Type{RT}, backend::Const, x::Duplicated + ) where {RT} + y = func.val(backend.val, x.val) + # Zero the elements on the host; device-array wrapper metadata can be inactive. + dy = func.val(backend.val, Enzyme.make_zero(Array(x.val))) + return EnzymeRules.AugmentedReturn( + EnzymeRules.needs_primal(config) ? y : nothing, + EnzymeRules.needs_shadow(config) ? dy : nothing, dy + ) +end + +_copy_payload!(dest, src) = (copyto!(dest, src); nothing) + +function EnzymeRules.reverse( + config::EnzymeRules.RevConfig, ::Const{typeof(DiffEqGPU._kernel_transfer)}, + ::Type{RT}, dy, backend::Const, x::Duplicated + ) where {RT} + # Let Enzyme accumulate every active field, including captured functions and times. + primal = Array(x.val) + dx = Array(x.dval) + Enzyme.autodiff( + Reverse, _copy_payload!, Const, + Duplicated(copy(primal), Array(dy)), Duplicated(primal, dx) + ) + copyto!(x.dval, dx) + return (nothing, nothing) +end + +end diff --git a/src/DiffEqGPU.jl b/src/DiffEqGPU.jl index 7abc6a30..491ef03e 100644 --- a/src/DiffEqGPU.jl +++ b/src/DiffEqGPU.jl @@ -5,6 +5,7 @@ module DiffEqGPU using DocStringExtensions: DocStringExtensions using SciMLPublic: @public +using EnzymeCore: within_autodiff using KernelAbstractions: KernelAbstractions, @Const, @index, @kernel, CPU import KernelAbstractions: get_backend, allocate using SciMLBase: SciMLBase, CallbackSet, CheckInit, ContinuousCallback, diff --git a/src/ensemblegpukernel/kernels.jl b/src/ensemblegpukernel/kernels.jl index 6eb72cb5..9d2daf5a 100644 --- a/src/ensemblegpukernel/kernels.jl +++ b/src/ensemblegpukernel/kernels.jl @@ -1,22 +1,25 @@ +# Shared EnsembleGPUKernel ODE launchers. Non-@Const probs + scalar unpacking so +# Enzyme reverse works; Duals / plain floats both go through _load_kernel_arg. + +@inline _load_kernel_arg(x::Number) = x +@inline _load_kernel_arg(x::AbstractArray{<:Any, 0}) = @inbounds x[] +@inline _load_kernel_arg(x::AbstractArray) = x + @kernel function ode_solve_kernel( - @Const(probs), alg, _us, _ts, dt, callback, - tstops, nsteps, + probs, alg, _us, _ts, _dt, callback, + tstops, saveat, ::Val{save_everystep} ) where {save_everystep} i = @index(Global, Linear) + dt = _load_kernel_arg(_dt) - # get the actual problem for this thread prob = @inbounds probs[i] - - # get the input/output arrays for this thread ts = @inbounds view(_ts, :, i) us = @inbounds view(_us, :, i) _saveat = get(prob.kwargs, :saveat, nothing) - saveat = _saveat === nothing ? saveat : _saveat - # Check if initialization is needed for DAEs u0, p_init, init_success = if SciMLBase.has_initialization_data(prob.f) gpu_initialization_solve(prob, SimpleTrustRegion(), 1.0e-6, 1.0e-6) @@ -25,7 +28,6 @@ end if init_success - # Use initialized values integ = init( alg, prob.f, false, u0, prob.tspan[1], dt, p_init, tstops, callback, save_everystep, saveat @@ -45,7 +47,6 @@ end integ.step_idx += 1 - # FSAL while integ.t < tspan[2] && integ.retcode != DiffEqBase.ReturnCode.Terminated saved_in_cb = step!(integ, ts, us) !saved_in_cb && savevalues!(integ, ts, us) @@ -55,12 +56,10 @@ @inbounds ts[2] = integ.t end if integ.t > tspan[2] && saveat === nothing - ## Interpolate to tf @inbounds us[end] = integ(tspan[2]) @inbounds ts[end] = tspan[2] end else - # Initialization failed — store initial values and bail out @inbounds us[1] = prob.u0 @inbounds ts[1] = prob.tspan[1] if saveat === nothing && !save_everystep @@ -71,25 +70,23 @@ end @kernel function ode_asolve_kernel( - @Const(probs), alg, _us, _ts, dt, callback, tstops, - abstol, reltol, + probs, alg, _us, _ts, _dt, callback, tstops, + _abstol, _reltol, saveat, ::Val{save_everystep} ) where {save_everystep} i = @index(Global, Linear) + dt = _load_kernel_arg(_dt) + abstol = _load_kernel_arg(_abstol) + reltol = _load_kernel_arg(_reltol) - # get the actual problem for this thread prob = @inbounds probs[i] - # get the input/output arrays for this thread ts = @inbounds view(_ts, :, i) us = @inbounds view(_us, :, i) - # TODO: optimize contiguous view to return a CuDeviceArray _saveat = get(prob.kwargs, :saveat, nothing) - saveat = _saveat === nothing ? saveat : _saveat - # Check if initialization is needed for DAEs u0, p_init, init_success = if SciMLBase.has_initialization_data(prob.f) gpu_initialization_solve(prob, SimpleTrustRegion(), abstol, reltol) @@ -99,15 +96,9 @@ end if init_success tspan = prob.tspan - f = prob.f - p = p_init - - t = tspan[1] - tf = prob.tspan[2] - integ = init( alg, prob.f, false, u0, prob.tspan[1], prob.tspan[2], dt, - p, + p_init, abstol, reltol, DiffEqBase.ODE_DEFAULT_NORM, tstops, callback, saveat ) @@ -130,7 +121,6 @@ end end if integ.t > tspan[2] && saveat === nothing - ## Interpolate to tf @inbounds us[end] = integ(tspan[2]) @inbounds ts[end] = tspan[2] end @@ -140,7 +130,6 @@ end @inbounds ts[2] = integ.t end else - # Initialization failed — store initial values and bail out @inbounds us[1] = prob.u0 @inbounds ts[1] = prob.tspan[1] if saveat === nothing && !save_everystep diff --git a/src/ensemblegpukernel/lowerlevel_solve.jl b/src/ensemblegpukernel/lowerlevel_solve.jl index a9e34003..b2812956 100644 --- a/src/ensemblegpukernel/lowerlevel_solve.jl +++ b/src/ensemblegpukernel/lowerlevel_solve.jl @@ -52,6 +52,32 @@ ts, us = DiffEqGPU.vectorized_solve( """ function vectorized_solve end + +function _kernel_transfer(backend, x::AbstractArray) + dest = allocate(backend, eltype(x), size(x)) + copyto!(dest, x) + return dest +end + +# GPU reverse kernels require duplicated storage for active scalar arguments. +function _pack_kernel_scalar(backend, x::AbstractFloat) + return within_autodiff() ? _kernel_transfer(backend, fill(x)) : x +end +_pack_kernel_scalar(backend, x) = x +_pack_kernel_scalar(::CPU, x::AbstractFloat) = x + +@kernel function _fill_time_kernel!(ts, t0) + i = @index(Global, Linear) + @inbounds ts[i] = _load_kernel_arg(t0) +end + +function _init_time_matrix!(ts, t0) + backend = get_backend(ts) + _fill_time_kernel!(backend)(ts, _pack_kernel_scalar(backend, t0); ndrange = length(ts)) + return ts +end +_init_time_matrix!(ts::Array, t0) = fill!(ts, t0) + function vectorized_solve( probs, prob::ODEProblem, alg; dt, saveat = nothing, @@ -61,10 +87,6 @@ function vectorized_solve( ) backend = get_backend(probs) backend = maybe_prefer_blocks(backend) - # if saveat is specified, we'll use a vector of timestamps. - # otherwise it's a matrix that may be different for each ODE. - timeseries = prob.tspan[1]:dt:prob.tspan[2] - nsteps = length(timeseries) prob = convert(ImmutableODEProblem, prob) dt = convert(eltype(prob.tspan), dt) @@ -72,28 +94,24 @@ function vectorized_solve( if saveat === nothing if save_everystep - len = length(prob.tspan[1]:dt:prob.tspan[2]) + timeseries = prob.tspan[1]:dt:prob.tspan[2] + len = length(timeseries) if tstops !== nothing len += length(tstops) - count(x -> x in tstops, timeseries) - nsteps += length(tstops) - count(x -> x in tstops, timeseries) end else len = 2 end ts = allocate(backend, typeof(dt), (len, length(probs))) - fill!(ts, prob.tspan[1]) + _init_time_matrix!(ts, prob.tspan[1]) us = allocate(backend, typeof(prob.u0), (len, length(probs))) else - # Get the time type from the problem Tt = eltype(prob.tspan) - - # FIX for Issue #379: Convert saveat to proper type saveat_converted = if saveat isa AbstractRange Tt.(collect(range(Tt(first(saveat)), Tt(last(saveat)), length = length(saveat)))) elseif saveat isa AbstractVector Tt.(collect(saveat)) else - # saveat is a Number (step size) t0, tf = Tt.(prob.tspan) if Tt(saveat) == Tt(0.0) Tt.([t0, tf]) @@ -102,33 +120,24 @@ function vectorized_solve( Tt.(collect(range(t0, tf, length = num_points))) end end - saveat_converted = adapt(backend, saveat_converted) - ts = allocate(backend, typeof(dt), (length(saveat_converted), length(probs))) - fill!(ts, prob.tspan[1]) + _init_time_matrix!(ts, prob.tspan[1]) us = allocate(backend, typeof(prob.u0), (length(saveat_converted), length(probs))) end tstops = adapt(backend, tstops) - kernel = ode_solve_kernel(backend) - if backend isa CPU @warn "Running the kernel on CPU" end - kernel( - probs, alg, us, ts, dt, callback, tstops, nsteps, saveat_converted, - Val(save_everystep); + ode_solve_kernel(backend)( + probs, alg, us, ts, _pack_kernel_scalar(backend, dt), callback, tstops, + saveat_converted, Val(save_everystep); ndrange = length(probs) ) - # we build the actual solution object on the CPU because the GPU would create one - # containing CuDeviceArrays, which we cannot use on the host (not GC tracked, - # no useful operations, etc). That's unfortunate though, since this loop is - # generally slower than the entire GPU execution, and necessitates synchronization - # EDIT: Done when using with DiffEqGPU return ts, us end @@ -152,7 +161,7 @@ function vectorized_solve( len = 2 end ts = allocate(backend, typeof(dt), (len, length(probs))) - fill!(ts, prob.tspan[1]) + _init_time_matrix!(ts, prob.tspan[1]) us = allocate(backend, typeof(prob.u0), (len, length(probs))) else # Get the time type from the problem @@ -175,7 +184,7 @@ function vectorized_solve( end ts = allocate(backend, typeof(dt), (length(saveat_converted), length(probs))) - fill!(ts, prob.tspan[1]) + _init_time_matrix!(ts, prob.tspan[1]) us = allocate(backend, typeof(prob.u0), (length(saveat_converted), length(probs))) end if saveat_converted !== nothing @@ -321,11 +330,11 @@ function vectorized_asolve( len = 2 end ts = allocate(backend, typeof(dt), (len, length(probs))) - fill!(ts, prob.tspan[1]) + _init_time_matrix!(ts, prob.tspan[1]) us = allocate(backend, typeof(prob.u0), (len, length(probs))) else ts = allocate(backend, typeof(dt), (length(saveat_converted), length(probs))) - fill!(ts, prob.tspan[1]) + _init_time_matrix!(ts, prob.tspan[1]) us = allocate(backend, typeof(prob.u0), (length(saveat_converted), length(probs))) end @@ -336,15 +345,14 @@ function vectorized_asolve( if saveat_converted !== nothing saveat_converted = adapt(backend, saveat_converted) end - kernel = ode_asolve_kernel(backend) - if backend isa CPU @warn "Running the kernel on CPU" end - kernel( - probs, alg, us, ts, dt, callback, tstops, - abstol, reltol, saveat_converted, Val(save_everystep); + ode_asolve_kernel(backend)( + probs, alg, us, ts, _pack_kernel_scalar(backend, dt), callback, tstops, + _pack_kernel_scalar(backend, abstol), _pack_kernel_scalar(backend, reltol), + saveat_converted, Val(save_everystep); ndrange = length(probs) ) diff --git a/src/solve.jl b/src/solve.jl index d4179444..0e5a06c1 100644 --- a/src/solve.jl +++ b/src/solve.jl @@ -52,8 +52,8 @@ function SciMLBase.__solve( num_batches = gpu_trajectories ÷ batch_size num_batches * batch_size != gpu_trajectories && (num_batches += 1) - t = nothing - if cpu_trajectories != 0 && ensembleprob.reduction === SciMLBase.DEFAULT_REDUCTION + cpu_work = if cpu_trajectories != 0 && + ensembleprob.reduction === SciMLBase.DEFAULT_REDUCTION cpu_II = (gpu_trajectories + 1):trajectories _alg = if alg isa GPUODEAlgorithm if adaptive == false @@ -83,6 +83,9 @@ function SciMLBase.__solve( put!(cpu_sols, f()) end schedule(t) + (; cpu_sols, t) + else + nothing end if num_batches == 1 && ensembleprob.reduction === SciMLBase.DEFAULT_REDUCTION @@ -92,9 +95,9 @@ function SciMLBase.__solve( sim_seeds, rng_func, master_rng = rng, unstable_check, kwargs... ) - if cpu_trajectories != 0 - wait(t) - sol = vcat(sol, take!(cpu_sols)) + if cpu_work !== nothing + wait(cpu_work.t) + sol = vcat(sol, take!(cpu_work.cpu_sols)) end return SciMLBase.EnsembleSolution(sol, time, true) end @@ -157,9 +160,9 @@ function SciMLBase.__solve( end return if ensembleprob.reduction === SciMLBase.DEFAULT_REDUCTION - if cpu_trajectories != 0 - wait(t) - sols = vcat(reduce(vcat, vec.(sols)), take!(cpu_sols)) + if cpu_work !== nothing + wait(cpu_work.t) + sols = vcat(reduce(vcat, vec.(sols)), take!(cpu_work.cpu_sols)) else sols = reduce(vcat, sols) end @@ -176,6 +179,48 @@ function _make_ensemble_context(i, sim_seeds, rng_func, master_rng) return @set pre_ctx.rng = sim_rng end +# Enzyme needs numeric fields first and uninlined construction to retain aggregate type information. +struct KernelODEProblem{U, T, IIP, P, F, K, PT} <: SciMLBase.AbstractODEProblem{U, T, IIP} + p::P + u0::U + tspan::T + f::F + kwargs::K + problem_type::PT +end +_kernel_record(prob) = prob +@noinline function _kernel_record(prob::SciMLBase.ImmutableODEProblem{U, T, IIP, P, F, K, PT}) where {U, T, IIP, P, F, K, PT} + return KernelODEProblem{U, T, IIP, P, F, K, PT}(prob.p, prob.u0, prob.tspan, prob.f, prob.kwargs, prob.problem_type) +end + +@noinline function _make_kernel_problem(ensembleprob, i, sim_seeds, rng_func, master_rng) + ctx = _make_ensemble_context(i, sim_seeds, rng_func, master_rng) + prob = ensembleprob.safetycopy ? deepcopy(ensembleprob.prob) : ensembleprob.prob + return make_prob_compatible(ensembleprob.prob_func(prob, ctx)) +end + +# Enzyme needs reference storage for host aggregate shadows, but primal solves do not. +@inline _wrap_kernel_host(prob) = within_autodiff() ? Ref(prob) : prob +@inline _unwrap_kernel_host(prob) = prob +@inline _unwrap_kernel_host(prob::Ref) = prob[] + +function _prepare_kernel_problems(ensembleprob, backend, I, sim_seeds, rng_func, master_rng) + first_prob = _make_kernel_problem(ensembleprob, first(I), sim_seeds, rng_func, master_rng) + first_adapted = _kernel_record(adapt(backend, first_prob)) + first_ref = _wrap_kernel_host(first_prob) + probs = Vector{typeof(first_ref)}(undef, length(I)) + adapted_probs = Vector{typeof(first_adapted)}(undef, length(I)) + probs[1] = first_ref + adapted_probs[1] = first_adapted + # Adapt during construction: a separate broadcast over isbits problems loses Enzyme gradients. + for j in 2:length(I) + prob = _make_kernel_problem(ensembleprob, I[j], sim_seeds, rng_func, master_rng) + probs[j] = _wrap_kernel_host(prob) + adapted_probs[j] = _kernel_record(adapt(backend, prob)) + end + return probs, adapted_probs +end + function batch_solve( ensembleprob, alg, ensemblealg::Union{EnsembleArrayAlgorithm, EnsembleKernelAlgorithm}, I, @@ -186,39 +231,22 @@ function batch_solve( kwargs... ) @assert !isempty(I) - #@assert all(p->p.f === probs[1].f,probs) return if ensemblealg isa EnsembleGPUKernel - if ensembleprob.safetycopy - probs = map(I) do i - ctx = _make_ensemble_context(i, sim_seeds, rng_func, master_rng) - make_prob_compatible( - ensembleprob.prob_func( - deepcopy(ensembleprob.prob), - ctx - ) - ) - end - else - probs = map(I) do i - ctx = _make_ensemble_context(i, sim_seeds, rng_func, master_rng) - make_prob_compatible(ensembleprob.prob_func(ensembleprob.prob, ctx)) - end - end - # Using inner saveat requires all of them to be of same size, - # because the dimension of CuMatrix is decided by it. - # The columns of it are accessed at each thread. + kernel_probs, adapted_kernel_probs = _prepare_kernel_problems( + ensembleprob, ensemblealg.dev, I, sim_seeds, rng_func, master_rng + ) if !all( Base.Fix2( - (prob1, prob2) -> isequal(prob1.tspan, prob2.tspan), - probs[1] + (prob1, prob2) -> isequal(_unwrap_kernel_host(prob1).tspan, _unwrap_kernel_host(prob2).tspan), + kernel_probs[1] ), - probs + kernel_probs ) if !iszero(ensemblealg.cpu_offload) error("Different time spans in an Ensemble Simulation with CPU offloading is not supported yet.") end - if get(probs[1].kwargs, :saveat, nothing) === nothing && !adaptive && + if get(_unwrap_kernel_host(kernel_probs[1]).kwargs, :saveat, nothing) === nothing && !adaptive && get(kwargs, :save_everystep, true) error("Using different time-spans require either turning off save_everystep or using saveat. If using saveat, it should be of same length across the ensemble.") end @@ -228,68 +256,53 @@ function batch_solve( prob1, prob2, ) -> isequal( - sizeof( - get( - prob1.kwargs, :saveat, - nothing - ) - ), - sizeof( - get( - prob2.kwargs, :saveat, - nothing - ) - ) + sizeof(get(_unwrap_kernel_host(prob1).kwargs, :saveat, nothing)), + sizeof(get(_unwrap_kernel_host(prob2).kwargs, :saveat, nothing)) ), - probs[1] + kernel_probs[1] ), - probs + kernel_probs ) error("Using different saveat in EnsembleGPUKernel requires all of them to be of same length. Use saveats of same size only.") end end - if alg isa Union{GPUODEAlgorithm, GPUSDEAlgorithm} - # Get inner saveat if global one isn't specified - _saveat = get(probs[1].kwargs, :saveat, nothing) - saveat = _saveat === nothing ? get(kwargs, :saveat, nothing) : _saveat - solts, - solus = batch_solve_up_kernel( - ensembleprob, probs, alg, ensemblealg, I, - adaptive; saveat, kwargs... - ) - [ - begin - ts = @view solts[:, i] - us = @view solus[:, i] - sol_idx = findlast(x -> x != probs[i].tspan[1], ts) - if sol_idx === nothing - @error "No solution found" tspan = probs[i].tspan[1] ts - error("Batch solve failed") - end - @views ensembleprob.output_func( - SciMLBase.build_solution( - probs[i], - alg, - ts[1:sol_idx], - us[1:sol_idx], - k = nothing, - stats = nothing, - calculate_error = false, - retcode = sol_idx != - length(ts) ? - ReturnCode.Terminated : - ReturnCode.Success - ), - _make_ensemble_context(I[i], sim_seeds, rng_func, master_rng) - )[1] - end - for i in eachindex(probs) - ] - - else + if !(alg isa Union{GPUODEAlgorithm, GPUSDEAlgorithm}) error("We don't have solvers implemented for this algorithm yet") end + + _saveat = get(_unwrap_kernel_host(kernel_probs[1]).kwargs, :saveat, nothing) + saveat = _saveat === nothing ? get(kwargs, :saveat, nothing) : _saveat + solts, kernel_solus = batch_solve_up_kernel( + ensembleprob, kernel_probs, adapted_kernel_probs, alg, ensemblealg, I, + adaptive; saveat, kwargs... + ) + [ + begin + ts = @view solts[:, i] + us = @view kernel_solus[:, i] + sol_idx = findlast(x -> x != _unwrap_kernel_host(kernel_probs[i]).tspan[1], ts) + if sol_idx === nothing + @error "No solution found" tspan = _unwrap_kernel_host(kernel_probs[i]).tspan[1] ts + error("Batch solve failed") + end + @views ensembleprob.output_func( + SciMLBase.build_solution( + _unwrap_kernel_host(kernel_probs[i]), + alg, + ts[1:sol_idx], + us[1:sol_idx], + k = nothing, + stats = nothing, + calculate_error = false, + retcode = sol_idx != length(ts) ? ReturnCode.Terminated : + ReturnCode.Success + ), + _make_ensemble_context(I[i], sim_seeds, rng_func, master_rng) + )[1] + end + for i in eachindex(kernel_probs) + ] else if ensembleprob.safetycopy probs = map(I) do i @@ -385,10 +398,10 @@ function batch_solve( end function batch_solve_up_kernel( - ensembleprob, probs, alg, ensemblealg, I, adaptive; + ensembleprob, probs, adapted_probs, alg, ensemblealg, I, adaptive; kwargs... ) - _callback = CallbackSet(generate_callback(probs[1], length(I), ensemblealg; kwargs...)) + _callback = CallbackSet(generate_callback(_unwrap_kernel_host(probs[1]), length(I), ensemblealg; kwargs...)) _callback = CallbackSet( convert.( @@ -402,9 +415,8 @@ function batch_solve_up_kernel( ) dev = ensemblealg.dev - probs = adapt(dev, adapt.((dev,), probs)) + probs = _kernel_transfer(dev, adapted_probs) - #Adaptive version only works with saveat if adaptive ts, us = vectorized_asolve( @@ -418,8 +430,8 @@ function batch_solve_up_kernel( kwargs..., callback = _callback ) end - solus = Array(us) - solts = Array(ts) + solus = _kernel_transfer(CPU(), us) + solts = _kernel_transfer(CPU(), ts) return (solts, solus) end diff --git a/src/utils.jl b/src/utils.jl index 8d4c7b5d..c149de96 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -204,7 +204,7 @@ end function _maybe_convert_mass_matrix(prob) mm = prob.f.mass_matrix # Already an SArray, UniformScaling, or I — nothing to do - (mm isa StaticArrays.StaticArray || mm === LinearAlgebra.I) && return prob + (mm isa Union{StaticArrays.StaticArray, LinearAlgebra.UniformScaling}) && return prob # Convert to SMatrix N = length(prob.u0) smm = StaticArrays.SMatrix{N, N}(mm) diff --git a/test/enzyme/Project.toml b/test/enzyme/Project.toml new file mode 100644 index 00000000..ec171a2f --- /dev/null +++ b/test/enzyme/Project.toml @@ -0,0 +1,27 @@ +[deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" +DiffEqGPU = "071ae1c0-96b5-11e9-1965-c90190d839ea" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" +JLArrays = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" +KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +SciMLTesting = "09d9d899-5365-40a9-917a-5f67fddea283" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[sources] +DiffEqGPU = {path = "../.."} + +[compat] +CUDA = "6" +CUDACore = "6.2" +Enzyme = "0.13.201" +ExplicitImports = "1.15" +JLArrays = "0.3" +KernelAbstractions = "0.9.38" +SciMLBase = "3.49" +SciMLTesting = "2.10" +StaticArrays = "1.9.14" +julia = "1.10" diff --git a/test/enzyme/cuda_records.jl b/test/enzyme/cuda_records.jl new file mode 100644 index 00000000..0e289d01 --- /dev/null +++ b/test/enzyme/cuda_records.jl @@ -0,0 +1,21 @@ +using Enzyme, CUDACore, Test +struct EnzymeAllocationRecord + value::Float64 + flag::Bool +end + +recordalloc(dims) = CuArray{EnzymeAllocationRecord}(undef, dims) + +@testset "Allocate records $dims" for dims in ((), (0,), (3,), (2, 3)) + dup = Enzyme.autodiff(ForwardWithPrimal, recordalloc, Duplicated, Const(dims)) + @test size(dup[1]) == dims + @test all(x -> iszero(x.value) && !x.flag, Array(dup[1])) + + fwd, rev = Enzyme.autodiff_thunk( + ReverseSplitWithPrimal, Const{typeof(recordalloc)}, + Duplicated, Const{typeof(dims)} + ) + tape, prim, shad = fwd(Const(recordalloc), Const(dims)) + @test size(shad) == dims + @test all(x -> iszero(x.value) && !x.flag, Array(shad)) +end diff --git a/test/enzyme/gradients.jl b/test/enzyme/gradients.jl new file mode 100644 index 00000000..d9c4ec2f --- /dev/null +++ b/test/enzyme/gradients.jl @@ -0,0 +1,122 @@ +using DiffEqGPU, Enzyme, KernelAbstractions, SciMLBase, StaticArrays, Test + +const backend = if get(ENV, "GROUP", "Enzyme") == "CUDA" + using CUDA + CUDA.CUDABackend() +else + CPU() +end + +exponential_rhs(u, p, t) = p[1] * u + +function ensemble_loss(p::Vector{T}, backend, adaptive) where {T} + prob = ODEProblem{false}( + exponential_rhs, SVector(one(T)), (zero(T), one(T)), SVector(p[1]) + ) + prob_func = (prob, ctx) -> remake(prob; p = SVector(p[ctx.sim_id])) + ensemble = EnsembleProblem(prob; prob_func, safetycopy = false) + sol = solve( + ensemble, GPUTsit5(), EnsembleGPUKernel(backend, 0.0); + trajectories = length(p), adaptive, dt = T(0.05), + abstol = T(1.0e-7), reltol = T(1.0e-6), save_everystep = false + ) + return sum(s -> sum(s.u[end]), sol.u) +end + +@testset "Enzyme ensemble gradients ($T, adaptive=$adaptive)" for + T in (Float32, Float64), adaptive in (false, true) + p = T[0.2, -0.3, 0.1] + dp = zero(p) + Enzyme.autodiff( + Reverse, ensemble_loss, Active, Duplicated(p, dp), Const(backend), Const(adaptive) + ) + @test dp ≈ exp.(p) rtol = (T === Float32 ? 2.0e-5 : 1.0e-7) +end + +function lorenz_rhs(u, p, t) + x, y, z = u + return SVector(10 * (y - x), x * (p[1] - z) - y, x * y - (8 / 3) * z) +end + +function lorenz_sensitivity_rhs(u, p, t) + x, y, z, sx, sy, sz = u + return SVector( + 10 * (y - x), x * (p[1] - z) - y, x * y - (8 / 3) * z, + 10 * (sy - sx), (p[1] - z) * sx - sy - x * sz + x, + y * sx + x * sy - (8 / 3) * sz + ) +end + +function lorenz_loss(p, backend, adaptive) + prob = ODEProblem{false}(lorenz_rhs, SVector(1.0, 0.0, 0.0), (0.0, 1.0), SVector(p[1])) + prob_func = (prob, ctx) -> remake(prob; p = SVector(p[ctx.sim_id])) + ensemble = EnsembleProblem(prob; prob_func, safetycopy = false) + sol = solve( + ensemble, GPUTsit5(), EnsembleGPUKernel(backend, 0.0); + trajectories = length(p), adaptive, dt = 0.001, + abstol = 1.0e-10, reltol = 1.0e-9, save_everystep = false + ) + return sum(s -> sum(abs2, s.u[end]), sol.u) +end + +@testset "Lorenz parameter gradients" begin + p = [28.0, 29.0, 30.0] + prob = ODEProblem{false}( + lorenz_sensitivity_rhs, SVector(1.0, 0.0, 0.0, 0.0, 0.0, 0.0), + (0.0, 1.0), SVector(p[1]) + ) + prob_func = (prob, ctx) -> remake(prob; p = SVector(p[ctx.sim_id])) + reference = solve( + EnsembleProblem(prob; prob_func, safetycopy = false), GPUTsit5(), + EnsembleGPUKernel(CPU(), 0.0); trajectories = length(p), + adaptive = false, dt = 0.0001, save_everystep = false + ) + expected = map(reference.u) do sol + u = sol.u[end] + 2 * sum(u[i] * u[i + 3] for i in 1:3) + end + for adaptive in (false, true) + dp = zero(p) + Enzyme.autodiff( + Reverse, lorenz_loss, Active, Duplicated(p, dp), Const(backend), Const(adaptive) + ) + @test dp ≈ expected rtol = 1.0e-6 + end +end + +function time_loss(p, backend) + prob = ODEProblem{false}((u, p, t) -> SVector(t), SVector(0.0), (p[1], p[2])) + sol = solve( + EnsembleProblem(prob; safetycopy = false), GPUTsit5(), + EnsembleGPUKernel(backend, 0.0); trajectories = 3, + adaptive = false, dt = p[3], save_everystep = false + ) + return sum(s -> only(s.u[end]), sol.u) +end + +@testset "Initial time, final time, and step size gradients" begin + p = [0.2, 0.93, 0.07] + @test time_loss(p, backend) ≈ 1.5 * (p[2]^2 - p[1]^2) rtol = 1.0e-10 + dp = zero(p) + Enzyme.autodiff(Reverse, time_loss, Active, Duplicated(p, dp), Const(backend)) + @test dp ≈ 3 .* [-p[1], p[2], 0.0] rtol = 1.0e-9 atol = 1.0e-10 +end + +function initial_state_loss(u0, backend) + prob = ODEProblem{false}(exponential_rhs, SVector(u0[1]), (0.0, 1.0), SVector(0.2)) + prob_func = (prob, ctx) -> remake(prob; u0 = SVector(u0[ctx.sim_id])) + sol = solve( + EnsembleProblem(prob; prob_func, safetycopy = false), GPUTsit5(), + EnsembleGPUKernel(backend, 0.0); trajectories = length(u0), + adaptive = false, dt = 0.05, save_everystep = false + ) + return sum(s -> only(s.u[end]), sol.u) +end + +@testset "Initial state gradients" begin + u0 = [0.4, 0.5, 0.7] + @test initial_state_loss(u0, backend) ≈ sum(u0) * exp(0.2) rtol = 1.0e-7 + du0 = zero(u0) + Enzyme.autodiff(Reverse, initial_state_loss, Active, Duplicated(u0, du0), Const(backend)) + @test du0 ≈ fill(exp(0.2), length(u0)) rtol = 1.0e-7 +end diff --git a/test/enzyme/imports.jl b/test/enzyme/imports.jl new file mode 100644 index 00000000..137386e4 --- /dev/null +++ b/test/enzyme/imports.jl @@ -0,0 +1,18 @@ +using DiffEqGPU, Enzyme, ExplicitImports, Test + +@testset "Enzyme extension imports" begin + ext = Base.get_extension(DiffEqGPU, :EnzymeExt) + @test ext !== nothing + for check in ( + check_no_implicit_imports, check_no_stale_explicit_imports, + check_all_explicit_imports_are_public, check_all_explicit_imports_via_owners, + check_all_qualified_accesses_via_owners, check_no_self_qualified_accesses, + ) + @test check(ext) === nothing + end + # _kernel_transfer belongs to this package. The other two names are Enzyme's + # documented, mandatory custom-rule entry points, which lack `public` declarations. + @test check_all_qualified_accesses_are_public( + ext; ignore = (:_kernel_transfer, :augmented_primal, :reverse) + ) === nothing +end diff --git a/test/enzyme/transfers.jl b/test/enzyme/transfers.jl new file mode 100644 index 00000000..56a921d6 --- /dev/null +++ b/test/enzyme/transfers.jl @@ -0,0 +1,54 @@ +using DiffEqGPU, Enzyme, KernelAbstractions, StaticArrays, Test + +struct TransferRecord{F} + p::SVector{2, Float64} + tspan::Tuple{Float64, Float64} + flag::Bool + f::F +end + +function transfer_loss(p, backend) + f = let a = p[5] + x -> a * x + end + records = [TransferRecord(SVector(p[1], p[2]), (p[3], p[4]), true, f)] + device = DiffEqGPU._kernel_transfer(backend, records) + result = only(DiffEqGPU._kernel_transfer(CPU(), device)) + return result.flag ? + sum(abs2, result.p) + result.tspan[1] * result.tspan[2] + result.f(3.0) : -1.0 +end + +const transfer_backends = if get(ENV, "GROUP", "Enzyme") == "CUDA" + using CUDA + (CUDA.CUDABackend(),) +else + using JLArrays + (CPU(), JLArrays.JLBackend()) +end + +@testset "Aggregate transfers preserve all active fields ($backend)" for backend in transfer_backends + @test Base.get_extension(DiffEqGPU, :EnzymeExt) !== nothing + p = [2.0, 3.0, 4.0, 5.0, 6.0] + @test transfer_loss(p, backend) == 51.0 + dp = ones(5) + for repetitions in 1:2 + Enzyme.autodiff(Reverse, transfer_loss, Active, Duplicated(p, dp), Const(backend)) + @test dp == ones(5) + repetitions * [4, 6, 5, 4, 3] + end +end + +function packed_scalar_loss(p, backend) + arg = DiffEqGPU._pack_kernel_scalar(backend, p[1]) + value = arg isa AbstractArray ? only(DiffEqGPU._kernel_transfer(CPU(), arg)) : arg + return value^2 +end + +@testset "Scalar storage is only needed during differentiation ($backend)" for backend in transfer_backends + for T in (Float32, Float64) + p = T[2] + @test DiffEqGPU._pack_kernel_scalar(backend, p[1]) === p[1] + dp = zero(p) + Enzyme.autodiff(Reverse, packed_scalar_loss, Active, Duplicated(p, dp), Const(backend)) + @test dp == T[4] + end +end diff --git a/test/enzyme_environment.jl b/test/enzyme_environment.jl new file mode 100644 index 00000000..90eb6140 --- /dev/null +++ b/test/enzyme_environment.jl @@ -0,0 +1,17 @@ +let + cuda = get(ENV, "GROUP", "Enzyme") == "CUDA" + envdir = joinpath(@__DIR__, "enzyme") + code = """ + using Pkg + Pkg.instantiate() + using SciMLTesting + SciMLTesting.activate_group_env($(repr(envdir))) + include($(repr(joinpath(envdir, "imports.jl")))) + if $cuda + include($(repr(joinpath(envdir, "cuda_records.jl")))) + end + include($(repr(joinpath(envdir, "transfers.jl")))) + include($(repr(joinpath(envdir, "gradients.jl")))) + """ + run(`$(Base.julia_cmd()) --project=$envdir -e $code`) +end diff --git a/test/gpu_kernel_de/conversions.jl b/test/gpu_kernel_de/conversions.jl index b4537aec..97ab0e8e 100644 --- a/test/gpu_kernel_de/conversions.jl +++ b/test/gpu_kernel_de/conversions.jl @@ -1,4 +1,4 @@ -using DiffEqGPU, OrdinaryDiffEq, StaticArrays, LinearAlgebra, Test +using DiffEqGPU, OrdinaryDiffEq, SciMLBase, StaticArrays, LinearAlgebra, Test include("../utils.jl") function lorenz(u, p, t) @@ -57,3 +57,65 @@ end trajectories = 10_000, saveat = [1.0, 5.0, 10.0] ).u[1].t == [1.0f0, 5.0f0, 10.0f0] + +@testset "Host problems and construction count" for safetycopy in (false, true) + calls = zeros(Int, 3) + analytic(u0, p, t) = u0 * exp(p[1] * t) + f = ODEFunction{false}((u, p, t) -> p[1] * u; analytic) + prob = ODEProblem(f, [1.0f0], (0.0f0, 1.0f0), SVector(0.0f0)) + prob_func = function (prob, ctx) + calls[ctx.sim_id] += 1 + return remake(prob; p = SVector(Float32(ctx.sim_id))) + end + ensemble = EnsembleProblem(prob; prob_func, safetycopy) + sol = solve( + ensemble, GPUTsit5(), EnsembleGPUKernel(backend, 0.0); + trajectories = 3, adaptive = false, dt = 0.01f0, save_everystep = false + ) + @test remake(sol.u[1].prob; p = SVector(2.0f0)).p == SVector(2.0f0) + @test calls == ones(Int, 3) + @test [s.prob.p for s in sol.u] == [SVector(Float32(i)) for i in 1:3] + @test all(s -> s.prob.f.analytic === analytic, sol.u) + @test all(s -> s.prob.u0 isa SVector{1, Float32}, sol.u) +end + +@testset "Initialization preprocessing runs once" begin + updates = Ref(0) + initprob = NonlinearProblem{false}((u, p) -> u .- 1.0f0, SVector(0.0f0)) + update_init = function (initprob, prob) + updates[] += 1 + return initprob + end + initdata = SciMLBase.OverrideInitData( + initprob, update_init, sol -> sol.u, nothing, nothing, Val(true) + ) + f = ODEFunction{false}((u, p, t) -> zero(u); initialization_data = initdata) + prob = ODEProblem(f, SVector(0.0f0), (0.0f0, 0.1f0)) + sol = solve( + EnsembleProblem(prob; safetycopy = false), GPUTsit5(), + EnsembleGPUKernel(backend, 0.0); trajectories = 3, + adaptive = false, dt = 0.1f0, save_everystep = false + ) + @test updates[] == 3 + @test all(s -> s.u[end] ≈ SVector(1.0f0), sol.u) +end + +@testset "Saving a stop already on the fixed time grid" begin + prob = ODEProblem{false}((u, p, t) -> -u, SVector(1.0f0), (0.0f0, 1.0f0)) + sol = solve( + EnsembleProblem(prob), GPUTsit5(), EnsembleGPUKernel(backend, 0.0); + trajectories = 3, adaptive = false, dt = 0.25f0, tstops = [0.5f0], + save_everystep = true + ) + @test all(s -> s.retcode == ReturnCode.Success, sol.u) + @test all(s -> s.t == Float32[0, 0.25, 0.5, 0.75, 1], sol.u) +end + +@testset "Unboxed host problems outside differentiation" begin + prob = ODEProblem{false}((u, p, t) -> p[1] * u, SVector(1.0f0), (0.0f0, 1.0f0), SVector(0.2f0)) + ensemble = EnsembleProblem(prob; safetycopy = false) + host, device = DiffEqGPU._prepare_kernel_problems( + ensemble, backend, 1:3, nothing, ctx -> nothing, nothing + ) + @test isbitstype(eltype(host)) +end diff --git a/test/gpu_kernel_de/gpu_ode_continuous_callbacks.jl b/test/gpu_kernel_de/gpu_ode_continuous_callbacks.jl index e0befe69..869b9521 100644 --- a/test/gpu_kernel_de/gpu_ode_continuous_callbacks.jl +++ b/test/gpu_kernel_de/gpu_ode_continuous_callbacks.jl @@ -10,7 +10,8 @@ function f(u, p, t) end u0 = @SVector[45.0f0, 0.0f0] -tspan = (0.0f0, 15.0f0) +# End away from the velocity jump at t = 15 so roundoff cannot change the endpoint side. +tspan = (0.0f0, 16.0f0) p = @SVector [10.0f0] prob = ODEProblem{false}(f, u0, tspan, p) prob_func = (prob, ctx) -> remake(prob, p = prob.p) @@ -105,13 +106,14 @@ for (alg, diffeq_alg) in zip(algs, diffeq_algs) local sol = solve( monteprob, alg, EnsembleGPUKernel(backend), trajectories = 2, - adaptive = true, dt = 0.1f0, callback = cb, merge_callbacks = true + adaptive = true, dt = 0.1f0, callback = cb, merge_callbacks = true, + abstol = 1.0f-9, reltol = 1.0f-6 ) bench_sol = solve( prob, diffeq_alg, adaptive = true, save_everystep = false, dt = 0.1f0, callback = cb, - merge_callbacks = true + merge_callbacks = true, abstol = 1.0f-9, reltol = 1.0f-6 ) @test norm(bench_sol.u[end] - sol.u[1].u[end]) < 1.0e-2 @@ -123,13 +125,14 @@ for (alg, diffeq_alg) in zip(algs, diffeq_algs) local sol = solve( monteprob, alg, EnsembleGPUKernel(backend), trajectories = 2, - adaptive = true, dt = 0.1f0, callback = cb, merge_callbacks = true + adaptive = true, dt = 0.1f0, callback = cb, merge_callbacks = true, + abstol = 1.0f-9, reltol = 1.0f-6 ) bench_sol = solve( prob, diffeq_alg, adaptive = true, dt = 0.1f0, save_everystep = false, callback = cb, - merge_callbacks = true + merge_callbacks = true, abstol = 1.0f-9, reltol = 1.0f-6 ) @test norm(bench_sol.u[end] - sol.u[1].u[end]) < 1.0e-2 diff --git a/test/gpu_kernel_de/stiff_ode/gpu_ode_mass_matrix.jl b/test/gpu_kernel_de/stiff_ode/gpu_ode_mass_matrix.jl index 9db8fffc..3e089675 100644 --- a/test/gpu_kernel_de/stiff_ode/gpu_ode_mass_matrix.jl +++ b/test/gpu_kernel_de/stiff_ode/gpu_ode_mass_matrix.jl @@ -52,3 +52,18 @@ sol = solve( @test norm(bench_sol.u[1] - sol.u[1].u[1]) < 8.0e-4 @test norm(bench_sol.u[end] - sol.u[1].u[end]) < 8.0e-4 + +@testset "UniformScaling mass matrix" for mass in (I, 2.0f0I, 0.0f0I) + initial = iszero(mass.λ) ? 0.0f0 : 1.0f0 + f = ODEFunction{false}((u, p, t) -> -u; mass_matrix = mass) + prob = ODEProblem(f, SVector(initial), (0.0f0, 1.0f0)) + compatible = @inferred DiffEqGPU.make_prob_compatible(prob) + @test compatible.f.mass_matrix === mass + sol = solve( + EnsembleProblem(prob), GPURosenbrock23(), EnsembleGPUKernel(backend, 0.0); + trajectories = 2, dt = 0.001f0, adaptive = true, + abstol = 1.0f-8, reltol = 1.0f-8, save_everystep = false + ) + expected = iszero(initial) ? initial : exp(-1.0f0 / mass.λ) + @test all(s -> isapprox(s.u[end][1], expected; atol = 2.0f-6), sol.u) +end diff --git a/test/runtests.jl b/test/runtests.jl index d52ada1d..85eb326e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,6 +19,13 @@ const GROUP = get(ENV, "GROUP", "CUDA") using SafeTestsets, Test +if GROUP == "Enzyme" + @time @safetestset "Enzyme ensemble gradients" begin + include("enzyme_environment.jl") + end + exit() +end + if GROUP == "QA" import Pkg Pkg.activate(joinpath(@__DIR__, "qa")) @@ -141,6 +148,12 @@ if GROUP == "CUDA" end end +if GROUP == "CUDA" + @time @safetestset "Enzyme ensemble gradients" begin + include("enzyme_environment.jl") + end +end + # Allocation tests run separately to avoid precompilation interference if GROUP == "all" || GROUP == "nopre" @time @safetestset "Allocation Tests" begin diff --git a/test/test_groups.toml b/test/test_groups.toml index e17e52be..5baaac20 100644 --- a/test/test_groups.toml +++ b/test/test_groups.toml @@ -15,3 +15,6 @@ versions = ["lts", "1", "pre"] [QA] versions = ["lts", "1"] + +[Enzyme] +versions = ["lts", "1"]