diff --git a/lib/OrdinaryDiffEqLinear/Project.toml b/lib/OrdinaryDiffEqLinear/Project.toml index 514b1477108..39708e4de45 100644 --- a/lib/OrdinaryDiffEqLinear/Project.toml +++ b/lib/OrdinaryDiffEqLinear/Project.toml @@ -23,6 +23,8 @@ OrdinaryDiffEqVerner = "79d7bb75-1356-48c1-b8c0-6832512096c2" OrdinaryDiffEqRosenbrock = "43230ef6-c299-4910-a778-202eb28ce4ce" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" SciMLTesting = "09d9d899-5365-40a9-917a-5f67fddea283" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [sources] DiffEqBase = {path = "../DiffEqBase"} @@ -47,9 +49,11 @@ DiffEqBase = "7" SafeTestsets = "0.1.0" OrdinaryDiffEqRosenbrock = "2" Reexport = "1.2.2" +SparseArrays = "1.10" +Unitful = "1.25.1" [targets] -test = ["DiffEqDevTools", "Random", "OrdinaryDiffEqRosenbrock", "SafeTestsets", "Test", "OrdinaryDiffEqVerner", "OrdinaryDiffEqTsit5", "Pkg", "SciMLTesting"] +test = ["DiffEqDevTools", "Random", "OrdinaryDiffEqRosenbrock", "SafeTestsets", "Test", "OrdinaryDiffEqVerner", "OrdinaryDiffEqTsit5", "Pkg", "SciMLTesting", "SparseArrays", "Unitful"] [sources.OrdinaryDiffEqTsit5] path = "../OrdinaryDiffEqTsit5" diff --git a/lib/OrdinaryDiffEqLinear/src/linear_caches.jl b/lib/OrdinaryDiffEqLinear/src/linear_caches.jl index f482308450a..5d5aeafa01b 100644 --- a/lib/OrdinaryDiffEqLinear/src/linear_caches.jl +++ b/lib/OrdinaryDiffEqLinear/src/linear_caches.jl @@ -401,7 +401,42 @@ function alg_cache( return MagnusGL8ConstantCache() end -@cache struct MagnusNC6Cache{uType, rateType, WType, expType} <: LinearMutableCache +# Preallocated stage buffers only for a dense operator matrix whose element type survives +# the stage arithmetic (`Float64` coefficients, `dt` scaling). Otherwise `nothing`: the +# stages are then formed out of place and keep the operator's own matrix type. +function magnus_dense_stages(::Type{S}, f, dt) where {S} + A = convert(AbstractMatrix, f.f) + T = eltype(A) + if A isa StridedMatrix && T <: Number && promote_type(T, Float64, typeof(dt)) === T + return S(A) + end + return nothing +end + +struct MagnusNC6Stages{M} + A0::M + A1::M + A2::M + A3::M + B0::M + B1::M + B2::M + C1::M + D::M + E::M + F::M + G::M + P1::M + P2::M + Ω2::M + Ω3_4::M + Ωtot::M +end + +MagnusNC6Stages(A::AbstractMatrix) = MagnusNC6Stages(ntuple(_ -> zero(A), Val(17))...) + +@cache struct MagnusNC6Cache{uType, rateType, WType, expType, stagesType} <: + LinearMutableCache u::uType uprev::uType uprev2::uType @@ -410,6 +445,7 @@ end W::WType k::rateType exp_cache::expType + stages::stagesType end function alg_cache( @@ -422,7 +458,10 @@ function alg_cache( k = zero(rate_prototype) fsalfirst = zero(rate_prototype) exp_cache = ExponentialUtilities.alloc_mem(f, ExpMethodGeneric()) - return MagnusNC6Cache(u, uprev, uprev2, zero(u), fsalfirst, W, k, exp_cache) + stages = magnus_dense_stages(MagnusNC6Stages, f, dt) + return MagnusNC6Cache( + u, uprev, uprev2, zero(u), fsalfirst, W, k, exp_cache, stages + ) end struct MagnusNC6ConstantCache <: OrdinaryDiffEqConstantCache @@ -472,7 +511,16 @@ function alg_cache( ) where {uEltypeNoUnits, uBottomEltypeNoUnits, tTypeNoUnits} return MagnusGL6ConstantCache() end -@cache struct MagnusGauss4Cache{uType, rateType, WType, expType} <: +struct MagnusGauss4Stages{M} + A::M + P1::M + P2::M + G::M +end + +MagnusGauss4Stages(A::AbstractMatrix) = MagnusGauss4Stages(zero(A), zero(A), zero(A), zero(A)) + +@cache struct MagnusGauss4Cache{uType, rateType, WType, expType, stagesType} <: LinearMutableCache u::uType uprev::uType @@ -482,6 +530,7 @@ end W::WType k::rateType exp_cache::expType + stages::stagesType end function alg_cache( @@ -494,7 +543,10 @@ function alg_cache( k = zero(rate_prototype) fsalfirst = zero(rate_prototype) exp_cache = ExponentialUtilities.alloc_mem(f, ExpMethodGeneric()) - return MagnusGauss4Cache(u, uprev, uprev2, zero(u), fsalfirst, W, k, exp_cache) + stages = magnus_dense_stages(MagnusGauss4Stages, f, dt) + return MagnusGauss4Cache( + u, uprev, uprev2, zero(u), fsalfirst, W, k, exp_cache, stages + ) end struct MagnusGauss4ConstantCache <: OrdinaryDiffEqConstantCache diff --git a/lib/OrdinaryDiffEqLinear/src/linear_perform_step.jl b/lib/OrdinaryDiffEqLinear/src/linear_perform_step.jl index 08a34bcdf04..f4fd13f112b 100644 --- a/lib/OrdinaryDiffEqLinear/src/linear_perform_step.jl +++ b/lib/OrdinaryDiffEqLinear/src/linear_perform_step.jl @@ -1,3 +1,12 @@ +# Writes the matrix commutator [X, Y] = X*Y - Y*X into `dest`, using `tmp1`/`tmp2` +# as scratch for the two products so no new matrix is allocated. +@inline function _magnus_commutator!(dest, X, Y, tmp1, tmp2) + mul!(tmp1, X, Y) + mul!(tmp2, Y, X) + @. dest = tmp1 - tmp2 + return dest +end + function initialize!(integrator, cache::MagnusMidpointCache) integrator.kshortsize = 2 @@ -562,14 +571,7 @@ function initialize!(integrator, cache::MagnusNC6Cache) return OrdinaryDiffEqCore.increment_nf!(integrator.stats, 1) end -function perform_step!(integrator, cache::MagnusNC6Cache, repeat_step = false) - (; t, dt, uprev, u, p) = integrator - alg = unwrap_alg(integrator, nothing) - (; W, k, tmp, exp_cache) = cache - mass_matrix = integrator.f.mass_matrix - exp_method = ExpMethodGeneric() - - L = integrator.f.f +function magnus_nc6_exponent!(::Nothing, L, uprev, p, t, dt) update_coefficients!(L, uprev, p, t) A0 = deepcopy(convert(AbstractMatrix, L)) update_coefficients!(L, uprev, p, t + dt / 4) @@ -591,13 +593,55 @@ function perform_step!(integrator, cache::MagnusNC6Cache, repeat_step = false) (B0 * (dt * B2 / 2 - Ω2 / 60) - (dt * B2 / 2 - Ω2 / 60) * B0) * B0 ) + (3 * dt / 5) * (B1 * Ω2 - Ω2 * B1) + return Ω1 + Ω2 + Ω3_4 +end + +function magnus_nc6_exponent!(stages::MagnusNC6Stages, L, uprev, p, t, dt) + (; A0, A1, A2, A3, B0, B1, B2, C1, D, E, F, G, P1, P2, Ω2, Ω3_4, Ωtot) = stages + update_coefficients!(L, uprev, p, t) + copyto!(A0, convert(AbstractMatrix, L)) + update_coefficients!(L, uprev, p, t + dt / 4) + copyto!(A1, convert(AbstractMatrix, L)) + update_coefficients!(L, uprev, p, t + dt / 2) + copyto!(A2, convert(AbstractMatrix, L)) + update_coefficients!(L, uprev, p, t + 3 * dt / 4) + copyto!(A3, convert(AbstractMatrix, L)) + update_coefficients!(L, uprev, p, t + dt) + A4 = convert(AbstractMatrix, L) + + @. B0 = (1 / 90) * (7 * (A0 + A4) + 32 * (A1 + A3) + 12 * (A2)) + @. B1 = (1 / 90) * ((7 / 2) * (A4 - A0) + 8 * (A3 - A1)) + @. B2 = (1 / 90) * ((7 / 4) * (A0 + A4) + 2 * (A1 + A3)) + + @. C1 = 3 * B0 / 2 - 6 * B2 + _magnus_commutator!(Ω2, B1, C1, P1, P2) + @. Ω2 = (dt * dt) * Ω2 + + @. D = dt * B2 / 2 - Ω2 / 60 + _magnus_commutator!(E, B0, D, P1, P2) + _magnus_commutator!(F, B0, E, P1, P2) + _magnus_commutator!(G, B1, Ω2, P1, P2) + @. Ω3_4 = (dt * dt) * F + (3 * dt / 5) * G + + @. Ωtot = dt * B0 + Ω2 + Ω3_4 + return Ωtot +end + +function perform_step!(integrator, cache::MagnusNC6Cache, repeat_step = false) + (; t, dt, uprev, u, p) = integrator + alg = unwrap_alg(integrator, nothing) + (; exp_cache, stages) = cache + exp_method = ExpMethodGeneric() + + L = integrator.f.f + Ω = magnus_nc6_exponent!(stages, L, uprev, p, t, dt) if alg.krylov u .= expv( - 1.0, Ω1 + Ω2 + Ω3_4, uprev; m = min(alg.m, size(L, 1)), + 1.0, Ω, uprev; m = min(alg.m, size(L, 1)), opnorm = integrator.opts.internalopnorm, alg.iop ) else - u .= exponential!(Ω1 + Ω2 + Ω3_4, exp_method, exp_cache) * uprev + u .= exponential!(Ω, exp_method, exp_cache) * uprev end integrator.f(integrator.fsallast, u, p, t + dt) return OrdinaryDiffEqCore.increment_nf!(integrator.stats, 1) @@ -660,31 +704,49 @@ function initialize!(integrator, cache::MagnusGauss4Cache) return OrdinaryDiffEqCore.increment_nf!(integrator.stats, 1) end +# The Krylov path takes the generator scaled by `1/dt`, since `expv` applies the `dt`. +function magnus_gauss4_exponent!(::Nothing, A, B, dt, krylov) + if krylov + return (A + B) ./ 2 + (dt * sqrt(3)) .* (B * A - A * B) ./ 12 + else + return (dt / 2) .* (A + B) + ((dt^2) * (sqrt(3) / 12)) .* (B * A - A * B) + end +end + +function magnus_gauss4_exponent!(stages::MagnusGauss4Stages, A, B, dt, krylov) + (; P1, P2, G) = stages + mul!(P1, B, A) + mul!(P2, A, B) + if krylov + @. G = (A + B) / 2 + (dt * sqrt(3)) * (P1 - P2) / 12 + else + @. G = (dt / 2) * (A + B) + ((dt^2) * (sqrt(3) / 12)) * (P1 - P2) + end + return G +end + +magnus_snapshot!(::Nothing, A) = deepcopy(A) +magnus_snapshot!(stages::MagnusGauss4Stages, A) = copyto!(stages.A, A) + function perform_step!(integrator, cache::MagnusGauss4Cache, repeat_step = false) (; t, dt, uprev, u, p) = integrator alg = unwrap_alg(integrator, nothing) - (; W, k, tmp, exp_cache) = cache - mass_matrix = integrator.f.mass_matrix + (; exp_cache, stages) = cache exp_method = ExpMethodGeneric() L = integrator.f.f update_coefficients!(L, uprev, p, t + dt * (1 / 2 + sqrt(3) / 6)) - A = deepcopy(convert(AbstractMatrix, L)) + A = magnus_snapshot!(stages, convert(AbstractMatrix, L)) update_coefficients!(L, uprev, p, t + dt * (1 / 2 - sqrt(3) / 6)) B = convert(AbstractMatrix, L) + Ω = magnus_gauss4_exponent!(stages, A, B, dt, alg.krylov) if alg.krylov u .= expv( - dt, (A + B) ./ 2 + (dt * sqrt(3)) .* (B * A - A * B) ./ 12, u; - m = min(alg.m, size(L, 1)), opnorm = integrator.opts.internalopnorm, + dt, Ω, u; m = min(alg.m, size(L, 1)), opnorm = integrator.opts.internalopnorm, alg.iop ) else - u .= exponential!( - (dt / 2) .* (A + B) + - ((dt^2) * (sqrt(3) / 12)) .* (B * A - A * B), - exp_method, - exp_cache - ) * uprev + u .= exponential!(Ω, exp_method, exp_cache) * uprev end integrator.f(integrator.fsallast, u, p, t + dt) return OrdinaryDiffEqCore.increment_nf!(integrator.stats, 1) diff --git a/lib/OrdinaryDiffEqLinear/test/linear_method_tests.jl b/lib/OrdinaryDiffEqLinear/test/linear_method_tests.jl index 027a8b6096e..70282f0d1f7 100644 --- a/lib/OrdinaryDiffEqLinear/test/linear_method_tests.jl +++ b/lib/OrdinaryDiffEqLinear/test/linear_method_tests.jl @@ -1,6 +1,6 @@ using OrdinaryDiffEqLinear, Test, DiffEqDevTools using OrdinaryDiffEqRosenbrock, OrdinaryDiffEqVerner, OrdinaryDiffEqTsit5 -using LinearAlgebra, Random +using LinearAlgebra, Random, SparseArrays, Unitful using SciMLOperators: MatrixOperator # Linear exponential solvers @@ -308,3 +308,41 @@ test_setup = Dict(:alg => Vern9(), :reltol => 1.0e-14, :abstol => 1.0e-14) sim = analyticless_test_convergence(dts, prob, CayleyEuler(), test_setup) @test sim.𝒪est[:l2] ≈ 1 atol = 0.2 + +@testset "MagnusNC6 and MagnusGauss4 operator matrix types" begin + Au = [-0.1 0.0; 0.0 -0.2] + prob_unitful = ODEProblem(MatrixOperator(Au), [1.0, 2.0]u"m", (0.0, 1.0)) + @testset "unitful state, $(nameof(typeof(alg)))" for alg in (MagnusNC6(), MagnusGauss4()) + sol = solve(prob_unitful, alg; dt = 0.1, adaptive = false) + @test sol.retcode == ReturnCode.Success + @test ustrip.(u"m", sol.u[end]) ≈ exp(Au) * [1.0, 2.0] + end + + function tridiag_update!(A, u, p, t) + for j in axes(A, 2), i in max(1, j - 1):min(size(A, 1), j + 1) + A[i, j] = i == j ? -0.1 * i : (i < j ? 0.01 : 0.01 * sin(t)) + end + return nothing + end + N = 300 + pattern = spdiagm(-1 => ones(N - 1), 0 => ones(N), 1 => ones(N - 1)) + tridiag_prob(A) = ODEProblem( + MatrixOperator(A; update_func! = tridiag_update!), ones(N), (0.0, 0.1) + ) + prob_dense = tridiag_prob(Matrix(pattern)) + @testset "$(nameof(typeof(A))) operator, $(nameof(typeof(alg))) krylov=$(alg.krylov)" for (A, alg) in ( + (pattern, MagnusNC6(krylov = true)), + (pattern, MagnusGauss4(krylov = true)), + (Tridiagonal(Matrix(pattern)), MagnusNC6()), + (Tridiagonal(Matrix(pattern)), MagnusGauss4()), + ) + integrator = init(tridiag_prob(A), alg; dt = 0.01, adaptive = false) + if A isa SparseMatrixCSC + # The cache already holds one dense N×N matrix (`W`); the stages must not add more. + @test Base.summarysize(integrator.cache) < 2 * N^2 * sizeof(Float64) + end + solve!(integrator) + sol_dense = solve(prob_dense, alg; dt = 0.01, adaptive = false) + @test integrator.sol.u[end] ≈ sol_dense.u[end] rtol = 1.0e-12 + end +end diff --git a/lib/OrdinaryDiffEqLinear/test/qa/allocation_tests.jl b/lib/OrdinaryDiffEqLinear/test/qa/allocation_tests.jl index 735d6c7724e..ee975546560 100644 --- a/lib/OrdinaryDiffEqLinear/test/qa/allocation_tests.jl +++ b/lib/OrdinaryDiffEqLinear/test/qa/allocation_tests.jl @@ -5,6 +5,50 @@ using AllocCheck using LinearAlgebra using Test +# Regression bounds for the per-step allocation of the dense (non-Krylov) Magnus +# exponential integrators, which build several NxN stage/commutator matrices every +# step. `linear_perform_step.jl` preallocates those matrices in the cache rather than +# forming new ones on each call; these bounds are well below what master allocates +# (MagnusNC6/MagnusGauss4 at N=50: 1,431,528 / 222,728 bytes/step; at N=200 constant: +# 22,730,984 / 3,524,648 bytes/step) and comfortably above what the preallocated +# version allocates (~60,900 and ~961,900 bytes/step respectively, dominated by the +# scratch matrices ExponentialUtilities.exponential! allocates internally). +@testset "Magnus dense stage-matrix allocation regression" begin + function magnus_step_alloc(prob, alg; dt = 0.01) + integrator = init( + prob, alg; dt = dt, adaptive = false, save_everystep = false + ) + step!(integrator) + step!(integrator) + return @allocated step!(integrator) + end + + function timedep_update!(A, u, p, t) + @inbounds for j in axes(A, 2), i in axes(A, 1) + A[i, j] = (i == j ? -0.1i : 0.001 * sin(t + i * j)) + end + return nothing + end + + N = 50 + op = MatrixOperator(zeros(Float64, N, N); update_func! = timedep_update!) + prob_timedep = ODEProblem(op, ones(Float64, N), (0.0, 0.2)) + + Nc = 200 + op_const = MatrixOperator(-0.1 * Matrix{Float64}(I, Nc, Nc)) + prob_const = ODEProblem(op_const, ones(Float64, Nc), (0.0, 0.2)) + + @testset "$(typeof(alg)) step allocation bound" for (alg, prob, bound) in ( + (MagnusNC6(), prob_timedep, 150_000), + (MagnusGauss4(), prob_timedep, 150_000), + (MagnusNC6(), prob_const, 2_000_000), + (MagnusGauss4(), prob_const, 2_000_000), + ) + a = magnus_step_alloc(prob, alg) + @test a < bound + end +end + @testset "Linear Allocation Tests" begin A = MatrixOperator([-1.0 0.5; 0.0 -2.0]) prob = ODEProblem(A, [1.0, 1.0], (0.0, 1.0))