Skip to content

Commit dd5d2e0

Browse files
Execute source-order folds without sorting (#22)
1 parent 12b3fa9 commit dd5d2e0

5 files changed

Lines changed: 241 additions & 37 deletions

File tree

‎src/execution/ordered_fold_stage.jl‎

Lines changed: 68 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,20 @@ const _ORDERED_FOLD_BLOCK = 256
88
struct _OrderedFoldStageWorkspace{V, O, S, X, R, T}
99
values::V; order::O; status::S; validation::X; state::R; tree::T
1010
end
11-
struct _OrderedFoldRecurrenceStage{F, A, P, G}
11+
struct _DirectSparseSourceTraversal end
12+
struct _CompactedPrefixTraversal end
13+
14+
_ordered_fold_recurrence_traversal(::_SourceOrder) =
15+
_DirectSparseSourceTraversal()
16+
_ordered_fold_recurrence_traversal(::_PreparedCanonicalBy) =
17+
_CompactedPrefixTraversal()
18+
19+
struct _OrderedFoldRecurrenceStage{F, A, P, G, T}
1220
fields::F
1321
accesses::A
1422
prefix::P
1523
gate::G
24+
traversal::T
1625
source_count::Int32
1726
end
1827
struct _OrderedFoldStagePreparation{B, S, W, V}
@@ -347,6 +356,8 @@ end
347356
end
348357
end
349358

359+
_ordered_fold_stage_launch_order!(backend, ::_SourceOrder, workspace) = nothing
360+
350361
function _ordered_fold_stage_launch_order!(backend, order_law, workspace)
351362
extent = length(workspace.order)
352363
width = 2
@@ -365,8 +376,61 @@ function _ordered_fold_stage_launch_order!(backend, order_law, workspace)
365376
return nothing
366377
end
367378

368-
@inline function _ordered_fold_stage_execute!(run)
379+
@inline function _ordered_fold_stage_apply_item!(
380+
run, accumulator, item::Int32, position::Int32
381+
)
369382
workspace = run.workspace
383+
step = run.transition(
384+
accumulator,
385+
@inbounds(workspace.values[item]), item,
386+
_stage_reads(
387+
run.stage, item,
388+
_OrderedFoldEvaluationValidation(workspace.status)
389+
)
390+
)
391+
_ordered_fold_stage_success(run) || return false
392+
code, component_index, witness = _ordered_fold_stage_validate_step!(
393+
run.state, workspace.state, step
394+
)
395+
if code != 0
396+
_ordered_fold_stage_fail!(
397+
run, code, component_index, item, position, witness
398+
)
399+
return false
400+
end
401+
_ordered_fold_stage_apply_step!(run.state, workspace.state, step)
402+
return !step.halt
403+
end
404+
405+
@inline function _ordered_fold_stage_recur!(
406+
run, ::_DirectSparseSourceTraversal, accumulator
407+
)
408+
position = Int32(0)
409+
for source_item in Int32(1):run.stage.source_count
410+
item = @inbounds run.workspace.order[source_item]
411+
item == 0 && continue
412+
position += Int32(1)
413+
_ordered_fold_stage_apply_item!(
414+
run, accumulator, item, position
415+
) || return
416+
end
417+
return
418+
end
419+
420+
@inline function _ordered_fold_stage_recur!(
421+
run, ::_CompactedPrefixTraversal, accumulator
422+
)
423+
for position in Int32(1):run.stage.source_count
424+
item = @inbounds run.workspace.order[position]
425+
item == 0 && break
426+
_ordered_fold_stage_apply_item!(
427+
run, accumulator, item, position
428+
) || return
429+
end
430+
return
431+
end
432+
433+
@inline function _ordered_fold_stage_execute!(run)
370434
_ordered_fold_stage_prefix_ok(run) || return
371435
gate = _stage_gate_open(
372436
run.stage.gate, run.stage,
@@ -381,30 +445,7 @@ end
381445
return _ordered_fold_stage_fail!(run, Int32(_CANDIDATE_STATUS_INVALID_CONTROL))
382446
_ordered_fold_stage_success(run) || return
383447
accumulator = _prepared_fold_accumulator(run.workspace.state)
384-
for position in Int32(1):run.stage.source_count
385-
item = @inbounds workspace.order[position]
386-
item == 0 && break
387-
step = run.transition(
388-
accumulator,
389-
@inbounds(workspace.values[item]), item,
390-
_stage_reads(
391-
run.stage, item,
392-
_OrderedFoldEvaluationValidation(workspace.status)
393-
)
394-
)
395-
_ordered_fold_stage_success(run) || return
396-
code, component_index, witness =
397-
_ordered_fold_stage_validate_step!(
398-
run.state, run.workspace.state, step
399-
)
400-
code == 0 || return _ordered_fold_stage_fail!(
401-
run, code,
402-
component_index, item, position, witness
403-
)
404-
_ordered_fold_stage_apply_step!(run.state, run.workspace.state, step)
405-
step.halt && return
406-
end
407-
return
448+
return _ordered_fold_stage_recur!(run, run.stage.traversal, accumulator)
408449
end
409450

410451
@generated function _ordered_fold_stage_commit_item!(
@@ -568,6 +609,7 @@ function _execute_ordered_fold_stage!(
568609
recurrence_stage = _OrderedFoldRecurrenceStage(
569610
prepared.stage.fields, prepared.stage.accesses,
570611
prepared.stage.control.prefix, prepared.stage.control.gate,
612+
_ordered_fold_recurrence_traversal(law.order),
571613
prepared.stage.source_count
572614
)
573615
boundary_parameters = (;

‎src/execution/program_inspection.jl‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -432,19 +432,22 @@ function _planned_stage_phases(entry::_StageLoweringEntry{
432432
phases = Any[_phase_fact(:ordered_fold_reset)]
433433
append!(phases, _planned_relation_phases(entry))
434434
push!(phases, _phase_fact(:ordered_fold_evaluate))
435-
extent = nextpow(2, max(Int(entry.admission.stage.source_count), 1))
436-
bitonic = 0
437-
width = 2
438-
while width <= extent
439-
distance = width >>> 1
440-
while distance >= 1
441-
bitonic += 1
442-
distance >>>= 1
435+
order = only(entry.admission.stage.publications).law.order
436+
if !(order isa _SourceOrder)
437+
extent = nextpow(2, max(Int(entry.admission.stage.source_count), 1))
438+
bitonic = 0
439+
width = 2
440+
while width <= extent
441+
distance = width >>> 1
442+
while distance >= 1
443+
bitonic += 1
444+
distance >>>= 1
445+
end
446+
width <<= 1
443447
end
444-
width <<= 1
448+
bitonic == 0 || push!(phases,
449+
_phase_fact(:ordered_fold_bitonic, bitonic))
445450
end
446-
bitonic == 0 || push!(phases,
447-
_phase_fact(:ordered_fold_bitonic, bitonic))
448451
append!(phases, (_phase_fact(:ordered_fold_validate_initialize),
449452
_phase_fact(:ordered_fold_apply),
450453
_phase_fact(:ordered_fold_finalize)))
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
using Test
2+
import KernelAbstractions
3+
import LocalMath
4+
5+
struct OrderedFoldSourceOrderDomain end
6+
struct OrderedFoldSourceOrderEvaluator end
7+
8+
struct OrderedFoldCanonicalKey end
9+
@inline (::OrderedFoldCanonicalKey)(value::Int32) = value
10+
struct OrderedFoldCanonicalIdentity end
11+
@inline (::OrderedFoldCanonicalIdentity)(value::Int32) = value
12+
struct AlternateOrderedFoldCanonicalKey end
13+
@inline (::AlternateOrderedFoldCanonicalKey)(value::Int32) = value
14+
struct AlternateOrderedFoldCanonicalIdentity end
15+
@inline (::AlternateOrderedFoldCanonicalIdentity)(value::Int32) = value
16+
17+
@inline function (::OrderedFoldSourceOrderEvaluator)(item::Int32, reads, parameters)
18+
return (event = LocalMath.FoldValue(
19+
item, something(reads[1][1].value)),)
20+
end
21+
22+
struct OrderedFoldSourceOrderTrace
23+
halt_at::Int32
24+
invalid_at::Int32
25+
end
26+
27+
@inline function (transition::OrderedFoldSourceOrderTrace)(
28+
state, value, item, reads)
29+
destination = item == transition.invalid_at ? Int32(2) : Int32(1)
30+
next = state.result[Int32(1)] * Int32(10) + value
31+
return LocalMath.FoldStep((result = LocalMath.BoundedWrites(
32+
(destination,), (next,), Int32(1)),);
33+
halt = item == transition.halt_at)
34+
end
35+
36+
function _ordered_fold_source_order_prepared(
37+
array_type, selected_values, emitted_values;
38+
selection::Symbol, halt_at::Int32 = Int32(0),
39+
invalid_at::Int32 = Int32(0), order = LocalMath.source_order())
40+
n = length(selected_values)
41+
source = LocalMath.Space(OrderedFoldSourceOrderDomain, n)
42+
singleton = LocalMath.Space(OrderedFoldSourceOrderDomain, 1)
43+
selected = LocalMath.Field(source, Bool)
44+
emitted = LocalMath.Field(source, Bool)
45+
initial = LocalMath.Field(singleton, Int32)
46+
result = LocalMath.Field(singleton, Int32)
47+
identity_relation = LocalMath.IdentityRelation(source)
48+
control = selection === :mask ? LocalMath.Control(; mask = selected) :
49+
selection === :subset ? LocalMath.Control(; subset =
50+
LocalMath.MaskedRelation(identity_relation, selected)) :
51+
error("unknown source-order selection")
52+
state = LocalMath.initialized_state(;
53+
result = LocalMath.FoldComponent(result; from = initial))
54+
publication = LocalMath.Publication((LocalMath.FoldPublication(
55+
LocalMath.PublicationValue(:event)),), LocalMath.OrderedFold(
56+
Int32, state,
57+
OrderedFoldSourceOrderTrace(halt_at, invalid_at); order))
58+
stage = LocalMath.Stage(
59+
source,
60+
(emitted = LocalMath.Access(emitted, identity_relation;
61+
required = true),),
62+
(publication,), LocalMath.Evaluator(OrderedFoldSourceOrderEvaluator()),
63+
control,
64+
LocalMath.SourceOrigin(
65+
@__FILE__, @__LINE__; label = :source_order_direct_traversal))
66+
selected_storage = array_type(selected_values)
67+
emitted_storage = array_type(emitted_values)
68+
destination = array_type(Int32[91])
69+
prepared = LocalMath.prepare(
70+
LocalMath.LocalLaw(stage),
71+
selected => selected_storage,
72+
emitted => emitted_storage,
73+
initial => array_type(Int32[0]), result => destination;
74+
backend = KernelAbstractions.get_backend(destination))
75+
return (; prepared, destination)
76+
end
77+
78+
function _ordered_fold_source_order_execute(prepared)
79+
return try
80+
wait(LocalMath.execute!(prepared))
81+
nothing
82+
catch error
83+
error
84+
end
85+
end
86+
87+
function ordered_fold_source_order_contracts(array_type)
88+
return @testset "source-order direct traversal preserves sparse semantics" begin
89+
selected = Bool[false, true, true, false, true, false, true]
90+
emitted = Bool[true, true, false, true, true, true, true]
91+
for selection in (:mask, :subset)
92+
witness = _ordered_fold_source_order_prepared(
93+
array_type, selected, emitted; selection)
94+
@test _ordered_fold_source_order_execute(witness.prepared) === nothing
95+
@test Array(witness.destination) == Int32[257]
96+
facts = LocalMath.inspect(witness.prepared)
97+
phases = map(phase -> phase.kind,
98+
only(facts.stages).planning.phases)
99+
@test :ordered_fold_bitonic ∉ phases
100+
@test facts.planning.base_provider_launch_count == 6
101+
102+
halted = _ordered_fold_source_order_prepared(
103+
array_type, selected, emitted; selection,
104+
halt_at = Int32(5))
105+
@test _ordered_fold_source_order_execute(halted.prepared) === nothing
106+
@test Array(halted.destination) == Int32[25]
107+
108+
rejected = _ordered_fold_source_order_prepared(
109+
array_type, selected, emitted; selection,
110+
invalid_at = Int32(7))
111+
failure = _ordered_fold_source_order_execute(rejected.prepared)
112+
@test failure isa LocalMath.LocalMathValidationError
113+
@test failure.contract === :runtime_ordered_fold_validation
114+
@test failure.actual.failure_class === :invalid_destination
115+
@test failure.actual.source_item == Int32(7)
116+
@test failure.actual.canonical_position == Int32(3)
117+
@test failure.actual.witness == Int32(2)
118+
@test Array(rejected.destination) == Int32[91]
119+
end
120+
121+
empty = _ordered_fold_source_order_prepared(
122+
array_type, Bool[], Bool[]; selection = :mask)
123+
@test _ordered_fold_source_order_execute(empty.prepared) === nothing
124+
@test Array(empty.destination) == Int32[0]
125+
@test LocalMath.inspect(empty.prepared).planning.base_provider_launch_count == 6
126+
127+
canonical = _ordered_fold_source_order_prepared(
128+
array_type, trues(7), trues(7); selection = :mask,
129+
order = LocalMath.canonical_by(
130+
OrderedFoldCanonicalKey(), OrderedFoldCanonicalIdentity()))
131+
@test _ordered_fold_source_order_execute(canonical.prepared) === nothing
132+
@test Array(canonical.destination) == Int32[1234567]
133+
canonical_facts = LocalMath.inspect(canonical.prepared)
134+
canonical_phases = map(phase -> phase.kind,
135+
only(canonical_facts.stages).planning.phases)
136+
@test :ordered_fold_bitonic in canonical_phases
137+
@test canonical_facts.planning.base_provider_launch_count == 12
138+
139+
alternate_canonical = _ordered_fold_source_order_prepared(
140+
array_type, trues(7), trues(7); selection = :mask,
141+
order = LocalMath.canonical_by(
142+
AlternateOrderedFoldCanonicalKey(),
143+
AlternateOrderedFoldCanonicalIdentity()))
144+
@test _ordered_fold_source_order_execute(
145+
alternate_canonical.prepared) === nothing
146+
@test Array(alternate_canonical.destination) == Int32[1234567]
147+
alternate_facts = LocalMath.inspect(alternate_canonical.prepared)
148+
@test only(alternate_facts.stages).planning.phases ==
149+
only(canonical_facts.stages).planning.phases
150+
@test alternate_facts.planning.base_provider_launch_count ==
151+
canonical_facts.planning.base_provider_launch_count
152+
end
153+
end

‎test/metal/ordered_fold_control.jl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using Metal
22
include(joinpath(@__DIR__, "..", "fixtures", "ordered_fold_control_contracts.jl"))
3+
include(joinpath(@__DIR__, "..", "fixtures", "ordered_fold_source_order_contracts.jl"))
34
Metal.functional() || error("ordered-fold control tests require functional Metal")
45
Metal.allowscalar(false)
56
ordered_fold_control_contracts(Metal.MtlArray)
7+
ordered_fold_source_order_contracts(Metal.MtlArray)

‎test/test_ordered_fold_stage_execution.jl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ using Test
22
import LocalMath
33
const LWFSE = LocalMath
44

5+
include(joinpath(@__DIR__, "fixtures", "ordered_fold_source_order_contracts.jl"))
6+
57
struct OrderedFoldExecutionNode end
68
struct OrderedFoldExecutionEvaluator end
79
@inline (::OrderedFoldExecutionEvaluator)(item::Int32, reads, parameters) =
@@ -285,3 +287,5 @@ end
285287
@test prepared.runtime.launches[1].stage.stage.parameter_slots ==
286288
(LWFSE._ParameterSlot{1}(),)
287289
end
290+
291+
ordered_fold_source_order_contracts(Array)

0 commit comments

Comments
 (0)