diff --git a/Cargo.lock b/Cargo.lock index 7f5065e35..a3c912aff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -112,7 +112,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -976,6 +976,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1352,6 +1363,27 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "dbus", + "zeroize", +] + [[package]] name = "der-parser" version = "10.0.0" @@ -1460,7 +1492,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1558,7 +1590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1590,6 +1622,12 @@ dependencies = [ "syn", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastbloom" version = "0.14.1" @@ -1598,7 +1636,7 @@ checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" dependencies = [ "getrandom 0.3.4", "libm", - "rand", + "rand 0.9.4", "siphasher", ] @@ -1607,6 +1645,11 @@ name = "fastcdc" version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf51ceb43e96afbfe4dd5c6f6082af5dfd60e220820b8123792d61963f2ce6bc" +dependencies = [ + "async-stream", + "tokio", + "tokio-stream", +] [[package]] name = "fastrand" @@ -1785,7 +1828,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1812,6 +1855,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -1857,7 +1901,7 @@ dependencies = [ "parking_lot", "portable-atomic", "quanta", - "rand", + "rand 0.9.4", "smallvec", "spinning_top", "web-time", @@ -2384,6 +2428,15 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "pkg-config", +] + [[package]] name = "libm" version = "0.2.16" @@ -2466,7 +2519,7 @@ dependencies = [ "lore-storage", "lore-transport", "parking_lot", - "rand", + "rand 0.9.4", "regex", "serde", "serde_json", @@ -2506,7 +2559,7 @@ dependencies = [ "opentelemetry", "opentelemetry-semantic-conventions", "opentelemetry_sdk", - "rand", + "rand 0.9.4", "serde", "serde_bytes", "serde_dynamo", @@ -2540,7 +2593,7 @@ dependencies = [ "lore-io", "parking_lot", "pin-project", - "rand", + "rand 0.9.4", "serde", "serde_json", "thiserror 2.0.18", @@ -2559,7 +2612,7 @@ dependencies = [ "clap", "lore", "parking_lot", - "rand", + "rand 0.9.4", "rand_distr", "serde", "serde_json", @@ -2597,6 +2650,7 @@ name = "lore-credential" version = "0.9.1-nightly" dependencies = [ "base64 0.22.1", + "dbus-secret-service", "jsonwebtoken", "keyring", "lore-base", @@ -2640,7 +2694,7 @@ dependencies = [ "mockall", "opentelemetry", "opentelemetry_sdk", - "rand", + "rand 0.9.4", "rs-consul", "temp-env", "tokio", @@ -2668,7 +2722,7 @@ dependencies = [ "lore-server", "lore-storage", "lore-transport", - "rand", + "rand 0.9.4", "rcgen", "reqwest", "ring", @@ -2776,7 +2830,7 @@ dependencies = [ "parking_lot", "prost-types", "quinn", - "rand", + "rand 0.9.4", "ring", "rustls", "rustls-native-certs", @@ -2817,6 +2871,7 @@ dependencies = [ "config", "ctor", "dashmap", + "diffy", "enum_dispatch", "futures", "hex", @@ -2844,9 +2899,11 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "pin-project", + "postgres", + "prost", "prost-types", "quinn", - "rand", + "rand 0.9.4", "rcgen", "reqwest", "ring", @@ -2904,7 +2961,7 @@ dependencies = [ "lz4-sys", "opentelemetry", "parking_lot", - "rand", + "rand 0.9.4", "serde", "smallvec", "thiserror 2.0.18", @@ -2929,7 +2986,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "rand", + "rand 0.9.4", "regex", "serde", "smallvec", @@ -2958,8 +3015,9 @@ dependencies = [ "prost-types", "quinn", "quinn-proto", - "rand", + "rand 0.9.4", "rcgen", + "reqwest", "rustls", "rustls-native-certs", "serde", @@ -3058,7 +3116,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -3125,7 +3183,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3236,6 +3294,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.37.3" @@ -3371,7 +3438,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand", + "rand 0.9.4", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -3445,6 +3512,25 @@ dependencies = [ "indexmap", ] +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -3489,6 +3575,50 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "postgres" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ad20e0aa0b24f5a394eab4f78c781d248982b22b25cecc7e3aa46a681605bd" +dependencies = [ + "bytes", + "fallible-iterator", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.10.1", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", + "uuid", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -3651,7 +3781,7 @@ dependencies = [ "libc", "once_cell", "raw-cpuid", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "web-sys", "winapi", ] @@ -3684,7 +3814,7 @@ dependencies = [ "fastbloom", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -3738,7 +3868,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", ] [[package]] @@ -3748,7 +3889,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -3760,6 +3901,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.5.1" @@ -3767,7 +3914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand", + "rand 0.9.4", ] [[package]] @@ -3948,7 +4095,7 @@ dependencies = [ "futures-util", "http 1.4.0", "mime", - "rand", + "rand 0.9.4", "thiserror 2.0.18", ] @@ -3992,7 +4139,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4406,7 +4553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4430,6 +4577,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4539,7 +4697,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4704,6 +4862,32 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.1", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -5129,12 +5313,33 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-width" version = "0.2.2" @@ -5294,6 +5499,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -5312,6 +5526,15 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.121" @@ -5430,6 +5653,19 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -5452,7 +5688,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6016,6 +6252,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "zerotrie" diff --git a/Cargo.toml b/Cargo.toml index ca2516df2..a3d31f776 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,20 +54,24 @@ bitcode = { version = "0.6.9", features = ["serde"] } bitflags = "2.9.4" blake3 = "1.8.2" bytes = { version = "1.11.1", features = ["serde"] } -chrono = { version = "0.4.42", default-features = false, features = ["alloc", "clock"] } +chrono = { version = "0.4.42", default-features = false, features = [ + "alloc", + "clock", +] } clap = { version = "4.5.47", features = ["derive", "string", "unicode"] } clap-markdown = "0.1.5" config = { version = "0.15.15", features = ["toml"], default-features = false } ctor = "0.2" crossbeam = "0.8.4" dashmap = "6.1.0" +dbus-secret-service = "4.1.0" diffy = "0.4.2" directories = "6.0.0" enum_dispatch = "0.3.13" tracing-appender = "0.2.3" tracing-ecs = "0.4.1" tracing-opentelemetry = "0.32.0" -fastcdc = "3.2.1" +fastcdc = { version = "3.2.1", features = ["tokio"] } futures = "0.3.31" glob-match = "0.2.1" governor = "0.10.1" @@ -85,17 +89,29 @@ keyring = { version = "3.6.3", features = [ libc = "0.2.175" lz4-sys = "1.11" zstd-sys = { version = "2.0", default-features = false, features = ["experimental"] } +memmap2 = "0.9.11" mockall = "0.13.1" opentelemetry = "0.31.0" -opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic", "tls", "tls-roots"] } +opentelemetry-otlp = { version = "0.31.0", features = [ + "grpc-tonic", + "tls", + "tls-roots", +] } opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] } -opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] } +opentelemetry-semantic-conventions = { version = "0.31.0", features = [ + "semconv_experimental", +] } parking_lot = { version = "0.12.4", features = ["arc_lock"] } pin-project = "1.1.10" +postgres = { version = "0.19.14", features = ["with-uuid-1"] } proc-macro2 = "1" prost = "0.14.1" prost-types = "0.14.1" -quinn = { version = "0.11.9", default-features = false, features = ["runtime-tokio", "rustls-ring", "bloom"] } +quinn = { version = "0.11.9", default-features = false, features = [ + "runtime-tokio", + "rustls-ring", + "bloom", +] } quinn-proto = { version = "=0.11.13", default-features = false } quote = "1.0.40" rand = "0.9.2" @@ -121,7 +137,10 @@ smallvec = "1.15.1" serde = { version = "1.0.219", default-features = false, features = ["derive"] } serde_bytes = "0.11.17" serde_json = { version = "1.0.143", default-features = false } -serde_with = { version = "3.14.0", default-features = false, features = ["alloc", "macros"] } +serde_with = { version = "3.14.0", default-features = false, features = [ + "alloc", + "macros", +] } serial_test = { version = "3.2.0" } socket2 = "0.6.0" syn = "2.0.106" @@ -140,12 +159,22 @@ tokio = { version = "1.47.1", features = [ "sync", "time", ] } -tokio-metrics = { version = "0.4.5", default-features = false, features = ["rt"] } -tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } +tokio-metrics = { version = "0.4.5", default-features = false, features = [ + "rt", +] } +tokio-stream = { version = "0.1.17", default-features = false, features = [ + "sync", +] } tokio-util = { version = "0.7.16", default-features = false, features = ["rt"] } toml = "0.9.5" trybuild = "1" -tonic = { version = "0.14.2", default-features = false, features = ["codegen", "router", "server", "tls-native-roots", "tls-webpki-roots"] } +tonic = { version = "0.14.2", default-features = false, features = [ + "codegen", + "router", + "server", + "tls-native-roots", + "tls-webpki-roots", +] } tonic-prost = "0.14.2" tonic-prost-build = "0.14.2" tower = { version = "0.5.2", default-features = false } @@ -156,16 +185,19 @@ tracing-test = "0.2.5" url = "2.5.7" urlencoding = "2.1.3" uuid = { version = "1.18.1", default-features = false, features = ["v4", "v7"] } -vergen = { version = "9.0.6", default-features = false, features = ["cargo", "rustc", "build", "si"] } +vergen = { version = "9.0.6", default-features = false, features = [ + "cargo", + "rustc", + "build", + "si", +] } webpki-roots = "1.0.7" windows-sys = "0.61.0" x509-parser = "0.18.0" -xxhash-rust = { version = "0.8.15", default-features = false, features = ["xxh3"] } -zerocopy = { version = "0.8.26", features = [ - "derive", - "alloc", - "simd", +xxhash-rust = { version = "0.8.15", default-features = false, features = [ + "xxh3", ] } +zerocopy = { version = "0.8.26", features = ["derive", "alloc", "simd"] } # Note: we currently control debug symbol splitting in .cargo/config.toml diff --git a/GRPC-REVISION-TREE-PLAN.md b/GRPC-REVISION-TREE-PLAN.md new file mode 100644 index 000000000..e4e47af30 --- /dev/null +++ b/GRPC-REVISION-TREE-PLAN.md @@ -0,0 +1,543 @@ +# Running the Memory-Based Revision Workflows over the gRPC Contracts + +Implementation plan for the browser → Node server → Lore server flow: what +the wire surface and server backends already provide, what the new +`lore_revision_tree_*` capabilities need from them, and what still has to be +built to run the no-working-tree write workflow through gRPC without buffering +whole files in the Node layer. Companion to `PROGRESS.md` (the SDK-side implementation) and +`docs/proposals/2026-05-14-low-level-revision-api.md` (which explicitly +scoped gRPC out and anticipated this follow-on: *"The proposal does not +preclude exposing a subset of these operations as RPCs later for genuinely +remote callers; that is a separate LEP"*). + +## Implementation progress + +- [x] Prove nested empty directories survive commit and reload. +- [x] Add shared `lore_revision::revision_tree` edit primitives for add, + exact directory creation, modify, delete, and move; keep the SDK verbs as + handle/event adapters. The SDK revision-tree suite passes (104 tests). +- [x] Split immutable revision construction from branch publication. +- [x] Add bounded streaming content ingestion and `UploadContent`. +- [x] Add `RevisionCreate`, idempotency, limits, and shared authoritative + publication orchestration. +- [x] Add server integration/end-to-end coverage and finish the contract docs. +- [x] Add the independent SDK remote-push follow-through (G1). +- [x] Add revision/path file downloads for thin clients: exact path + resolution, pinned revision metadata, short-lived logical Lore URLs, + bounded defragmentation streaming, propagated mid-stream errors, empty-file + handling, and single-range HTTP resume support. + +Implemented on 2026-07-14. The server test path now exercises +`UploadContent` → `RevisionCreate` → `ThinClientService.RevisionTree`, including +large multi-fragment input, top-level and nested empty directories, +idempotent retries, ordered modify/move/delete, changeset and upload limits, +tip conflicts, branch protection, and service-account bypass. Live +AWS/S3-backed persistence remains deployment-specific validation; the same +streaming writer targets the configured `ImmutableStore`, so the server sends +bounded fragments directly to the Lore S3 backend without a Node staging +file or bucket. + +The revision/path download extension below was implemented on 2026-07-15. + +The read-side browser flow is now implemented as +`ThinClientService.RevisionFileDownload`: + +```text +browser ──repo/revision/path──> Node +Node ──RevisionFileDownload──> Lore gRPC +Node <──pinned revision + URL suffix── Lore +browser ──signed GET/Range──> Lore HTTP ──bounded fragment reads──> store +``` + +The RPC resolves `identifier` (including `number = 0` tip queries) or +`signature` to a concrete revision, follows the exact file path through +authorized repository links, verifies the file's full repository/context +association and logical size, and returns the resolved repository, address, +size, mode, filename, expiry, and `/v1/presigned/...` URL suffix. The Node +server can return that URL or issue a redirect after joining it to the public +Lore HTTP origin. + +The signed URL streams the logical file through Lore rather than exposing raw +S3 fragment objects. It supports `Range: bytes=...`, `206`, `416`, +`Content-Range`, `Accept-Ranges`, `If-Range`, and an immutable address-based +`ETag`. +Fragment-list walks skip non-overlapping leaf payloads and slice boundary +leaves, so ranged downloads remain bounded. Stream items now carry errors; +missing or corrupt fragments terminate the HTTP body with an error instead of +being logged as a successful short response. Zero-address, zero-length files +return an empty body without a store read. + +This feature requires the Lore HTTP endpoint and +`presigned_url_hmac_key`; when they are unavailable the gRPC operation returns +`FailedPrecondition`. There is deliberately no materialized S3 download cache +and no download file-size cap. Deployments should bound concurrent downloads, +bandwidth, and idle duration operationally rather than buffering or rejecting +large logical files. + +--- + +## 1. The protocol landscape today + +There are **two generations** of wire contracts, both served simultaneously +(`lore-server/src/grpc/server.rs` registers every service below): + +### 1a. Legacy `urc.*` services + +- Protos: `lore-server/src/legacy/proto/{storage,revision,repository,environment}.proto` + (packages `urc.rpc` / `urc.model`; generated into + `lore-server/src/legacy/generated/` and `lore-proto/src/grpc/urc.*`). +- This is what the **SDK/CLI talks to today**: `lore-transport` defines the + `Storage` / `Revision` / `Repository` / … traits (`lore-transport/src/traits.rs`) + with both gRPC (`lore-transport/src/grpc/`) and QUIC implementations. The + storage handle's remote (`StoreInternal::remote` → + `endpoint.session_connection(partition)`) hands out exactly these traits. +- Server handlers are shared with v1 where possible via + `lore-server/src/grpc/handlers/*` (e.g. `branch_push.rs` backs both + generations). + +### 1b. `lore.*.v1` services (the go-forward contracts) + +Protos in `lore-proto/proto/lore/`, implementations in +`lore-server/src/grpc//v1/`: + +| Service | RPCs | Notes | +|---|---|---| +| `lore.storage.v1.StorageService` | `Get`, `GetMetadata`, `Put`, `Query`, `PresignDownload`, `Verify`, `Copy`, `MutableLoad`, `MutableStore`, `MutableCompareAndSwap` | Bidirectional streams for bulk transfer. `Put` **validates the payload hash server-side** (`protocol/storage/put.rs::validate_hash` via `lore_storage::hash_fragment`). Repository id travels as gRPC metadata (`REPOSITORY_ID_KEY`); identity via JWT interceptor. | +| `lore.revision.v1.RevisionService` | `BranchCreate/Delete/Get/List/Push`, `BranchMetadataGet/Set`, `RevisionList` | Charter (proto header): *"minimal, stable set of graph primitives."* `BranchPush` is the **only revision-graph write**: takes a revision signature already present in CAS, deserializes the state server-side, checks `parent_self` against the current tip (tip-collision → `FailedPrecondition` with the current latest embedded), verifies referenced fragments exist (`verify_fragments`), honors branch protection (service accounts bypass), runs pre/post hooks, emits notifications, and supports `force` / `fast_forward_merge`. | +| `lore.thin_client.v1.ThinClientService` | `RevisionInfo`, `RevisionTree`, `RevisionDiff`, `ContentDiff`, `RevisionFileDownload` | Charter: *"presentation helpers for clients that lack local cache or compute (web UIs)."* Read handlers build a pathless `RepositoryContext::new_server_context(...)` and walk `State` server-side — the very same machinery the revision_tree SDK surface uses. `RevisionTree` streams `TreeNode {path, node_type, address, last_changed_revision}`; `RevisionFileDownload` resolves an exact file and returns a pinned, short-lived logical Lore HTTP URL. | +| `lore.repository.v1.RepositoryService` | `RepositoryCreate/Delete/Get/List`, `RepositoryMetadataGet/Set` | Repo lifecycle incl. default-branch creation. Caller pre-generates ids (UUIDv7) for idempotent retries. | +| `lore.environment.v1`, lock, notification, admin | — | Not relevant to this workflow. | + +### 1c. The new SDK surface (recap) + +`lore_revision_tree_*` (this fork, see `PROGRESS.md`): load / resolve_path / +list_children / node_info / node_path / info / add / modify / delete / move / +metadata_set / metadata_get / commit / close, all against a storage handle, +no working tree. The commit pipeline (`lore_revision::commit::commit_tree`) +**already lives in `lore-revision`** — reachable from `lore-server`. The +per-verb edit logic (validation + node surgery + delta recording) currently +lives in the `lore` crate's verb bodies (`lore/src/revision_tree/*.rs`). + +**Dependency fact that shapes everything below:** `lore-server` depends on +`lore-revision`, `lore-storage`, `lore-transport`, `lore-proto` — **not** on +the `lore` crate. Server handlers can call `commit_tree` today but cannot +call the `lore` crate's revision_tree verbs. + +There is one additional boundary to fix before the server calls the commit +pipeline: `commit_tree` currently serializes the revision **and writes the +branch latest**. A server-side construct operation must not update the +authoritative branch before `BranchPush` protection, fragment verification, +hooks, notifications, and compare-and-swap run. The construction-only portion +must therefore be extracted from `commit_tree`; see **G4**. + +--- + +## 2. What already works end-to-end (thick-client path) + +A service that embeds the SDK (in-process `lore` crate) can run the full +no-clone write workflow against a remote server **today**, using only +existing wire contracts: + +``` +1. lore_storage_open (remote endpoint configured) +2. lore_storage_put → bytes land locally; content address returned +3. lore_revision_tree_load (store, repository, base revision) +4. add / modify / delete / move / metadata_set (in memory) +5. lore_revision_tree_commit remote_write=1 + → tree blocks, name tables, delta block, metadata fragment, file + fragments, and the 320-byte revision record are uploaded to the + server CAS through the existing storage protocol (immutable::write + with remote_write), and the *local* branch tip advances +6. urc.rpc / lore.revision.v1 BranchPush(branch_id, new_revision_signature) + → server re-validates (tip CAS, fragment verification, protection, + hooks) and advances the authoritative tip +``` + +Step 6 is the one seam that is not yet wrapped by the SDK surface: the +existing `lore branch push` verb is working-tree-based, and the transport's +`Revision::branch_push` is not yet called from any revision_tree verb. See +gap **G1**. + +The server-side validation story required by the LEP ("commit through this +surface honors the same branch semantics") holds on this path because +`BranchPush` is the same gate the file-system flow goes through. + +--- + +## 3. What a pure thin client (no SDK) can and cannot do + +Can, today: + +- **Read everything** the workflows need: `ThinClientService.RevisionTree` + (tree by revision, path-prefix + depth filters), `RevisionInfo`, + `RevisionDiff`, `ContentDiff`, `StorageService.Get/GetMetadata` (bytes by + address), `RevisionService.RevisionList` / `BranchGet` (by name!) / + `BranchList`. +- **Download one logical file by revision and path** through + `ThinClientService.RevisionFileDownload`, then redeem the returned Lore HTTP + URL directly from the browser with range/resume support. +- **Stream arbitrary raw file bytes into Lore** through `UploadContent`, then + construct and publish an address-backed changeset through `RevisionCreate`. +- **Upload already-framed storage fragments** through `StorageService.Put`. + This RPC validates the supplied fragment metadata and payload hash, but it + is not a raw-file upload API: the caller must already understand Lore's + chunking, compression flags, hashes, fragment-reference lists, and recursive + fragment-list roots. +- **Advance a tip to an existing revision** through `BranchPush`. + +Original gaps, now closed by this plan: + +- `UploadContent` now streams an arbitrary file as raw bytes and returns a Lore + address without requiring the caller to understand Lore fragmentation. +- `RevisionCreate` now turns a base revision plus path operations and uploaded + content addresses into a constructed, authoritatively published revision. + +The target thin-client flow is therefore two-phase: + +```text +browser body ──stream──> Node ──UploadContent stream──> Lore ──> S3 + │ + └── Address + logical size + +Node ──RevisionCreate(address-backed operations)──> Lore + │ + └── revision signature + number +``` + +Node is a backpressure-preserving proxy. It must not write temporary files or +concatenate the request body into one buffer. + +--- + +## 4. Gaps and required work + +### G1 — SDK: remote tip advance from the revision_tree surface (small) + +**Implemented.** `remote_write=1` now publishes through the remote revision +service after the local construction/upload. A lost push response is resolved +by querying the remote branch: observing the new revision is success; +observing a tip that moved from the expected base reports `BranchAdvanced` +and carries that tip on the terminal event. Other server rejections remain +commit failures. Remote publication failure invalidates the handle. + +`lore_revision_tree_commit(remote_write=1)` uploads all revision data but +advances only the local tip. For an SDK pipeline that targets a server, add +the follow-through: after a successful commit with `remote_write`, call the +remote's `Revision::branch_push` and surface the server's accept/reject on the +commit terminal event. This is useful for thick clients but is not on the +critical path for the browser → Node → Lore flow. + +### G2 — Storage: raw streaming content upload + +**Implemented.** `lore_storage::write_content_stream` uses async FastCDC and +an incremental recursive fragment-list builder. The gRPC handler adapts the +HTTP/2 stream directly to `AsyncRead`, and +`feature.upload_content_max_bytes` optionally enforces actual bytes read +(default: no file-size cap). + +Add a high-level client-streaming RPC to +`lore.storage.v1.StorageService`. Keep the existing `Put` unchanged as the +low-level fragment protocol. + +```proto +rpc UploadContent(stream UploadContentRequest) + returns (UploadContentResponse); + +message UploadContentRequest { + oneof part { + UploadContentHeader header = 1; // exactly once, first + bytes chunk = 2; + } +} + +message UploadContentHeader { + // New file: UUIDv7 generated by the Node service. Modification: the + // existing node's address.context. Zero may ask Lore to mint one. + bytes file_id = 1; + optional uint64 expected_size = 2; + // Caller-generated UUIDv7 used to make an interrupted retry identifiable. + bytes request_id = 3; +} + +message UploadContentResponse { + lore.model.v1.Address address = 1; + uint64 size = 2; // actual logical byte count observed by Lore +} +``` + +Implement a streaming counterpart to `lore_storage::write_content`: + +1. Adapt the tonic message stream to `AsyncRead`. +2. Feed it through `fastcdc::v2020::AsyncStreamCDC` using the same minimum, + expected, and maximum sizes as the current buffer writer. +3. Hash/compress/store each bounded leaf through the existing + `store_fragment` path so all current validation, deduplication, remote + storage, and write tracking remain shared. +4. Build fragment-reference lists incrementally. Flush a full list page to + storage and feed its reference into the next level instead of retaining one + reference for every file fragment until EOF. This keeps memory bounded for + very large files, including at the fragment-list levels. +5. Verify `expected_size` when supplied and return the root address plus the + actual logical size. + +With the AWS immutable store configured, this path naturally writes Lore +fragments to S3 and records Lore's DynamoDB metadata/associations. Do not add +browser-presigned writes directly into the final Lore bucket in v1: an +arbitrary S3 object is not a Lore fragment graph and bypasses hash validation +and repository/context association. If bypassing Node/Lore bandwidth later +becomes necessary, use a separate staging bucket plus a finalize/import RPC; +do not expose the final Lore object layout to browsers. + +Keep bytes out of `RevisionCreate`, including for small files. A batched +small-content upload can be added later if profiling shows per-file RPC +overhead matters. + +### G3 — RevisionService: one atomic RevisionCreate RPC + +**Implemented.** The unary changeset handler validates and reserves the +request id, derives file sizes from immutable metadata, applies operations in +order, constructs without changing latest, and publishes through the shared +authoritative BranchPush orchestration. + +Add one path-keyed changeset RPC rather than stateful 1:1 handle verbs. The +client has one logical operation; the server still separates immutable +revision construction from authoritative branch publication internally. + +```proto +rpc RevisionCreate (RevisionCreateRequest) returns (RevisionCreateResponse); + +message RevisionCreateRequest { + bytes request_id = 1; // caller-generated UUIDv7 + bytes branch_id = 2; + bytes base_revision_signature = 3; // caller's observed tip; zero = initial + string commit_message = 4; + repeated MetadataEntry metadata = 5; + repeated Operation operations = 6; +} + +message MetadataEntry { + string key = 1; + bytes value = 2; + uint32 format = 3; // validated against Lore's supported metadata formats +} + +message Operation { + oneof op { + PutFile put_file = 1; + CreateDirectory create_directory = 2; + DeletePath delete_path = 3; + MovePath move_path = 4; + } +} + +message PutFile { + string path = 1; + uint32 mode = 2; + lore.model.v1.Address address = 3; + // No caller-supplied size: derive it from stored fragment metadata. +} + +message CreateDirectory { + string path = 1; + uint32 mode = 2; +} + +message DeletePath { string path = 1; } +message MovePath { string source = 1; string destination = 2; } + +message RevisionCreateResponse { + bytes revision_signature = 1; + uint64 revision_number = 2; +} +``` + +Server semantics: + +1. Validate request byte/count limits, ids, paths, operation shapes, and + idempotency key before loading a tree. +2. Confirm the authoritative branch tip equals + `base_revision_signature`; return `FailedPrecondition` with the current tip + on a mismatch. +3. Build a pathless `RepositoryContext`, deserialize the explicit base, and + apply operations in request order through the shared edit primitives. +4. Batch-load `PutFile` address metadata, require every address to exist in + the repository/context, derive each logical size server-side, and preserve + an existing file's file id on modification. +5. Construct and serialize the new revision into immutable CAS **without + changing the branch latest**. +6. Publish it through the same protection, hook, fragment-verification, + notification, history-acceleration, and branch compare-and-swap path as + `BranchPush`. +7. Return the published revision signature and number. A final CAS race is a + `FailedPrecondition` carrying the newly observed tip. + +Do not expose `force` or `fast_forward_merge` on `RevisionCreate` v1. A stale +web edit must fail explicitly so the Node application can reload and decide +how to reconcile it. + +The branch update is the atomic visibility point. Immutable content and a +constructed revision may remain unreachable after cancellation or a CAS +loss; that is acceptable content-addressed-storage behavior and can be handled +by retention/garbage collection rather than rollback. + +Use `request_id` for a real idempotency record keyed by repository + branch + +request id. A retry with the same request digest returns the original result; +reusing the id for different content is rejected. Deterministic addresses +alone are not enough because a lost successful response leaves the branch +already advanced past the request's base. + +### G4 — Code movement and the construction/publication split + +**Implemented.** Shared edits live in `lore-revision`; SDK verbs retain their +event/handle adapters. `construct_tree_revision` and the shared server +`publish_revision` boundary prevent pre-publication tip changes. + +Because `lore-server` cannot depend on the `lore` crate, move the validated +tree-edit logic currently inline in +`lore/src/revision_tree/{add,modify,delete,move_node}.rs` into a +`lore_revision::revision_tree` module. Include the directory-create primitive +described in **G5**. Leave the `lore` verbs as event/handle wrappers around +those shared operations. + +Also split the current commit pipeline: + +- `construct_tree_revision(...)`: freeze edits, rehash directories, create the + delta/history/metadata, and serialize the immutable revision; never load or + update branch latest. +- `commit_tree(...)`: the SDK wrapper that performs its existing local + tip-collision check, calls `construct_tree_revision`, and advances the local + branch tip. +- A shared server publication orchestration used by both `BranchPush` and + `RevisionCreate`: protection, pre-hook, fragment verification, CAS, + notification, post-hook, and response hooks. + +Do **not** implement the server path as current `commit_tree` followed by +`handlers::branch_push::push()`. Today `commit_tree` writes latest first; +`push()` can then take its "already current" early return before verifying new +fragments. `RevisionCreate` also must not call only the low-level `push()` +helper and accidentally skip the hooks/notifications owned by the handler. + +### G5 — Explicit empty-directory support + +**Implemented.** Exact directory creation is shared by SDK and server paths, +with commit/reload and thin-client tree coverage for top-level and nested +empty directories. + +Empty directories are a v1 operation, not a deferred extension. + +Current SDK status: + +- `lore_revision_tree_add(kind = DIRECTORY)` already accepts an empty + directory with `size = 0` and a zero address, calls `State::node_add`, and + marks it `StagedAdd`. +- `commit_tree_freeze` records staged directories and `rehash_directory` + computes the empty-directory hash/size and clears the staged bits. +- Read operations already represent an empty directory as a directory with no + child events. + +What is still required: + +- Extract/add a server-callable `create_directory(state, parent, name, mode)` + primitive in `lore-revision` and expose it as the path-keyed + `CreateDirectory` operation. +- Define it as exact creation, not implicit `mkdir -p`: the parent must exist + and be a directory; an occupied target path is rejected. Operations are + ordered, so callers create missing ancestors explicitly first. +- Add a focused SDK commit/reload test proving a newly added empty directory + survives revision serialization. The existing add tests prove the + in-memory mutation but do not currently cover commit + reload for an empty + directory. +- Add server tests for a root empty directory, nested empty directory, + directory beside files, missing parent, occupied path, move, and delete. + The end-to-end test must confirm `ThinClientService.RevisionTree` emits the + empty directory after `RevisionCreate`. + +### G6 — Service placement + +`RevisionCreate` belongs in `lore.revision.v1.RevisionService`: it is an +authoritative revision-graph write and shares the `BranchPush` publication +gate. Keep `ThinClientService` read-only. `UploadContent` belongs in +`lore.storage.v1.StorageService` because it turns bytes into a content address +without revision semantics. No new one-RPC services are needed. + +### G7 — Repository bootstrap for the no-clone flow + +Creating a repository and default branch remotely already works through +`RepositoryService.RepositoryCreate`, and `BranchGet(name)` resolves branch +names to ids. No new RPC is required; the Node layer should resolve once and +use ids for upload/create calls. + +### G8 — Tests and contract documentation + +**Implemented for the in-memory/local-store server suite.** Live S3-backed +verification is intentionally left to an environment with the configured AWS +backend and credentials. + +- Storage unit/integration tests: empty content, below/at/above fragment + threshold, many chunks, recursive fragment-list levels, cancellation, + declared-size mismatch, retry with the same file id/request id, bounded + memory/backpressure, and AWS-backed persistence where available. +- Revision handler tests mirroring `branch_push.rs`: tip collision, protected + branch, service-account bypass, hooks, notifications, idempotent retry, + deleted branch, and a CAS race after construction. +- Construct-specific tests: every operation kind, duplicate/conflicting paths, + unknown base, address absent from CAS, context/file-id mismatch, server-side + size derivation, empty operation list, and the empty-directory cases in G5. +- End-to-end: `UploadContent` → `RevisionCreate` → + `ThinClientService.RevisionTree` / `RevisionService.RevisionList`, including + at least one large streamed file and one empty directory. +- Proto documentation: first-message upload header rules, cancellation and + orphan behavior, size/count limits, idempotency, conflict details, and the + exact branch-publication semantics. + +--- + +## 5. Settled design decisions + +- **Q1 — File ids.** The Node service generates UUIDv7 for new files and + reuses the existing `address.context` for modifications. The upload header + carries that id; Lore may mint one only when omitted. The browser does not + implement Lore identity rules. +- **Q2 — Upload framing.** Add `StorageService.UploadContent`; browser/Node + callers stream raw bytes and Lore owns fragmentation. Keep bytes, including + tiny inline payloads, out of `RevisionCreate` v1. +- **Q3 — Tip advance.** `RevisionCreate` is one construct-and-publish RPC to + the caller, but internally immutable construction and authoritative branch + publication are separate. The final shared BranchPush CAS is the visibility + point. +- **Q4 — Operations.** V1 includes `PutFile`, `CreateDirectory`, `DeletePath`, + and `MovePath`. Empty-directory creation is required now; links remain + additive follow-up work. +- **Q5 — Limits.** `RevisionCreate` limits are changeset limits: encoded + request bytes, operation count, metadata count/bytes, and path sizes. Keep it + unary; streaming operations would bypass the protobuf envelope limit but + would not bound the mutated `State` retained until publication. Configure a + cap and return `ResourceExhausted` with the applicable limit. +- **Upload size limits.** File size is independent of `RevisionCreate`. A + correct streaming writer does not need a small file-size cap for memory + safety, but a browser-facing deployment still needs configurable per-upload, + per-user/repository quota, concurrency, bandwidth, deadline, and idle limits. + Enforce total bytes incrementally; do not trust only the declared size or + HTTP `Content-Length`. +- **S3.** S3 is the durable backend behind Lore, not a temporary Node handoff + and not a final bucket browsers write into directly. A staging-bucket import + flow is a later bandwidth optimization if measurements justify it. + +--- + +## 6. Suggested order of work + +1. **G4 + G5** — extract the shared edit primitives, add exact empty-directory + creation, add commit/reload coverage, and split immutable construction from + branch publication while keeping the SDK tests green. +2. **G2 storage core** — implement the bounded streaming content writer and + incremental recursive fragment-list builder in `lore-storage`. +3. **G2 + G3 protos/services** — add `StorageService.UploadContent` and + `RevisionService.RevisionCreate`, generated bindings, registration, and + configuration limits. +4. **G3 server handler** — apply address-backed operations, construct without + advancing latest, and publish through the shared BranchPush orchestration. +5. **G8** — complete handler, cancellation, conflict, empty-directory, large + upload, and end-to-end tests plus proto/contract documentation. +6. **G1** — add SDK remote-push follow-through independently or in parallel if + the thick-client path is also needed. diff --git a/PROGRESS.md b/PROGRESS.md new file mode 100644 index 000000000..f1e4033b7 --- /dev/null +++ b/PROGRESS.md @@ -0,0 +1,125 @@ +# Low-Level Memory-Based Revision Control API — Implementation Progress + +Tracking the completion of `docs/proposals/2026-05-14-low-level-revision-api.md` +(the low-level part): the `lore_revision_tree_*` surface in the `lore` crate and +its capi. + +## Baseline (already landed before this work) + +- `lore/src/revision_tree/` module: handle type + process-global registry + + `RevisionTreeGuard` in-flight protocol (`handle.rs`), shared dispatcher + (`call.rs`), and full implementations with tests for the read/lifecycle side: + `load`, `close`, `resolve_path`, `list_children`, `node_info`, `node_path`, + `info`. +- All `lore_revision_tree_*` event payload structs in + `lore-revision/src/event/revision_tree.rs` and their `LoreEvent` variants — + including the write-side events. +- Argument structs (doc + layout only, no implementations) for the write verbs: + `add.rs`, `modify.rs`, `delete.rs`, `move_node.rs`, `metadata_set.rs`, + `metadata_get.rs`, `commit.rs`. +- capi wrappers + generated `lore.h` declarations for the read side only. +- `lore/cbindgen.toml` already listed every write-verb args type in the export + include list. + +## Work items + +- [x] `lore-revision/src/event.rs` — appended `BranchAdvanced = 5` to + `LoreErrorCode` (the `RevisionTreeCommitComplete` contract references + it). `lore/src/storage/mod.rs` severity ranking extended to match. +- [x] `lore/src/revision_tree/handle.rs` — added the `pending_delta` + accumulator (delta entries recorded by `delete`), the per-commit + write-context builder (`synth_repository_write_context`), and moved the + handle's `State` behind an outer `RwLock>` so a successful + commit can swap in a state freshly deserialized from the new revision + (verbs snapshot once at entry via `RevisionTreeInternal::state()`). +- [x] `add` verb — parent/kind/name validation (duplicate names, link + parents, discarded slots, separator names rejected at call time), + `State::node_add` + `node_mark(StagedAdd)`, terminal + `RevisionTreeAddComplete` with the new node id. 7 tests. +- [x] `modify` verb — leaf-only update of `mode`/`size`/`address.hash` + preserving the `file_id` context (mismatched non-zero context rejected); + staged-add nodes stay adds. 5 tests. +- [x] `delete` verb — recursive `StagedDelete` mark (mirrors `stage_delete` + minus filesystem events) then immediate discard (children via + `node_discard_nopatch`, target via `node_discard_patch`), recording + `NodeDelta` entries on the handle for commit. Reads no longer observe + the subtree. 6 tests. +- [x] `move` verb — unlink/relink chain surgery mirroring `stage_move`, with + optional rename via the block name table; `file_id` preserved; + cycle-into-own-subtree and occupied destination names rejected at call + time; `StagedMove` marking recurses through moved directories. 7 tests. +- [x] `metadata_set` / `metadata_get` verbs — pending-metadata accumulator + write with per-format decoding; reads consult pending edits first, then + the loaded revision's frozen Metadata fragment; a missing key emits no + value event and completes 0. 9 tests. +- [x] `lore-revision/src/commit.rs` — `commit_tree`: the memory-based commit + pipeline. Same tip-collision check, `NothingStaged` guards, + `prepare_commit_metadata`, delta block, `weave_history`, + `State::serialize`, and `branch::store_latest` advance as the + file-system-based commit — with the working-tree walk replaced by + `commit_tree_freeze` (records staged nodes into the delta and clears + leaf change flags; `rehash_directory` consumes directory bits). + Graceful tracker drain mirrors `commit_staged_revision`. +- [x] `commit` verb — wraps `commit_tree` behind the revision-tree + dispatcher: per-repository write token (no working-tree path to key + on), write-capable sibling context, remote gating via + `set_disable_upload`, then `Revision::branch_push` when + `remote_write=1`; a lost response is reconciled with `branch_query`. + `BranchAdvanced` is reported with the observed local or remote tip on + the terminal event, followed by state swap + pending-buffer drain on + success and handle invalidation on failure (per the handle contract). +- [x] Discarded-slot guards on the node-id entry points (`node_info`, + `node_path`, `list_children` targets and the write verbs) — a + discarded slot keeps its name for history weaving, so ids of deleted + nodes now fail with `INVALID_ARGUMENTS` instead of reporting stale + records. +- [x] `lore/src/interface.rs` — sync + async capi wrappers for the seven new + verbs; `LoreCommand` variants added for the service-delegation path; + `lore-capi/lore.h` regenerated by the build (14 new entry points, + `LORE_ERROR_CODE_BRANCH_ADVANCED`). +- [x] Build + tests: `cargo test -p lore -p lore-revision --no-fail-fast` — + all targets pass (lore lib: 132, lore-revision lib: 150, plus + integration suites) except `lore-revision/tests/quic.rs + test_futures_size`, which fails identically on the pristine tree + (verified via stash) — pre-existing, unrelated (it measures a + `lore-transport` future size; that crate is untouched) - IGNORE this test. + +## Design decisions (unresolved questions the proposal deferred) + +- **`add` kind encoding** — the scaffolding doc referenced a non-existent + `NodeKind` (FILE=1, DIRECTORY=2, LINK=3); the implementation uses the + `LoreNodeType` encoding the read verbs already emit (DIRECTORY=0, FILE=1, + LINK=2), matching the proposal example's `LORE_NODE_TYPE_FILE`. +- **`add` on an existing name** — rejected at call time with + `INVALID_ARGUMENTS` (per "the handle-side write operations reject + malformed edits at call time"); callers delete or modify instead. +- **Non-leaf `add` arguments** — DIRECTORY honors `mode`, requires zero + `size`/`address` (computed at commit); LINK stores the target + `(revision, repository)` in `address`, targets the linked tree's root, + requires zero `size` and a non-zero target revision. +- **Address validation timing** — at commit: the rehash rejects zero-hash + non-empty files, and unreachable addresses are detected at push, same as + the file-system-based path. +- **Tip-collision behavior on `commit`** — follows the file-system-based + precedent: fail with `BranchAdvanced`; the terminal event carries the + observed tip in `new_tip_hash` so callers reload without a round-trip. +- **Node-id validity across commit** — a successful commit swaps in a state + freshly deserialized from the new revision; node ids are positional, so + ids remain stable, and ids of deleted nodes fail with + `INVALID_ARGUMENTS` via the discarded-slot guards. +- **Commit failure poisons the handle** — per the pre-existing handle + contract ("Set by close (or any commit failure)"); pre-flight argument + validation (zero branch id) fails without poisoning. + +## Log + +- 2026-07-13 — Survey completed. Read side + eventing + registry fully landed; + write verbs were argument-struct scaffolding; capi had read-side entry points + only. +- 2026-07-13 — Write verbs, `commit_tree`, capi wrappers, and tests landed. + Full suite green (one pre-existing unrelated failure, see above); + `lore-capi/lore.h` regenerated. 101 revision_tree unit tests total. +- 2026-07-14 — Shared tree-edit primitives and construction-only commit split + landed for the gRPC implementation. `remote_write=1` now completes the + authoritative remote branch push and surfaces rejection on the commit + terminal event. See `GRPC-REVISION-TREE-PLAN.md` for the server flow. diff --git a/TaskInsights.md b/TaskInsights.md new file mode 100644 index 000000000..fd9178edd --- /dev/null +++ b/TaskInsights.md @@ -0,0 +1,203 @@ +## Bottom line + +The right abstraction is : + +```text +upload bytes → receive content address +load revision tree in memory +add / modify / delete nodes +commit tree directly to a branch +``` + +Lore has a draft proposal called **Low-Level Memory-Based Revision Control API** that explicitly targets automated services which need to construct revisions from buffers without creating a working tree. + +## What exists today + +The normal Lore workflow is still fundamentally filesystem-based: + +* `stage` reads the requested file from the filesystem. +* A missing filesystem path is interpreted as a deletion. +* `commit` rereads staged files from disk and writes their contents into immutable storage. +* The current commit implementation calls `repository.require_path()` and ultimately `write_from_file_with_tracker(...)`, followed by filesystem metadata reads. + +That means a pathless `RepositoryContext` by itself is not sufficient. You could load the state and alter nodes, but the ordinary commit path would still expect files on disk. + +However, Lore already has most of the foundations. + +### Pathless repository contexts + +`RepositoryContext` already supports `path: None` specifically for server-side handlers and in-memory revision-tree operations. Filesystem operations use `require_path()` and deliberately fail on these contexts. Lore also has `new_server_context`, which connects the server stores without a checkout. + +### Uploading bytes without a checkout + +Lore already exposes a storage API that accepts buffers and returns content addresses. Its input includes: + +* Repository/partition + +* File identity context + +* Raw bytes + +* Whether to upload remotely + +### Direct revision-tree API + +The repository now contains a `revision_tree` API explicitly described as a low-level, memory-based revision-control API. It has modules for: + +```text +load +resolve_path +list_children +node_info +add +modify +delete +move +metadata_set +commit +close +``` + +The read side is implemented and has recently landed: load/close, path resolution, child listing, node info and node paths. + +But the important write files—`add.rs`, `modify.rs`, `delete.rs`, and `commit.rs`—currently contain API types and documentation rather than complete implementations. + +So the exact feature you need appears to be **actively under construction upstream**. + +## How difficult would it be? + +| Part | Difficulty | Why | +| ---------------------------------- | ----------: | --------------------------------------------------- | +| Store uploaded bytes | Low | Already implemented through `lore_storage_put` | +| Load a revision without checkout | Low | `revision_tree_load` already works | +| Resolve files/directories | Low | Read-side tree API already works | +| Add/modify/delete tree nodes | Low–medium | Internal `State` primitives already exist | +| Create a proper revision | Medium–high | Must avoid the current filesystem-based commit path | +| Branch advancement and concurrency | Medium | Existing branch semantics can be reused | +| HTTP API around it | Low | Thin Axum handler once the library API works | + +The hard part is **not editing the tree**. Lore already has internal operations capable of replacing or adding a node using its `address`, `size`, and `mode`, and direct deletion logic that marks a subtree deleted. + +The hard part is constructing and publishing the final revision without entering `commit_file()`, because `commit_file()` hashes and stores the file from the filesystem even if the node already has a valid uploaded address. + +The new proposal solves this by making content storage and tree mutation separate: + +```text +lore_storage_put(bytes) + ↓ +Address { hash, context } + ↓ +revision_tree_add / modify(address) + ↓ +revision_tree_commit(branch) +``` + +That exact sequence is documented as the canonical no-working-tree write path. + +## What the web API should look like + +Internally: + +```text +1. Read the current branch tip. +2. Verify it equals baseRevision. +3. Open the storage handle. +4. Load baseRevision into a revision-tree handle. +5. For every put: + a. Upload bytes through storage_put. + b. Resolve the existing node or parent directory. + c. Modify the node or add a new one. +6. For every delete: + a. Resolve the node. + b. Delete it from the in-memory tree. +7. Set commit metadata. +8. Commit the tree and atomically advance the branch. +9. Close the handle. +10. Return the new revision hash. +``` + +For modifying an existing file, preserve its existing file identity—the address `context`—while changing the content hash. Lore’s proposed `modify` operation explicitly preserves that identity. New files should receive a fresh context, such as a UUIDv7. + +Using `baseRevision` is important. The existing commit path already checks whether another instance has advanced the branch and rejects stale work with `BranchAdvanced`; your web endpoint should expose that as an HTTP conflict rather than silently overwriting changes. + +### Best long-term path + +Build against the new `revision_tree` API and implement the missing write verbs in your Lore fork if you need them before upstream finishes. + +The likely implementation work is: + +```text +revision_tree/add.rs +revision_tree/modify.rs +revision_tree/delete.rs +revision_tree/move_node.rs +revision_tree/metadata_set.rs +revision_tree/commit.rs +``` + +For add/delete/modify, reuse or extract logic from: + +```text +stage_single_node +stage_delete +State::node_add +State::node_mark +State::node_discard_* +``` + +For commit, do not adapt the filesystem walker. Create an address-backed commit path that: + +* Validates referenced content addresses. +* Clears temporary change flags. +* Rehashes affected directories. +* Generates the tree delta and history. +* Serializes metadata and the state. +* Advances the branch using existing protection/CAS behavior. + +The strongest conclusion here is: **Lore’s authors agree with your premise and are already implementing the exact no-checkout API you need.** The practical move is to follow or finish that API, then place a small atomic changeset endpoint over it. + + +**Code directory:** +[J:\test\lore-new-api\lore\src\revision_tree](J:\test\lore-new-api\lore\src\revision_tree) + +The API is being implemented incrementally. The module currently includes: + +```text +add.rs +close.rs +commit.rs +delete.rs +info.rs +list_children.rs +load.rs +metadata_get.rs +metadata_set.rs +modify.rs +move_node.rs +node_info.rs +node_path.rs +resolve_path.rs +``` + +### What is implemented + +The read/lifecycle side has real implementations: + +* `load` and `close` +* `resolve_path` +* `list_children` +* revision and node info +* `node_path` + +For example, `load` creates a pathless `RepositoryContext`, deserializes the requested revision into memory, registers an opaque handle, and returns it. + +### What is not implemented yet + +The parts you specifically need—directly adding, modifying, deleting and committing files—are currently largely **scaffolding/API definitions**: + +* `add.rs` defines the input containing parent node, name, mode, size and content address. +* `modify.rs` defines changing a leaf’s address, size and mode. +* `delete.rs` defines recursive deletion. +* `commit.rs` currently defines the commit arguments/options, but the fetched file does not yet contain the full implementation. + +So it is **real active implementation**, but the write path is not finished yet. diff --git a/common.ps1 b/common.ps1 new file mode 100644 index 000000000..f44382db0 --- /dev/null +++ b/common.ps1 @@ -0,0 +1,91 @@ +Set-Location J:\test\lore-new-api + +$grpc = "localhost:41337" +$http = "http://localhost:41339" +$protoRoot = ".\lore-proto\proto" + +# Run this and copy the Token from the block WITHOUT a Resource field. +lore auth list --with-token + +$env:LORE_GRPC_TOKEN = "eyJhbGciOiJSUzI1NiIsImtpZCI6IlZ4N1hNOTI5eTd3QUpwUmNzeDdCb3RVTElhSTZOWnVuIn0.eyJzdWIiOiIyZmExMDFjOC1hZDRlLTRiYWEtYjdhNC0wZjk3NmY0YjRhMDAiLCJpYXQiOjE3ODQyNTUwMjMsImV4cCI6MTc4Njg0NzAyMywibmFtZSI6IllhaXIiLCJ1c2VyX2lkIjoiMmZhMTAxYzgtYWQ0ZS00YmFhLWI3YTQtMGY5NzZmNGI0YTAwIiwicHJlZmVycmVkX3VzZXJuYW1lIjoib3p5YWlyODVAZ21haWwuY29tIiwiaXNfc2VydmljZV9hY2NvdW50IjpmYWxzZSwiZW52IjoiREVGQVVMVCIsImlkcCI6ImJldHRlci1hdXRoIiwiaXNzIjoibG9yZS1hdXRoLmxvY2FsIiwiYXVkIjpbImxvcmUueW91cmRvbWFpbi5jb20iLCJsb2NhbGhvc3QiLCIxMjcuMC4wLjEiLCJodHRwOi8vMTI3LjAuMC4xOjg3ODcvIl19.GYyxYvL53GYB0T8kGCtbt-2tGJrlxjxiAHZBXXEfXMC1UYaPDVawp1O9xpLkPaKV4VwrjU6Kox9FrkddlDJ_tgi7JPCfO2DTscxly2kAUdmPLLIaSmKysG-5kkiO6HN55iO0ArQDK2sEmnEHmHtPvH9eDlL_Wd1gF-kIDjzdviAn3YMTZsfVt_ZO5Fpwgdn_NmpvK39QrDfYGMrGbgOH1r_dlQVGyJotd-5bdizSGwZ19BU-46eTfE7DsFzYgW1UcU8_jMHUY3Jo76otBn1BlwdYFgiFxN-y3arAfxCB62wFJA5XLplpBDV6j_-2gj3n3xPx5SAuqKY73Hbhmt_tBw" + +$baseArgs = @("-plaintext", "-import-path", $protoRoot, "-expand-headers", "-rpc-header", 'authorization: Bearer ${LORE_GRPC_TOKEN}') + + +# '{}' | & grpcurl @baseArgs ` +# -proto "lore/repository/v1/repository.proto" ` +# -d '@' ` +# $grpc ` +# "lore.repository.v1.RepositoryService/RepositoryList" + + +$repoName = "space9" + +$repoRequest = @{ + name = $repoName +} | ConvertTo-Json -Compress + +$repoText = ($repoRequest | & grpcurl @baseArgs ` + -proto "lore/repository/v1/repository.proto" ` + -d '@' ` + $grpc ` + "lore.repository.v1.RepositoryService/RepositoryGet") -join "`n" + +$repo = $repoText | ConvertFrom-Json + +$repoB64 = $repo.repository.id +$branchB64 = $repo.repository.defaultBranchId + +$repoHex = ( + [Convert]::FromBase64String($repoB64) | + ForEach-Object { $_.ToString("x2") } +) -join "" + +"Repository resource: $repoHex" +"Default branch: $($repo.repository.defaultBranchName)" + +$env:LORE_GRPC_TOKEN = "eyJhbGciOiJSUzI1NiIsImtpZCI6IlZ4N1hNOTI5eTd3QUpwUmNzeDdCb3RVTElhSTZOWnVuIn0.eyJzdWIiOiIyZmExMDFjOC1hZDRlLTRiYWEtYjdhNC0wZjk3NmY0YjRhMDAiLCJpYXQiOjE3ODQyNTUwMjgsImV4cCI6MTc4Njg0NzAyOCwibmFtZSI6IllhaXIiLCJ1c2VyX2lkIjoiMmZhMTAxYzgtYWQ0ZS00YmFhLWI3YTQtMGY5NzZmNGI0YTAwIiwicHJlZmVycmVkX3VzZXJuYW1lIjoib3p5YWlyODVAZ21haWwuY29tIiwiaXNfc2VydmljZV9hY2NvdW50IjpmYWxzZSwiZW52IjoiREVGQVVMVCIsImlkcCI6ImJldHRlci1hdXRoIiwicmVzb3VyY2VzIjpbeyJyZXNvdXJjZV9pZCI6InVyYy02OTRiMTA4ZmEyMTA0YzNlYjkyNDZmNTU5YTRlYTk4OCIsInBlcm1pc3Npb24iOlsicmVhZCIsIndyaXRlIiwiYWRtaW4iLCJvd25lciIsIm9ibGl0ZXJhdGUiLCJtaWdyYXRlIl19XSwiZ3JvdXBzIjpbXSwiaXNzIjoibG9yZS1hdXRoLmxvY2FsIiwiYXVkIjpbImxvcmUueW91cmRvbWFpbi5jb20iLCJsb2NhbGhvc3QiLCIxMjcuMC4wLjEiLCJodHRwOi8vMTI3LjAuMC4xOjg3ODcvIl19.o3Xd9qDqgzB_Q8VsTO8oPSS8S15bB-pBpRV5cAuycLeSXA4HbihR1MPnbyE5VORuO5XnrBQCAor3MX8Ht_qW3W9kUf_C-ARkp9HiKmsIq7PNGWR3ttXhUMHPAgmJEvLEvn_Dg25Ky5mVsmsDBldVUotpGM1gWzAPhxxAM0_Hyke0H1lc9E3jb1bevxN6JWktH__Kbu2SInbDhJZBjkDoe_SDGzpGRquRimyjTMTDmtjYixfKDBlYhTZUAN2I5EG8BhYeBZkkgCKRefMmgUqPGKAsXjS_9TrvsBNS7q28OprhQ3GET5DiDBw21wAjTnEkswrwcpq9PUW0-Mqhwwzshw" + +$repoArgs = $baseArgs + @( + "-rpc-header", "urc-repository-id-bin: $repoB64" +) + +$treeRequest = @{ + identifier = @{ + branchId = $branchB64 + number = "0" + } + maxDepth = 3 +} | ConvertTo-Json -Depth 5 -Compress + +$treeRequest | & grpcurl @repoArgs ` + -proto "lore/thin_client/v1/thin_client.proto" ` + -d '@' ` + $grpc ` + "lore.thin_client.v1.ThinClientService/RevisionTree" + + +$filePath = "tes.txt" + +$downloadRequest = @{ + identifier = @{ + branchId = $branchB64 + number = "0" + } + path = $filePath + ttlSeconds = "300" + contentType = "application/octet-stream" + inline = $false +} | ConvertTo-Json -Depth 5 -Compress + +$downloadText = ($downloadRequest | & grpcurl @repoArgs ` + -proto "lore/thin_client/v1/thin_client.proto" ` + -d '@' ` + $grpc ` + "lore.thin_client.v1.ThinClientService/RevisionFileDownload") -join "`n" + +$download = $downloadText | ConvertFrom-Json +$download | Format-List + +$url = $http + $download.urlSuffix +$url \ No newline at end of file diff --git a/docs/reference/lore-server-config.md b/docs/reference/lore-server-config.md index b5936f97f..e47f4cf82 100644 --- a/docs/reference/lore-server-config.md +++ b/docs/reference/lore-server-config.md @@ -353,6 +353,13 @@ This repository ships two reference plugin implementations as source. Both requi - **`lore-aws`** backs the immutable store with Amazon S3 and DynamoDB, and the mutable and lock stores with DynamoDB. AWS credentials come from the SDK's default credential chain (environment, profile, instance role), not from the config file. - **`lore-hashicorp`** provides a Consul topology that discovers peers from a service catalog. It only *reads* the catalog — each server must be registered in Consul externally (by your orchestrator or a Consul agent) — returns only passing or healthy nodes, and re-polls on a fixed interval. +The AWS immutable store supports `direct_downloads_enabled` under `[plugins.aws.immutable_store]`. It defaults to `true`, allowing clients to fetch fragment payloads directly from S3 with short-lived presigned URLs. Set it to `false` to make presigning report unsupported so current clients fall back to normal server-proxied reads: + +```toml +[plugins.aws.immutable_store] +direct_downloads_enabled = false +``` + > [!NOTE] > Per-field configuration for these plugins depends on the binary that registers them and is out of scope for this page. A dedicated Server plugins guide — covering the plugin registry, the factory traits, and how to build a server binary with plugins compiled in — is planned. diff --git a/docs/specs/branch-merge-v1.md b/docs/specs/branch-merge-v1.md new file mode 100644 index 000000000..73a7c091d --- /dev/null +++ b/docs/specs/branch-merge-v1.md @@ -0,0 +1,72 @@ +# Authoritative branch merge v1 + +`lore.revision.v1.RevisionService.BranchMerge` merges an immutable source tip +into an immutable target tip and publishes a normal two-parent Lore revision. +The server, rather than a web client, owns base selection, three-way diffing, +conflict validation, tree construction, and the final branch compare-and-swap. + +## Lifecycle + +1. Read both branch tips and retain their signatures. +2. Optionally call `ThinClientService.RevisionDiff` from the source signature to + the target signature. Each three-way `DiffConflict` carries a stable + `conflict_id`. +3. Call `BranchMerge` with both branch ids, both pinned signatures, and any + target/source decisions. +4. If decisions are missing, the server returns `CONFLICTED` plus the missing + ids and paths. It does not construct a revision or move either branch. +5. Submit all decisions. The server re-resolves the base and recomputes the + merge, rejects unknown or duplicate ids, rechecks both tips, constructs the + merge revision, then publishes through the protected BranchPush boundary. +6. A successful response is `MERGED`. An identical retry with the same UUIDv7 + returns the original revision even though the target now points at it. + +If the source contributes no changes relative to the common ancestor, the +response is `ALREADY_UP_TO_DATE` and the target does not move. A stale source or +target signature is `FAILED_PRECONDITION`; disjoint histories are also rejected. + +## Conflict semantics + +A conflict pair is ordered `(source, target)` throughout the library and wire +API. `BRANCH_MERGE_SIDE_TARGET` keeps the target tree as-is. +`BRANCH_MERGE_SIDE_SOURCE` applies the complete source-side change, including +adds, deletes, moves, and directory changes. + +Conflict ids are hashes over a versioned domain, the pinned base/source/target +signatures, and canonical source/target change data. They exclude node ids and +stream-local linked-repository indices. Consequently an id is reusable between +preview and merge only while the same tips and orientation remain pinned. + +`BranchMergeResolution.resolution` is a `oneof`. A future field can carry an +uploaded/custom-content resolution while existing target/source clients remain +wire compatible. + +## Atomicity and retry behavior + +Immutable construction may leave unreachable CAS objects if a race is lost; +that is safe in Lore's content-addressed store. The only visibility point is +the existing protected BranchPush compare-and-swap. Branch protection, pre/post +push hooks, fragment verification, notifications, and revision-list +acceleration therefore remain on the same publication boundary as other +server-authored revisions. + +Only publishable, fully resolved attempts retain an idempotency reservation. +`CONFLICTED`, `ALREADY_UP_TO_DATE`, stale-tip, validation, and construction +failures release it. This permits a client to reuse its request id when adding +decisions after a conflict response; once a revision is published, changing +the request under that id is rejected. + +## Library API + +The transport-independent pieces live in `lore_revision::merge_resolution`: + +- `MergePlan::new(DiffResult)` attaches stable ids and selects executable + source-side changes from Lore's joined presentation diff. +- `MergePlan::resolve` applies partial target/source choices and reports + unresolved and unknown ids. +- `conflict_id` is shared by `RevisionDiff` and `BranchMerge`. +- `state::apply_tree_changes_for_commit` produces a settled target tree and + deletion deltas. +- `commit::construct_merge_revision` freezes, rehashes, weaves history, writes + metadata, and serializes a normal two-parent revision. An unchanged tree is + valid because the second parent records the merged lineage. diff --git a/lore-aws/src/s3.rs b/lore-aws/src/s3.rs index ffcbd696c..99da1aa74 100644 --- a/lore-aws/src/s3.rs +++ b/lore-aws/src/s3.rs @@ -21,6 +21,7 @@ use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsError; use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput; use aws_sdk_s3::operation::put_object::PutObjectError; use aws_sdk_s3::operation::put_object::PutObjectOutput; +use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::primitives::ByteStream; use bytes::Bytes; use lore_telemetry::InstrumentProvider; @@ -192,6 +193,26 @@ impl S3Impl { .map_err(AwsError::sdk_error) } + #[tracing::instrument(name = "S3Impl::presign_get_object", skip_all)] + pub async fn presign_get_object( + &self, + bucket: &str, + key: &str, + expires_in: Duration, + ) -> Result { + let presigning_config = PresigningConfig::expires_in(expires_in) + .map_err(|err| format!("failed to build S3 presigning config: {err}"))?; + + self.client + .get_object() + .bucket(bucket) + .key(key) + .presigned(presigning_config) + .await + .map(|request| request.uri().to_string()) + .map_err(|err| format!("failed to presign S3 get object: {err:?}")) + } + /// Store an object, optionally attaching object metadata carried as `x-amz-meta-*` headers. /// /// Object metadata is part of the object version rather than a separate record, so a reader diff --git a/lore-aws/src/store/immutable_store.rs b/lore-aws/src/store/immutable_store.rs index 16b4951e5..bde3b71b4 100644 --- a/lore-aws/src/store/immutable_store.rs +++ b/lore-aws/src/store/immutable_store.rs @@ -9,6 +9,8 @@ use std::string::ToString; use std::sync::Arc; use std::sync::LazyLock; use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; use async_trait::async_trait; use aws_sdk_dynamodb::operation::put_item::PutItemError; @@ -21,9 +23,11 @@ use aws_smithy_types::error::metadata::ProvideErrorMetadata; use bytes::Bytes; use bytes::BytesMut; use lore_base::error::AddressNotFound; +use lore_base::error::NotSupported; use lore_base::error::SlowDown; use lore_base::types::Address; use lore_base::types::Context; +use lore_base::types::DirectDownload; use lore_base::types::FRAGMENT_SIZE_THRESHOLD; use lore_base::types::Fragment; use lore_base::types::FragmentFlags; @@ -283,6 +287,10 @@ impl DynamoDbImmutableStoreSettings { /// The maximum number of individual exists tasks we'll allow to be submitted across all concurrent /// requests. +fn default_direct_downloads_enabled() -> bool { + true +} + #[derive(Clone, Debug, Deserialize)] #[serde(bound(deserialize = "'de: 'static"))] pub struct AwsImmutableStoreSettings { @@ -290,6 +298,8 @@ pub struct AwsImmutableStoreSettings { pub dynamodb: DynamoDbImmutableStoreSettings, #[serde(default)] pub force_write: bool, + #[serde(default = "default_direct_downloads_enabled")] + pub direct_downloads_enabled: bool, } impl AwsImmutableStoreSettings { @@ -302,8 +312,14 @@ impl AwsImmutableStoreSettings { s3, dynamodb, force_write, + direct_downloads_enabled: default_direct_downloads_enabled(), } } + + pub fn with_direct_downloads_enabled(mut self, enabled: bool) -> Self { + self.direct_downloads_enabled = enabled; + self + } } pub const FRAGMENTS_DYNAMO_PARTITION_KEY_ATTRIBUTE: &str = "hash"; @@ -542,6 +558,7 @@ pub struct AwsImmutableStore { /// deployment that has never written one, and reads accordingly refuse to guess. fragment_metadata_table_name: Option>, force_write: bool, + direct_downloads_enabled: bool, /// How long to wait between removing an association and counting what remains, so a put that /// had already passed its state probe has time to land its own association and be counted. obliteration_drain: Duration, @@ -588,6 +605,7 @@ impl AwsImmutableStore { .as_ref() .map(|name| Arc::from(name.clone())), force_write: settings.force_write, + direct_downloads_enabled: settings.direct_downloads_enabled, obliteration_drain: Duration::from_millis( settings .dynamodb @@ -1751,6 +1769,22 @@ impl AwsImmutableStore { info!("Done obliterating sub-fragments"); Ok(()) } + + async fn presign_payload( + &self, + hash: Hash, + expires_in: Duration, + ) -> Result { + let mut dst = [0u8; 64]; + let key = lore_revision::util::to_hex_str(hash.data(), &mut dst); + self.s3 + .presign_get_object(self.bucket.as_str(), key, expires_in) + .await + .map_err(|err| { + warn!("Failed to presign payload for hash {hash}: {err}"); + StoreError::internal(err) + }) + } } #[async_trait] @@ -1939,6 +1973,72 @@ impl ImmutableStoreTrait for AwsImmutableStore { }) } + #[lore_macro::lore_instrument] + #[tracing::instrument(name = "AwsImmutableStore::presign_downloads", skip(self, addresses))] + async fn presign_downloads( + self: Arc, + partition: Partition, + addresses: &[Address], + match_required: StoreMatch, + expires_in: Duration, + ) -> Result, StoreError> { + if !self.direct_downloads_enabled { + return Err(StoreError::from(NotSupported { + operation: "immutable direct download".to_string(), + })); + } + if match_required != StoreMatch::MatchFull { + return Err(StoreError::internal(format!( + "direct downloads require MatchFull, got {match_required:?}" + ))); + } + + let expires_at_epoch_seconds = SystemTime::now() + .checked_add(expires_in) + .and_then(|expires_at| expires_at.duration_since(UNIX_EPOCH).ok()) + .map(|duration| duration.as_secs()) + .unwrap_or_default(); + + let mut join_set = JoinSet::new(); + for (position, address) in addresses.iter().copied().enumerate() { + let store = self.clone(); + join_set.spawn(async move { + let result = store.clone().get_metadata(partition, address).await?; + if result.match_made != StoreMatch::MatchFull { + return Ok::<_, StoreError>(None); + } + + let mut fragment = result.fragment; + fragment.flags &= !FragmentFlags::PayloadStored.bits(); + fragment.flags |= FragmentFlags::PayloadStoredDurable.bits(); + let url = store.presign_payload(address.hash, expires_in).await?; + Ok(Some(( + position, + DirectDownload { + address, + fragment, + url, + expires_at_epoch_seconds, + }, + ))) + }); + } + + let mut downloads = Vec::new(); + while let Some(result) = join_set.join_next().await { + if let Some(download) = result + .map_err(|err| StoreError::internal_with_context(err, "joining presign task"))?? + { + downloads.push(download); + } + } + downloads.sort_by_key(|(position, _)| *position); + Ok(downloads + .into_iter() + .map(|(_, download)| download) + .collect()) + } + #[lore_macro::lore_instrument] #[tracing::instrument(name= "AwsImmutableStore::put" skip(self, fragment, payload))] async fn put( diff --git a/lore-aws/src/store/test_util.rs b/lore-aws/src/store/test_util.rs index 65a232c08..8e02fdd6d 100644 --- a/lore-aws/src/store/test_util.rs +++ b/lore-aws/src/store/test_util.rs @@ -625,6 +625,7 @@ pub(crate) async fn store_with( s3: S3StoreSettings::new(BUCKET.to_string()), dynamodb: dynamodb_settings, force_write, + direct_downloads_enabled: true, }; let execution = super::setup_execution("test".to_string()); @@ -667,6 +668,7 @@ pub(crate) async fn store_with_separate_metadata_table(fake: &Fake) -> Arc { diff --git a/lore-client/src/cli/commands/branch.rs b/lore-client/src/cli/commands/branch.rs index ac913949d..5dce7acff 100644 --- a/lore-client/src/cli/commands/branch.rs +++ b/lore-client/src/cli/commands/branch.rs @@ -37,6 +37,8 @@ use lore::interface::LoreBranchUnprotectArgs; use lore::interface::LoreEvent; use lore::interface::LoreGlobalArgs; use lore::interface::LoreMetadataType; +use lore::interface::LorePathMergeRule; +use lore::interface::LorePathMergeStrategy; use lore::interface::LoreString; use lore::interface::Partition; use lore::runtime; @@ -141,6 +143,21 @@ impl From<&BranchSourceSpecifier> for LoreString { } } +#[derive(Clone, Default, Args)] +pub struct BranchMergePathStrategyArgs { + /// Path merge rules as strategy:path entries, e.g. keep-target:.git,exclude:generated + #[clap(long, value_name = "strategy:path", value_delimiter = ',')] + merge_strategy: Vec, + + /// Keep the current target branch version for matching paths + #[clap(long, value_name = "path", value_delimiter = ',')] + keep_target: Vec, + + /// Exclude matching source changes from the merge + #[clap(long, value_name = "path", value_delimiter = ',')] + exclude: Vec, +} + #[derive(Args)] #[command(subcommand_negates_reqs = true)] pub struct BranchMergeArgs { @@ -155,6 +172,9 @@ pub struct BranchMergeArgs { /// Change the message for committing when no conflicts arise from the merge #[clap(long, action)] message: Option, + + #[clap(flatten)] + path_strategy: BranchMergePathStrategyArgs, } #[derive(Args)] @@ -182,6 +202,9 @@ pub struct BranchMergeStartArgs { /// Merge only the main repository, skipping all linked repositories #[clap(long, action, conflicts_with = "link")] ignore_links: bool, + + #[clap(flatten)] + path_strategy: BranchMergePathStrategyArgs, } #[derive(Args)] @@ -1069,13 +1092,74 @@ fn handle_branch_merge_into(globals: LoreGlobalArgs, args: &BranchMergeIntoArgs) return runtime().block_on(branch::merge_into(globals, merge_into_args, callback)) as u8; } +fn parse_path_merge_strategy(value: &str) -> Result { + match value.trim().to_ascii_lowercase().replace('_', "-").as_str() { + "merge" => Ok(LorePathMergeStrategy::Merge), + "keep-target" | "keeptarget" | "target" => Ok(LorePathMergeStrategy::KeepTarget), + "exclude" => Ok(LorePathMergeStrategy::Exclude), + _ => Err(format!( + "invalid path merge strategy '{value}'; expected merge, keep-target, or exclude" + )), + } +} + +fn push_path_merge_rule( + rules: &mut Vec, + strategy: LorePathMergeStrategy, + path: &str, +) -> Result<(), String> { + let path = path.trim(); + if path.is_empty() { + return Err("path merge strategy paths cannot be empty".to_string()); + } + rules.push(LorePathMergeRule { + path: LoreString::from(path), + strategy, + }); + Ok(()) +} + +fn parse_path_merge_rules( + args: &BranchMergePathStrategyArgs, +) -> Result, String> { + let mut rules = Vec::new(); + + for value in &args.merge_strategy { + let (strategy, path) = value.split_once(':').ok_or_else(|| { + format!( + "invalid path merge rule '{value}'; expected strategy:path, e.g. keep-target:.git" + ) + })?; + push_path_merge_rule(&mut rules, parse_path_merge_strategy(strategy)?, path)?; + } + + for path in &args.keep_target { + push_path_merge_rule(&mut rules, LorePathMergeStrategy::KeepTarget, path)?; + } + + for path in &args.exclude { + push_path_merge_rule(&mut rules, LorePathMergeStrategy::Exclude, path)?; + } + + Ok(rules) +} + fn handle_branch_merge_start(globals: LoreGlobalArgs, args: &BranchMergeStartArgs) -> u8 { + let path_merge_rules = match parse_path_merge_rules(&args.path_strategy) { + Ok(rules) => rules, + Err(err) => { + println!("error: {err}"); + return 1; + } + }; + let merge_start_args = LoreBranchMergeStartArgs { branch: LoreString::from(&args.branch), message: LoreString::from(&args.message), no_commit: args.no_commit as u8, link: LoreString::from(&args.link), ignore_links: args.ignore_links as u8, + path_merge_rules: LoreArray::from_vec(path_merge_rules), }; let debug = progress_debug(); @@ -1317,6 +1401,7 @@ pub fn handle_branch_merge(globals: LoreGlobalArgs, args: &BranchMergeArgs) -> u dry_run: false, link: None, ignore_links: false, + path_strategy: args.path_strategy.clone(), }; return handle_branch_merge_start(globals, &sub_args); diff --git a/lore-client/src/cli/commands/repository.rs b/lore-client/src/cli/commands/repository.rs index 11632fdb7..7c8a48701 100644 --- a/lore-client/src/cli/commands/repository.rs +++ b/lore-client/src/cli/commands/repository.rs @@ -816,8 +816,8 @@ pub fn handle_repository_info(globals: LoreGlobalArgs, args: &RepositoryInfoArgs anstyle::Reset, data.creator ); - if let Some(created) = DateTime::from_timestamp_millis(data.created as i64) - .map(|time| time.to_rfc2822()) + if let Some(created) = + DateTime::from_timestamp(data.created as i64, 0).map(|time| time.to_rfc2822()) { println!( "{}Created:{} {created}", @@ -884,6 +884,7 @@ pub fn handle_repository_create(globals: LoreGlobalArgs, args: &RepositoryCreate repository_url: url.into(), id: LoreString::from(&args.id), description: LoreString::from(&args.description), + default_branch_name: LoreString::default(), use_shared_store: if args.use_shared_store { LoreSharedStoreMode::Enabled } else { diff --git a/lore-client/src/cli/util.rs b/lore-client/src/cli/util.rs index 692c9571d..968a0ee91 100644 --- a/lore-client/src/cli/util.rs +++ b/lore-client/src/cli/util.rs @@ -40,7 +40,7 @@ pub fn get_repository_path(path: Option) -> LoreString { let current_dir = std::env::current_dir().unwrap_or_default(); let mut current_path = current_dir.as_path(); loop { - if current_path.join(".urc").is_dir() || current_path.join(".lore").is_dir() { + if current_path.join(".urc").is_dir() || current_path.join(".spacesync").is_dir() { break current_path.into(); } if let Some(parent_path) = current_path.parent() { diff --git a/lore-credential/Cargo.toml b/lore-credential/Cargo.toml index b0b61bc9f..886510ce6 100644 --- a/lore-credential/Cargo.toml +++ b/lore-credential/Cargo.toml @@ -24,5 +24,8 @@ url = { workspace = true } [features] trace_log = ["lore-base/trace_log"] +[target.'cfg(target_os = "linux")'.dependencies] +dbus-secret-service = { workspace = true } + [target.'cfg(target_family = "windows")'.dependencies] windows-sys = { workspace = true, features = ["Win32_Storage_FileSystem"] } diff --git a/lore-credential/src/token_store.rs b/lore-credential/src/token_store.rs index fca7c2f1d..2507922ca 100644 --- a/lore-credential/src/token_store.rs +++ b/lore-credential/src/token_store.rs @@ -1026,6 +1026,46 @@ fn secret_absent( } } +#[cfg(target_os = "linux")] +fn dbus_error_means_secret_service_is_unavailable(name: Option<&str>) -> bool { + matches!( + name, + Some( + "org.freedesktop.DBus.Error.ServiceUnknown" + | "org.freedesktop.DBus.Error.NameHasNoOwner" + | "org.freedesktop.DBus.Error.NoServer" + ) + ) +} + +/// Whether Linux has no Secret Service provider to read from. +/// +/// An unavailable provider is different from a provider that is locked or +/// denied access. Only the former may fall back without risking replacement of +/// an encryption key that still exists but is temporarily inaccessible. +#[cfg(target_os = "linux")] +fn secure_store_is_unavailable(err: &keyring::Error) -> bool { + let keyring::Error::PlatformFailure(platform_err) = err else { + return false; + }; + let Some(secret_service_err) = platform_err.downcast_ref::() else { + return false; + }; + + match secret_service_err { + dbus_secret_service::Error::Unavailable => true, + dbus_secret_service::Error::Dbus(err) => { + dbus_error_means_secret_service_is_unavailable(err.name()) + } + _ => false, + } +} + +#[cfg(not(target_os = "linux"))] +fn secure_store_is_unavailable(_err: &keyring::Error) -> bool { + false +} + /// Reads a secret from the OS secure store, if there is a usable one. /// /// `Ok(None)` means the store holds no such secret, or that no secure store is @@ -1053,6 +1093,10 @@ async fn secret_from_secure_store(target: &str) -> Result>, Token lore_debug!("No secret in secure store {target}"); Ok(None) } + Err(err) if secure_store_is_unavailable(&err) => { + lore_debug!("No secure store is available for {target}, using fallback storage"); + Ok(None) + } Err(err) => { lore_warn!("Failed to load secret from secure store {target}: {err}"); Err(TokenStoreError::internal_with_context( @@ -1292,6 +1336,32 @@ token = "tok-b" assert!(encryption_key_from_stored(&[0u8; 4 + 32]).is_none()); } + #[cfg(target_os = "linux")] + #[test] + fn absent_linux_secret_service_uses_fallback_without_hiding_other_failures() { + let unavailable = + keyring::Error::PlatformFailure(Box::new(dbus_secret_service::Error::Unavailable)); + assert!(secure_store_is_unavailable(&unavailable)); + + let locked = keyring::Error::NoStorageAccess(Box::new(dbus_secret_service::Error::Locked)); + assert!(!secure_store_is_unavailable(&locked)); + + let unrelated = keyring::Error::PlatformFailure(Box::new(std::io::Error::other( + "unrelated platform failure", + ))); + assert!(!secure_store_is_unavailable(&unrelated)); + + assert!(dbus_error_means_secret_service_is_unavailable(Some( + "org.freedesktop.DBus.Error.ServiceUnknown" + ))); + assert!(dbus_error_means_secret_service_is_unavailable(Some( + "org.freedesktop.DBus.Error.NameHasNoOwner" + ))); + assert!(!dbus_error_means_secret_service_is_unavailable(Some( + "org.freedesktop.DBus.Error.AccessDenied" + ))); + } + #[test] fn seal_and_open_token_round_trip() { let key = generate_encryption_key().unwrap(); diff --git a/lore-error-set/tests/compile_fail/forward_strict_missing_variant.stderr b/lore-error-set/tests/compile_fail/forward_strict_missing_variant.stderr index f9dbbcf65..1df683074 100644 --- a/lore-error-set/tests/compile_fail/forward_strict_missing_variant.stderr +++ b/lore-error-set/tests/compile_fail/forward_strict_missing_variant.stderr @@ -12,14 +12,14 @@ help: the trait `Has` is not implemented for `NarrowTarget` 49 | #[error_set] | ^^^^^^^^^^^^ = note: strict .forward requires the target to declare every variant of the source; see the `Has<…>` requirement below for the specific missing variant. Add the variant to `NarrowTarget` to preserve it. -help: `NarrowTarget` implements trait `Has` +help: the following other types implement trait `Has` --> tests/compile_fail/forward_strict_missing_variant.rs:49:1 | 49 | #[error_set] | ^^^^^^^^^^^^ | | - | `Has` - | `Has` + | `NarrowTarget` implements `Has` + | `NarrowTarget` implements `Has` = note: required for `NarrowTarget` to implement `HasAll>` = note: 1 redundant requirement hidden = note: required for `NarrowTarget` to implement `HasAll>>` diff --git a/lore-integration-tests/tests/shared_store_test.rs b/lore-integration-tests/tests/shared_store_test.rs index 6892794c6..d4a2f69f5 100644 --- a/lore-integration-tests/tests/shared_store_test.rs +++ b/lore-integration-tests/tests/shared_store_test.rs @@ -99,6 +99,7 @@ mod shared_store_tests { repository_url: REMOTE_URL.into(), description: LoreString::default(), id: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: mode, shared_store_path, }, diff --git a/lore-integration-tests/tests/storage_test.rs b/lore-integration-tests/tests/storage_test.rs index cea1eaaeb..68d32bd08 100644 --- a/lore-integration-tests/tests/storage_test.rs +++ b/lore-integration-tests/tests/storage_test.rs @@ -120,6 +120,7 @@ mod open_tests { repository_url: "lore://localhost/test-storage-open".into(), description: LoreString::default(), id: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }, diff --git a/lore-integration-tests/tests/store_keep_alive_test.rs b/lore-integration-tests/tests/store_keep_alive_test.rs index 37a74e56a..aafc910b7 100644 --- a/lore-integration-tests/tests/store_keep_alive_test.rs +++ b/lore-integration-tests/tests/store_keep_alive_test.rs @@ -82,6 +82,7 @@ mod store_keep_alive_tests { repository_url: "lore://localhost/test-keep-alive".into(), description: LoreString::default(), id: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }, @@ -226,6 +227,7 @@ mod store_keep_alive_tests { no_commit: 0, link: Default::default(), ignore_links: 0, + path_merge_rules: LoreArray::default(), }, None, ) diff --git a/lore-notification/src/lib.rs b/lore-notification/src/lib.rs index 832dfe318..44886756e 100644 --- a/lore-notification/src/lib.rs +++ b/lore-notification/src/lib.rs @@ -7,6 +7,8 @@ mod client; pub fn initialize() { let service = Arc::new(client::NotificationService); lore_revision::notification::register_notification_service("https", service.clone()); + lore_revision::notification::register_notification_service("spacesync", service.clone()); + lore_revision::notification::register_notification_service("spacesyncs", service.clone()); lore_revision::notification::register_notification_service("lore", service.clone()); lore_revision::notification::register_notification_service("lores", service.clone()); // Legacy protocol schemes for backwards compatibility diff --git a/lore-proto/proto/lore/revision/v1/revision.proto b/lore-proto/proto/lore/revision/v1/revision.proto index 689b6e789..bf7e4d92e 100644 --- a/lore-proto/proto/lore/revision/v1/revision.proto +++ b/lore-proto/proto/lore/revision/v1/revision.proto @@ -21,6 +21,10 @@ service RevisionService { // Set the tip of a branch to a specific revision. Supports force and // fast-forward-merge semantics. rpc BranchPush (BranchPushRequest) returns (BranchPushResponse); + // Authoritatively merge a pinned source branch tip into a pinned target + // branch tip, optionally resolving each conflict to the target or source + // side. The target moves only after the entire merge succeeds. + rpc BranchMerge (BranchMergeRequest) returns (BranchMergeResponse); // Cheap hash-only read of a branch's current metadata pointer. rpc BranchMetadataGet(BranchMetadataGetRequest) returns (BranchMetadataGetResponse); // Compare-and-swap update of a branch's metadata pointer. CAS miss is @@ -28,6 +32,10 @@ service RevisionService { rpc BranchMetadataSet(BranchMetadataSetRequest) returns (BranchMetadataSetResponse); // Page through a branch's revision history. Server picks page size. rpc RevisionList (RevisionListRequest) returns (RevisionListResponse); + // Atomically apply an address-backed path changeset to an observed branch + // tip. Immutable construction happens first; the same protected BranchPush + // CAS path used by BranchPush is the visibility point. + rpc RevisionCreate (RevisionCreateRequest) returns (RevisionCreateResponse); } // Request to create a new branch. The caller must supply a fully-resolved @@ -148,6 +156,86 @@ message BranchPushResponse { optional string message = 4; } +// Which complete side of a merge conflict should be retained. Values are +// intentionally prefixed because protobuf enum values share package scope. +enum BranchMergeSide { + BRANCH_MERGE_SIDE_UNSPECIFIED = 0; + // Keep the target branch's version of the conflicted path. + BRANCH_MERGE_SIDE_TARGET = 1; + // Apply the source branch's version of the conflicted path. + BRANCH_MERGE_SIDE_SOURCE = 2; +} + +// Terminal result of a BranchMerge attempt. +enum BranchMergeOutcome { + BRANCH_MERGE_OUTCOME_UNSPECIFIED = 0; + // A two-parent revision was published to the target branch. + BRANCH_MERGE_OUTCOME_MERGED = 1; + // The source introduced no changes relative to the resolved base. + BRANCH_MERGE_OUTCOME_ALREADY_UP_TO_DATE = 2; + // One or more conflicts still need an explicit resolution. + BRANCH_MERGE_OUTCOME_CONFLICTED = 3; +} + +// Request for an atomic, server-authored branch merge. Both tips are required +// optimistic-concurrency guards; stale tips fail with FAILED_PRECONDITION. +message BranchMergeRequest { + // Required caller-generated UUIDv7. A successfully published request is + // idempotent; reusing its id with different content is rejected. + bytes request_id = 1; + // Branch that will move when the merge succeeds. + bytes branch_id_target = 2; + // Exact target tip observed by the caller. + bytes revision_signature_target = 3; + // Branch whose changes will be merged into the target. + bytes branch_id_source = 4; + // Exact source tip observed by the caller. + bytes revision_signature_source = 5; + // Commit message for the server-authored merge revision. + string commit_message = 6; + // Per-conflict decisions keyed by RevisionDiff conflict_id. Missing + // decisions are returned in-band and cause no branch mutation. + repeated BranchMergeResolution resolutions = 7; +} + +// Resolution for one conflict. The oneof is an extension point for a future +// custom/uploaded-content resolution without changing existing clients. +message BranchMergeResolution { + // Stable conflict id emitted by RevisionDiff for the same pinned tips. + bytes conflict_id = 1; + // Requested resolution strategy. + oneof resolution { + // Keep the complete target or source side. + BranchMergeSide side = 2; + } +} + +// Minimal conflict identity returned by BranchMerge. Clients can obtain full +// content addresses and presentation diffs from RevisionDiff. +message BranchMergeConflict { + // Stable id accepted by BranchMergeResolution. + bytes conflict_id = 1; + // Path changed on the source side. + string path_source = 2; + // Path changed on the target side. + string path_target = 3; +} + +// Result of an authoritative merge attempt. CONFLICTED and +// ALREADY_UP_TO_DATE are successful, non-mutating outcomes. +message BranchMergeResponse { + // Machine-readable outcome. + BranchMergeOutcome outcome = 1; + // Published target tip for MERGED, otherwise the pinned target tip. + bytes revision_signature = 2; + // Revision number corresponding to revision_signature. + uint64 revision_number = 3; + // Common ancestor selected by the server. + bytes revision_signature_base = 4; + // Conflicts lacking a supplied decision. Empty for terminal outcomes. + repeated BranchMergeConflict unresolved_conflicts = 5; +} + // Request for a cheap hash-only read of a branch's metadata pointer. message BranchMetadataGetRequest { // Branch id. @@ -218,6 +306,69 @@ message RevisionListResponse { optional bytes signature_backward = 3; } +// Request to atomically apply a path changeset to a pinned branch tip. +message RevisionCreateRequest { + // Required caller-generated UUIDv7. Reusing it with identical request bytes + // returns the original result; reuse with different bytes is rejected. + bytes request_id = 1; + bytes branch_id = 2; + // Exact branch tip observed by the caller. Zero is valid only for an + // initial/root revision. Stale bases fail with FAILED_PRECONDITION. + bytes revision_signature_base = 3; + string commit_message = 4; + repeated RevisionCreateMetadataEntry metadata = 5; + repeated RevisionCreateOperation operations = 6; +} + +// Caller-supplied metadata entry for a new revision. +message RevisionCreateMetadataEntry { + string key = 1; + bytes value = 2; + // lore metadata type encoding (Address=1, Boolean=2, Context=3, Hash=4, + // Numeric=5, String=6, Binary=255). + uint32 format = 3; +} + +// One path-level edit in a RevisionCreate changeset. +message RevisionCreateOperation { + oneof op { + RevisionCreatePutFile put_file = 1; + RevisionCreateDirectory create_directory = 2; + RevisionCreateDeletePath delete_path = 3; + RevisionCreateMovePath move_path = 4; + } +} + +// Add or replace a file with content previously uploaded to Lore. +message RevisionCreatePutFile { + string path = 1; + uint32 mode = 2; + // UploadContent result. The server verifies the exact repository/context + // association and derives logical size from immutable fragment metadata. + lore.model.v1.Address address = 3; +} + +// Create an empty directory at a path. +message RevisionCreateDirectory { + string path = 1; + uint32 mode = 2; +} + +// Delete a file or directory tree at a path. +message RevisionCreateDeletePath { string path = 1; } + +// Move or rename a path while preserving its file identity. +message RevisionCreateMovePath { + string source = 1; + string destination = 2; +} + +// Published revision identity returned by RevisionCreate. +message RevisionCreateResponse { + bytes revision_signature = 1; + uint64 revision_number = 2; +} + // A service to receive requests forwarded on from another Lore Sever, // to perform an RPC on behalf of a client service ForwardedRevisionService { diff --git a/lore-proto/proto/lore/storage/v1/storage.proto b/lore-proto/proto/lore/storage/v1/storage.proto index d53bb85f7..235b18c81 100644 --- a/lore-proto/proto/lore/storage/v1/storage.proto +++ b/lore-proto/proto/lore/storage/v1/storage.proto @@ -34,8 +34,14 @@ service StorageService { // write side of GetResolved, and the only thing that makes a key resolvable. Shares // GetResolved's correlation and in-band status shape. rpc PutResolved(stream PutResolvedRequest) returns (stream PutResolvedResponse); + // UploadContent accepts one header followed by raw byte chunks. Lore owns + // chunking, hashing, compression, fragment-list construction, and durable + // storage. Cancellation can leave unreachable content-addressed fragments; + // it never publishes a revision or branch. + rpc UploadContent(stream UploadContentRequest) returns (UploadContentResponse); rpc Put(stream PutRequest) returns (stream PutResponse); rpc Query(QueryRequest) returns (QueryResponse); + rpc PresignDownload(PresignDownloadRequest) returns (PresignDownloadResponse); rpc Verify(VerifyRequest) returns (VerifyResponse); rpc Copy(stream CopyRequest) returns (stream CopyResponse); rpc MutableLoad(MutableLoadRequest) returns (MutableLoadResponse); @@ -120,6 +126,32 @@ message PutResolvedResponse { lore.model.v1.ItemStatus status = 2; } +message UploadContentRequest { + oneof part { + // Required exactly once as the first stream message. + UploadContentHeader header = 1; + // Raw file bytes. Chunks may be any non-empty transport-friendly size; + // callers must not pre-fragment them according to Lore internals. + bytes chunk = 2; + } +} + +message UploadContentHeader { + // UUIDv7 file identity. Use the existing address context when modifying a + // file. Empty asks Lore to mint a UUIDv7 for a new file. + bytes file_id = 1; + // Optional integrity assertion checked against the actual bytes read. + optional uint64 expected_size = 2; + // Required caller-generated UUIDv7 for retry/correlation diagnostics. + bytes request_id = 3; +} + +message UploadContentResponse { + lore.model.v1.Address address = 1; + // Actual logical byte count observed by Lore. + uint64 size = 2; +} + message PutRequest { lore.model.v1.Address address = 1; lore.model.v1.Fragment fragment = 2; @@ -142,6 +174,22 @@ message QueryResponse { repeated lore.model.v1.FragmentState results = 1; } +message PresignDownloadRequest { + repeated lore.model.v1.Address addresses = 1; + uint64 expires_in_seconds = 2; +} + +message PresignedDownload { + lore.model.v1.Address address = 1; + lore.model.v1.Fragment fragment = 2; + string url = 3; + uint64 expires_at_epoch_seconds = 4; +} + +message PresignDownloadResponse { + repeated PresignedDownload downloads = 1; +} + message VerifyRequest { lore.model.v1.Address address = 1; bool heal = 2; diff --git a/lore-proto/proto/lore/thin_client/v1/model.proto b/lore-proto/proto/lore/thin_client/v1/model.proto index bb7a9045f..ead0d3a01 100644 --- a/lore-proto/proto/lore/thin_client/v1/model.proto +++ b/lore-proto/proto/lore/thin_client/v1/model.proto @@ -99,6 +99,10 @@ message DiffConflict { DiffChange change_from = 1; // The change the other side made relative to the common ancestor. DiffChange change_to = 2; + // Stable identifier for this conflict when the resolved base and the + // request's from/to revisions remain pinned. Pass this value to + // lore.revision.v1.BranchMerge when selecting a resolution. + bytes conflict_id = 3; } // Per-stream partition table entry. Announces a linked partition and @@ -125,6 +129,10 @@ message TreeNode { NodeType node_type = 2; // Content address for FILE / LINK entries; unused for DIRECTORY. lore.model.v1.Address address = 3; + // Signature of the revision in the requested repository that most recently + // changed this entry. Empty for directories. Descendants of a linked + // repository inherit the revision that last changed the link pin. + bytes last_changed_revision_signature = 7; // Original size in bytes. For DIRECTORY entries, this is the cumulative size of its descendant files. uint64 size = 4; // File mode for this entry. For possible flags and values, see enum FileMode. diff --git a/lore-proto/proto/lore/thin_client/v1/thin_client.proto b/lore-proto/proto/lore/thin_client/v1/thin_client.proto index 417b6521e..c0e13e5c8 100644 --- a/lore-proto/proto/lore/thin_client/v1/thin_client.proto +++ b/lore-proto/proto/lore/thin_client/v1/thin_client.proto @@ -5,8 +5,8 @@ package lore.thin_client.v1; import "lore/model/v1/model.proto"; import "lore/thin_client/v1/model.proto"; -// Thin-client presentation helpers — describe / tree / diff RPCs that are -// convenience helpers for clients that lack local cache or compute ( web UIs). +// Thin-client presentation helpers — describe / tree / diff / download RPCs +// for clients that lack local cache or compute (web UIs). // Sits alongside the baseline lore.revision.v1.RevisionService and shares the // same backing revision-graph state. service ThinClientService { @@ -21,6 +21,10 @@ service ThinClientService { // Stream the file/directory tree at a revision as header + TreeNode // items. rpc RevisionTree(RevisionTreeRequest) returns (stream RevisionTreeResponse); + // Resolve one file at an immutable revision and create a short-lived URL + // that streams the reconstructed logical bytes through the Lore HTTP + // server. The returned URL never exposes storage fragment URLs. + rpc RevisionFileDownload(RevisionFileDownloadRequest) returns (RevisionFileDownloadResponse); } // Request to describe a single revision by identifier or signature. @@ -42,8 +46,20 @@ message RevisionInfoResponse { Revision revision = 1; } -// Request to diff two revisions. Server picks the diff mode (2-way vs -// 3-way) from revision metadata; callers do not declare it. +// Controls whether RevisionDiff compares two snapshots directly or computes +// their merge-relative changes through a common ancestor. +enum RevisionDiffMode { + // Preserve the server's historical mode selection: pairwise for revisions + // on the same branch (or at a branch point), merge otherwise. + REVISION_DIFF_MODE_AUTO = 0; + // Always compare the "from" snapshot directly with the "to" snapshot. + REVISION_DIFF_MODE_PAIRWISE = 1; + // Always resolve a common ancestor and compute a three-way merge diff. + REVISION_DIFF_MODE_MERGE = 2; +} + +// Request to diff two revisions. Callers may explicitly select pairwise or +// merge semantics; AUTO preserves the server's metadata-driven behavior. message RevisionDiffRequest { // "From" side specifier. oneof query_from { @@ -59,16 +75,18 @@ message RevisionDiffRequest { // Signature-based "to". bytes signature_to = 4; } - // When true and the server picks 3-way mode, auto-resolved - // DiffChange entries are flagged via `automerged = true`. Silently - // ignored in 2-way mode. + // When true and the effective mode is MERGE, auto-resolved DiffChange + // entries are flagged via `automerged = true`. Silently ignored in + // PAIRWISE mode. bool autoresolve = 5; + // Requested comparison semantics. AUTO preserves the legacy behavior. + RevisionDiffMode mode = 6; } // Header for a RevisionDiff stream. Echoes the resolved revisions so // callers querying by signature learn the (branch, number) and vice -// versa. Optional `_base` fields are populated only when the server -// selected a 3-way diff. +// versa. `mode` reports the effective mode after AUTO resolution. Optional +// `_base` fields are populated for non-identical MERGE comparisons. message RevisionDiffHeader { // Resolved (branch, number) of the "from" side. lore.model.v1.RevisionIdentifier identifier_from = 1; @@ -84,6 +102,8 @@ message RevisionDiffHeader { // Resolved signature of the common ancestor base. Set only in // 3-way mode. optional bytes signature_base = 6; + // Effective comparison mode after resolving AUTO. + RevisionDiffMode mode = 7; } // Server-streamed response for RevisionDiff. First message carries @@ -147,3 +167,48 @@ message RevisionTreeResponse { TreeNode node = 2; } } + +// Request a browser-ready download URL for one exact file path. +message RevisionFileDownloadRequest { + // Revision specifier. + oneof query { + // Identifier-based lookup; `number == 0` resolves to the tip of + // the branch and the response pins the concrete signature. + lore.model.v1.RevisionIdentifier identifier = 1; + // Signature-based lookup. + bytes signature = 2; + } + // Repository-relative path to a file. Directories and links are rejected. + string path = 3; + // Requested URL lifetime. Zero selects the server default; non-zero values + // are clamped to the configured minimum and maximum. + uint64 ttl_seconds = 4; + // Response Content-Type. Empty defaults to application/octet-stream. + string content_type = 5; + // When true, Content-Disposition is `inline`; otherwise it is `attachment`. + // In both cases Lore derives and safely encodes the filename from `path`. + bool inline = 6; +} + +// A pinned logical-file download. `url_suffix` is relative to the externally +// configured Lore HTTP origin because the gRPC server does not know ingress +// or reverse-proxy host names. +message RevisionFileDownloadResponse { + // Concrete revision selected by the request. + RevisionTreeHeader revision = 1; + // Repository that owns the resolved file. This can differ from the request + // repository when the path crosses an authorized repository link. + bytes repository_resolved = 2; + // Immutable logical-content address stored on the file node. + lore.model.v1.Address address = 3; + // Logical file length after defragmentation and decompression. + uint64 size = 4; + // Short-lived Lore HTTP path, including its signed query token. + string url_suffix = 5; + // Unix epoch seconds at which the URL expires. + uint64 expires_at_epoch_seconds = 6; + // Final path component used for Content-Disposition. + string file_name = 7; + // File mode stored on the revision node. + uint32 mode = 8; +} diff --git a/lore-proto/src/grpc/lore.revision.v1.rs b/lore-proto/src/grpc/lore.revision.v1.rs index 9eafeb6fb..0fa9bc486 100644 --- a/lore-proto/src/grpc/lore.revision.v1.rs +++ b/lore-proto/src/grpc/lore.revision.v1.rs @@ -245,6 +245,129 @@ impl ::prost::Name for BranchPushResponse { "/lore.revision.v1.BranchPushResponse".into() } } +/// Request for an atomic, server-authored branch merge. Both tips are required +/// optimistic-concurrency guards; stale tips fail with FAILED_PRECONDITION. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BranchMergeRequest { + /// Required caller-generated UUIDv7. A successfully published request is + /// idempotent; reusing its id with different content is rejected. + #[prost(bytes = "bytes", tag = "1")] + pub request_id: ::prost::bytes::Bytes, + /// Branch that will move when the merge succeeds. + #[prost(bytes = "bytes", tag = "2")] + pub branch_id_target: ::prost::bytes::Bytes, + /// Exact target tip observed by the caller. + #[prost(bytes = "bytes", tag = "3")] + pub revision_signature_target: ::prost::bytes::Bytes, + /// Branch whose changes will be merged into the target. + #[prost(bytes = "bytes", tag = "4")] + pub branch_id_source: ::prost::bytes::Bytes, + /// Exact source tip observed by the caller. + #[prost(bytes = "bytes", tag = "5")] + pub revision_signature_source: ::prost::bytes::Bytes, + /// Commit message for the server-authored merge revision. + #[prost(string, tag = "6")] + pub commit_message: ::prost::alloc::string::String, + /// Per-conflict decisions keyed by RevisionDiff conflict_id. Missing + /// decisions are returned in-band and cause no branch mutation. + #[prost(message, repeated, tag = "7")] + pub resolutions: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for BranchMergeRequest { + const NAME: &'static str = "BranchMergeRequest"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.BranchMergeRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.BranchMergeRequest".into() + } +} +/// Resolution for one conflict. The oneof is an extension point for a future +/// custom/uploaded-content resolution without changing existing clients. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BranchMergeResolution { + /// Stable conflict id emitted by RevisionDiff for the same pinned tips. + #[prost(bytes = "bytes", tag = "1")] + pub conflict_id: ::prost::bytes::Bytes, + /// Requested resolution strategy. + #[prost(oneof = "branch_merge_resolution::Resolution", tags = "2")] + pub resolution: ::core::option::Option, +} +/// Nested message and enum types in `BranchMergeResolution`. +pub mod branch_merge_resolution { + /// Requested resolution strategy. + #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Resolution { + /// Keep the complete target or source side. + #[prost(enumeration = "super::BranchMergeSide", tag = "2")] + Side(i32), + } +} +impl ::prost::Name for BranchMergeResolution { + const NAME: &'static str = "BranchMergeResolution"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.BranchMergeResolution".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.BranchMergeResolution".into() + } +} +/// Minimal conflict identity returned by BranchMerge. Clients can obtain full +/// content addresses and presentation diffs from RevisionDiff. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BranchMergeConflict { + /// Stable id accepted by BranchMergeResolution. + #[prost(bytes = "bytes", tag = "1")] + pub conflict_id: ::prost::bytes::Bytes, + /// Path changed on the source side. + #[prost(string, tag = "2")] + pub path_source: ::prost::alloc::string::String, + /// Path changed on the target side. + #[prost(string, tag = "3")] + pub path_target: ::prost::alloc::string::String, +} +impl ::prost::Name for BranchMergeConflict { + const NAME: &'static str = "BranchMergeConflict"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.BranchMergeConflict".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.BranchMergeConflict".into() + } +} +/// Result of an authoritative merge attempt. CONFLICTED and +/// ALREADY_UP_TO_DATE are successful, non-mutating outcomes. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BranchMergeResponse { + /// Machine-readable outcome. + #[prost(enumeration = "BranchMergeOutcome", tag = "1")] + pub outcome: i32, + /// Published target tip for MERGED, otherwise the pinned target tip. + #[prost(bytes = "bytes", tag = "2")] + pub revision_signature: ::prost::bytes::Bytes, + /// Revision number corresponding to revision_signature. + #[prost(uint64, tag = "3")] + pub revision_number: u64, + /// Common ancestor selected by the server. + #[prost(bytes = "bytes", tag = "4")] + pub revision_signature_base: ::prost::bytes::Bytes, + /// Conflicts lacking a supplied decision. Empty for terminal outcomes. + #[prost(message, repeated, tag = "5")] + pub unresolved_conflicts: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for BranchMergeResponse { + const NAME: &'static str = "BranchMergeResponse"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.BranchMergeResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.BranchMergeResponse".into() + } +} /// Request for a cheap hash-only read of a branch's metadata pointer. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BranchMetadataGetRequest { @@ -393,6 +516,249 @@ impl ::prost::Name for RevisionListResponse { "/lore.revision.v1.RevisionListResponse".into() } } +/// Request to atomically apply a path changeset to a pinned branch tip. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RevisionCreateRequest { + /// Required caller-generated UUIDv7. Reusing it with identical request bytes + /// returns the original result; reuse with different bytes is rejected. + #[prost(bytes = "bytes", tag = "1")] + pub request_id: ::prost::bytes::Bytes, + #[prost(bytes = "bytes", tag = "2")] + pub branch_id: ::prost::bytes::Bytes, + /// Exact branch tip observed by the caller. Zero is valid only for an + /// initial/root revision. Stale bases fail with FAILED_PRECONDITION. + #[prost(bytes = "bytes", tag = "3")] + pub revision_signature_base: ::prost::bytes::Bytes, + #[prost(string, tag = "4")] + pub commit_message: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "5")] + pub metadata: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "6")] + pub operations: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for RevisionCreateRequest { + const NAME: &'static str = "RevisionCreateRequest"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreateRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreateRequest".into() + } +} +/// Caller-supplied metadata entry for a new revision. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionCreateMetadataEntry { + #[prost(string, tag = "1")] + pub key: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "2")] + pub value: ::prost::bytes::Bytes, + /// lore metadata type encoding (Address=1, Boolean=2, Context=3, Hash=4, + /// Numeric=5, String=6, Binary=255). + #[prost(uint32, tag = "3")] + pub format: u32, +} +impl ::prost::Name for RevisionCreateMetadataEntry { + const NAME: &'static str = "RevisionCreateMetadataEntry"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreateMetadataEntry".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreateMetadataEntry".into() + } +} +/// One path-level edit in a RevisionCreate changeset. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionCreateOperation { + #[prost(oneof = "revision_create_operation::Op", tags = "1, 2, 3, 4")] + pub op: ::core::option::Option, +} +/// Nested message and enum types in `RevisionCreateOperation`. +pub mod revision_create_operation { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Op { + #[prost(message, tag = "1")] + PutFile(super::RevisionCreatePutFile), + #[prost(message, tag = "2")] + CreateDirectory(super::RevisionCreateDirectory), + #[prost(message, tag = "3")] + DeletePath(super::RevisionCreateDeletePath), + #[prost(message, tag = "4")] + MovePath(super::RevisionCreateMovePath), + } +} +impl ::prost::Name for RevisionCreateOperation { + const NAME: &'static str = "RevisionCreateOperation"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreateOperation".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreateOperation".into() + } +} +/// Add or replace a file with content previously uploaded to Lore. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionCreatePutFile { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + #[prost(uint32, tag = "2")] + pub mode: u32, + /// UploadContent result. The server verifies the exact repository/context + /// association and derives logical size from immutable fragment metadata. + #[prost(message, optional, tag = "3")] + pub address: ::core::option::Option, +} +impl ::prost::Name for RevisionCreatePutFile { + const NAME: &'static str = "RevisionCreatePutFile"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreatePutFile".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreatePutFile".into() + } +} +/// Create an empty directory at a path. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionCreateDirectory { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, + #[prost(uint32, tag = "2")] + pub mode: u32, +} +impl ::prost::Name for RevisionCreateDirectory { + const NAME: &'static str = "RevisionCreateDirectory"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreateDirectory".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreateDirectory".into() + } +} +/// Delete a file or directory tree at a path. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionCreateDeletePath { + #[prost(string, tag = "1")] + pub path: ::prost::alloc::string::String, +} +impl ::prost::Name for RevisionCreateDeletePath { + const NAME: &'static str = "RevisionCreateDeletePath"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreateDeletePath".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreateDeletePath".into() + } +} +/// Move or rename a path while preserving its file identity. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionCreateMovePath { + #[prost(string, tag = "1")] + pub source: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub destination: ::prost::alloc::string::String, +} +impl ::prost::Name for RevisionCreateMovePath { + const NAME: &'static str = "RevisionCreateMovePath"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreateMovePath".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreateMovePath".into() + } +} +/// Published revision identity returned by RevisionCreate. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionCreateResponse { + #[prost(bytes = "bytes", tag = "1")] + pub revision_signature: ::prost::bytes::Bytes, + #[prost(uint64, tag = "2")] + pub revision_number: u64, +} +impl ::prost::Name for RevisionCreateResponse { + const NAME: &'static str = "RevisionCreateResponse"; + const PACKAGE: &'static str = "lore.revision.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.revision.v1.RevisionCreateResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.revision.v1.RevisionCreateResponse".into() + } +} +/// Which complete side of a merge conflict should be retained. Values are +/// intentionally prefixed because protobuf enum values share package scope. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum BranchMergeSide { + Unspecified = 0, + /// Keep the target branch's version of the conflicted path. + Target = 1, + /// Apply the source branch's version of the conflicted path. + Source = 2, +} +impl BranchMergeSide { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "BRANCH_MERGE_SIDE_UNSPECIFIED", + Self::Target => "BRANCH_MERGE_SIDE_TARGET", + Self::Source => "BRANCH_MERGE_SIDE_SOURCE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "BRANCH_MERGE_SIDE_UNSPECIFIED" => Some(Self::Unspecified), + "BRANCH_MERGE_SIDE_TARGET" => Some(Self::Target), + "BRANCH_MERGE_SIDE_SOURCE" => Some(Self::Source), + _ => None, + } + } +} +/// Terminal result of a BranchMerge attempt. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum BranchMergeOutcome { + Unspecified = 0, + /// A two-parent revision was published to the target branch. + Merged = 1, + /// The source introduced no changes relative to the resolved base. + AlreadyUpToDate = 2, + /// One or more conflicts still need an explicit resolution. + Conflicted = 3, +} +impl BranchMergeOutcome { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "BRANCH_MERGE_OUTCOME_UNSPECIFIED", + Self::Merged => "BRANCH_MERGE_OUTCOME_MERGED", + Self::AlreadyUpToDate => "BRANCH_MERGE_OUTCOME_ALREADY_UP_TO_DATE", + Self::Conflicted => "BRANCH_MERGE_OUTCOME_CONFLICTED", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "BRANCH_MERGE_OUTCOME_UNSPECIFIED" => Some(Self::Unspecified), + "BRANCH_MERGE_OUTCOME_MERGED" => Some(Self::Merged), + "BRANCH_MERGE_OUTCOME_ALREADY_UP_TO_DATE" => Some(Self::AlreadyUpToDate), + "BRANCH_MERGE_OUTCOME_CONFLICTED" => Some(Self::Conflicted), + _ => None, + } + } +} /// Generated client implementations. pub mod revision_service_client { #![allow( @@ -625,6 +991,35 @@ pub mod revision_service_client { ); self.inner.unary(req, path, codec).await } + /// Authoritatively merge a pinned source branch tip into a pinned target + /// branch tip, optionally resolving each conflict to the target or source + /// side. The target moves only after the entire merge succeeds. + pub async fn branch_merge( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/lore.revision.v1.RevisionService/BranchMerge", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("lore.revision.v1.RevisionService", "BranchMerge"), + ); + self.inner.unary(req, path, codec).await + } /// Cheap hash-only read of a branch's current metadata pointer. pub async fn branch_metadata_get( &mut self, @@ -713,6 +1108,35 @@ pub mod revision_service_client { ); self.inner.unary(req, path, codec).await } + /// Atomically apply an address-backed path changeset to an observed branch + /// tip. Immutable construction happens first; the same protected BranchPush + /// CAS path used by BranchPush is the visibility point. + pub async fn revision_create( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/lore.revision.v1.RevisionService/RevisionCreate", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("lore.revision.v1.RevisionService", "RevisionCreate"), + ); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -773,6 +1197,16 @@ pub mod revision_service_server { tonic::Response, tonic::Status, >; + /// Authoritatively merge a pinned source branch tip into a pinned target + /// branch tip, optionally resolving each conflict to the target or source + /// side. The target moves only after the entire merge succeeds. + async fn branch_merge( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Cheap hash-only read of a branch's current metadata pointer. async fn branch_metadata_get( &self, @@ -798,6 +1232,16 @@ pub mod revision_service_server { tonic::Response, tonic::Status, >; + /// Atomically apply an address-backed path changeset to an observed branch + /// tip. Immutable construction happens first; the same protected BranchPush + /// CAS path used by BranchPush is the visibility point. + async fn revision_create( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } /// Baseline revision-graph service. Carries the minimal, stable set of /// graph primitives — branch lifecycle, metadata CAS, revision listing. @@ -1105,6 +1549,51 @@ pub mod revision_service_server { }; Box::pin(fut) } + "/lore.revision.v1.RevisionService/BranchMerge" => { + #[allow(non_camel_case_types)] + struct BranchMergeSvc(pub Arc); + impl< + T: RevisionService, + > tonic::server::UnaryService + for BranchMergeSvc { + type Response = super::BranchMergeResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::branch_merge(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = BranchMergeSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/lore.revision.v1.RevisionService/BranchMetadataGet" => { #[allow(non_camel_case_types)] struct BranchMetadataGetSvc(pub Arc); @@ -1242,6 +1731,52 @@ pub mod revision_service_server { }; Box::pin(fut) } + "/lore.revision.v1.RevisionService/RevisionCreate" => { + #[allow(non_camel_case_types)] + struct RevisionCreateSvc(pub Arc); + impl< + T: RevisionService, + > tonic::server::UnaryService + for RevisionCreateSvc { + type Response = super::RevisionCreateResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::revision_create(&inner, request) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RevisionCreateSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => { Box::pin(async move { let mut response = http::Response::new( diff --git a/lore-proto/src/grpc/lore.storage.v1.rs b/lore-proto/src/grpc/lore.storage.v1.rs index 8fddcefdd..01341a995 100644 --- a/lore-proto/src/grpc/lore.storage.v1.rs +++ b/lore-proto/src/grpc/lore.storage.v1.rs @@ -146,6 +146,75 @@ impl ::prost::Name for PutResolvedResponse { } } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct UploadContentRequest { + #[prost(oneof = "upload_content_request::Part", tags = "1, 2")] + pub part: ::core::option::Option, +} +/// Nested message and enum types in `UploadContentRequest`. +pub mod upload_content_request { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Part { + /// Required exactly once as the first stream message. + #[prost(message, tag = "1")] + Header(super::UploadContentHeader), + /// Raw file bytes. Chunks may be any non-empty transport-friendly size; + /// callers must not pre-fragment them according to Lore internals. + #[prost(bytes, tag = "2")] + Chunk(::prost::bytes::Bytes), + } +} +impl ::prost::Name for UploadContentRequest { + const NAME: &'static str = "UploadContentRequest"; + const PACKAGE: &'static str = "lore.storage.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.storage.v1.UploadContentRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.storage.v1.UploadContentRequest".into() + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct UploadContentHeader { + /// UUIDv7 file identity. Use the existing address context when modifying a + /// file. Empty asks Lore to mint a UUIDv7 for a new file. + #[prost(bytes = "bytes", tag = "1")] + pub file_id: ::prost::bytes::Bytes, + /// Optional integrity assertion checked against the actual bytes read. + #[prost(uint64, optional, tag = "2")] + pub expected_size: ::core::option::Option, + /// Required caller-generated UUIDv7 for retry/correlation diagnostics. + #[prost(bytes = "bytes", tag = "3")] + pub request_id: ::prost::bytes::Bytes, +} +impl ::prost::Name for UploadContentHeader { + const NAME: &'static str = "UploadContentHeader"; + const PACKAGE: &'static str = "lore.storage.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.storage.v1.UploadContentHeader".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.storage.v1.UploadContentHeader".into() + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct UploadContentResponse { + #[prost(message, optional, tag = "1")] + pub address: ::core::option::Option, + /// Actual logical byte count observed by Lore. + #[prost(uint64, tag = "2")] + pub size: u64, +} +impl ::prost::Name for UploadContentResponse { + const NAME: &'static str = "UploadContentResponse"; + const PACKAGE: &'static str = "lore.storage.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.storage.v1.UploadContentResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.storage.v1.UploadContentResponse".into() + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PutRequest { #[prost(message, optional, tag = "1")] pub address: ::core::option::Option, @@ -214,6 +283,59 @@ impl ::prost::Name for QueryResponse { "/lore.storage.v1.QueryResponse".into() } } +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PresignDownloadRequest { + #[prost(message, repeated, tag = "1")] + pub addresses: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "2")] + pub expires_in_seconds: u64, +} +impl ::prost::Name for PresignDownloadRequest { + const NAME: &'static str = "PresignDownloadRequest"; + const PACKAGE: &'static str = "lore.storage.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.storage.v1.PresignDownloadRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.storage.v1.PresignDownloadRequest".into() + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PresignedDownload { + #[prost(message, optional, tag = "1")] + pub address: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub fragment: ::core::option::Option, + #[prost(string, tag = "3")] + pub url: ::prost::alloc::string::String, + #[prost(uint64, tag = "4")] + pub expires_at_epoch_seconds: u64, +} +impl ::prost::Name for PresignedDownload { + const NAME: &'static str = "PresignedDownload"; + const PACKAGE: &'static str = "lore.storage.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.storage.v1.PresignedDownload".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.storage.v1.PresignedDownload".into() + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PresignDownloadResponse { + #[prost(message, repeated, tag = "1")] + pub downloads: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for PresignDownloadResponse { + const NAME: &'static str = "PresignDownloadResponse"; + const PACKAGE: &'static str = "lore.storage.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.storage.v1.PresignDownloadResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.storage.v1.PresignDownloadResponse".into() + } +} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct VerifyRequest { #[prost(message, optional, tag = "1")] @@ -647,6 +769,38 @@ pub mod storage_service_client { ); self.inner.streaming(req, path, codec).await } + /// UploadContent accepts one header followed by raw byte chunks. Lore owns + /// chunking, hashing, compression, fragment-list construction, and durable + /// storage. Cancellation can leave unreachable content-addressed fragments; + /// it never publishes a revision or branch. + pub async fn upload_content( + &mut self, + request: impl tonic::IntoStreamingRequest< + Message = super::UploadContentRequest, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/lore.storage.v1.StorageService/UploadContent", + ); + let mut req = request.into_streaming_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("lore.storage.v1.StorageService", "UploadContent"), + ); + self.inner.client_streaming(req, path, codec).await + } pub async fn put( &mut self, request: impl tonic::IntoStreamingRequest, @@ -692,6 +846,32 @@ pub mod storage_service_client { .insert(GrpcMethod::new("lore.storage.v1.StorageService", "Query")); self.inner.unary(req, path, codec).await } + pub async fn presign_download( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/lore.storage.v1.StorageService/PresignDownload", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("lore.storage.v1.StorageService", "PresignDownload"), + ); + self.inner.unary(req, path, codec).await + } pub async fn verify( &mut self, request: impl tonic::IntoRequest, @@ -892,6 +1072,17 @@ pub mod storage_service_server { tonic::Response, tonic::Status, >; + /// UploadContent accepts one header followed by raw byte chunks. Lore owns + /// chunking, hashing, compression, fragment-list construction, and durable + /// storage. Cancellation can leave unreachable content-addressed fragments; + /// it never publishes a revision or branch. + async fn upload_content( + &self, + request: tonic::Request>, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; /// Server streaming response type for the Put method. type PutStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, @@ -906,6 +1097,13 @@ pub mod storage_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn presign_download( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn verify( &self, request: tonic::Request, @@ -1225,6 +1423,53 @@ pub mod storage_service_server { }; Box::pin(fut) } + "/lore.storage.v1.StorageService/UploadContent" => { + #[allow(non_camel_case_types)] + struct UploadContentSvc(pub Arc); + impl< + T: StorageService, + > tonic::server::ClientStreamingService + for UploadContentSvc { + type Response = super::UploadContentResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + tonic::Streaming, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::upload_content(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = UploadContentSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.client_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/lore.storage.v1.StorageService/Put" => { #[allow(non_camel_case_types)] struct PutSvc(pub Arc); @@ -1314,6 +1559,52 @@ pub mod storage_service_server { }; Box::pin(fut) } + "/lore.storage.v1.StorageService/PresignDownload" => { + #[allow(non_camel_case_types)] + struct PresignDownloadSvc(pub Arc); + impl< + T: StorageService, + > tonic::server::UnaryService + for PresignDownloadSvc { + type Response = super::PresignDownloadResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::presign_download(&inner, request) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = PresignDownloadSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/lore.storage.v1.StorageService/Verify" => { #[allow(non_camel_case_types)] struct VerifySvc(pub Arc); diff --git a/lore-proto/src/grpc/lore.thin_client.v1.rs b/lore-proto/src/grpc/lore.thin_client.v1.rs index 9cf39eccc..dbc9037f3 100644 --- a/lore-proto/src/grpc/lore.thin_client.v1.rs +++ b/lore-proto/src/grpc/lore.thin_client.v1.rs @@ -65,6 +65,11 @@ pub struct DiffConflict { /// The change the other side made relative to the common ancestor. #[prost(message, optional, tag = "2")] pub change_to: ::core::option::Option, + /// Stable identifier for this conflict when the resolved base and the + /// request's from/to revisions remain pinned. Pass this value to + /// lore.revision.v1.BranchMerge when selecting a resolution. + #[prost(bytes = "bytes", tag = "3")] + pub conflict_id: ::prost::bytes::Bytes, } impl ::prost::Name for DiffConflict { const NAME: &'static str = "DiffConflict"; @@ -116,6 +121,11 @@ pub struct TreeNode { /// Content address for FILE / LINK entries; unused for DIRECTORY. #[prost(message, optional, tag = "3")] pub address: ::core::option::Option, + /// Signature of the revision in the requested repository that most recently + /// changed this entry. Empty for directories. Descendants of a linked + /// repository inherit the revision that last changed the link pin. + #[prost(bytes = "bytes", tag = "7")] + pub last_changed_revision_signature: ::prost::bytes::Bytes, /// Original size in bytes. For DIRECTORY entries, this is the cumulative size of its descendant files. #[prost(uint64, tag = "4")] pub size: u64, @@ -569,15 +579,18 @@ impl ::prost::Name for RevisionInfoResponse { "/lore.thin_client.v1.RevisionInfoResponse".into() } } -/// Request to diff two revisions. Server picks the diff mode (2-way vs -/// 3-way) from revision metadata; callers do not declare it. +/// Request to diff two revisions. Callers may explicitly select pairwise or +/// merge semantics; AUTO preserves the server's metadata-driven behavior. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RevisionDiffRequest { - /// When true and the server picks 3-way mode, auto-resolved - /// DiffChange entries are flagged via `automerged = true`. Silently - /// ignored in 2-way mode. + /// When true and the effective mode is MERGE, auto-resolved DiffChange + /// entries are flagged via `automerged = true`. Silently ignored in + /// PAIRWISE mode. #[prost(bool, tag = "5")] pub autoresolve: bool, + /// Requested comparison semantics. AUTO preserves the legacy behavior. + #[prost(enumeration = "RevisionDiffMode", tag = "6")] + pub mode: i32, /// "From" side specifier. #[prost(oneof = "revision_diff_request::QueryFrom", tags = "1, 2")] pub query_from: ::core::option::Option, @@ -620,8 +633,8 @@ impl ::prost::Name for RevisionDiffRequest { } /// Header for a RevisionDiff stream. Echoes the resolved revisions so /// callers querying by signature learn the (branch, number) and vice -/// versa. Optional `_base` fields are populated only when the server -/// selected a 3-way diff. +/// versa. `mode` reports the effective mode after AUTO resolution. Optional +/// `_base` fields are populated for non-identical MERGE comparisons. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct RevisionDiffHeader { /// Resolved (branch, number) of the "from" side. @@ -650,6 +663,9 @@ pub struct RevisionDiffHeader { /// 3-way mode. #[prost(bytes = "bytes", optional, tag = "6")] pub signature_base: ::core::option::Option<::prost::bytes::Bytes>, + /// Effective comparison mode after resolving AUTO. + #[prost(enumeration = "RevisionDiffMode", tag = "7")] + pub mode: i32, } impl ::prost::Name for RevisionDiffHeader { const NAME: &'static str = "RevisionDiffHeader"; @@ -794,6 +810,127 @@ impl ::prost::Name for RevisionTreeResponse { "/lore.thin_client.v1.RevisionTreeResponse".into() } } +/// Request a browser-ready download URL for one exact file path. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionFileDownloadRequest { + /// Repository-relative path to a file. Directories and links are rejected. + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + /// Requested URL lifetime. Zero selects the server default; non-zero values + /// are clamped to the configured minimum and maximum. + #[prost(uint64, tag = "4")] + pub ttl_seconds: u64, + /// Response Content-Type. Empty defaults to application/octet-stream. + #[prost(string, tag = "5")] + pub content_type: ::prost::alloc::string::String, + /// When true, Content-Disposition is `inline`; otherwise it is `attachment`. + /// In both cases Lore derives and safely encodes the filename from `path`. + #[prost(bool, tag = "6")] + pub inline: bool, + /// Revision specifier. + #[prost(oneof = "revision_file_download_request::Query", tags = "1, 2")] + pub query: ::core::option::Option, +} +/// Nested message and enum types in `RevisionFileDownloadRequest`. +pub mod revision_file_download_request { + /// Revision specifier. + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Query { + /// Identifier-based lookup; `number == 0` resolves to the tip of + /// the branch and the response pins the concrete signature. + #[prost(message, tag = "1")] + Identifier(crate::lore::model::v1::RevisionIdentifier), + /// Signature-based lookup. + #[prost(bytes, tag = "2")] + Signature(::prost::bytes::Bytes), + } +} +impl ::prost::Name for RevisionFileDownloadRequest { + const NAME: &'static str = "RevisionFileDownloadRequest"; + const PACKAGE: &'static str = "lore.thin_client.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.thin_client.v1.RevisionFileDownloadRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.thin_client.v1.RevisionFileDownloadRequest".into() + } +} +/// A pinned logical-file download. `url_suffix` is relative to the externally +/// configured Lore HTTP origin because the gRPC server does not know ingress +/// or reverse-proxy host names. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RevisionFileDownloadResponse { + /// Concrete revision selected by the request. + #[prost(message, optional, tag = "1")] + pub revision: ::core::option::Option, + /// Repository that owns the resolved file. This can differ from the request + /// repository when the path crosses an authorized repository link. + #[prost(bytes = "bytes", tag = "2")] + pub repository_resolved: ::prost::bytes::Bytes, + /// Immutable logical-content address stored on the file node. + #[prost(message, optional, tag = "3")] + pub address: ::core::option::Option, + /// Logical file length after defragmentation and decompression. + #[prost(uint64, tag = "4")] + pub size: u64, + /// Short-lived Lore HTTP path, including its signed query token. + #[prost(string, tag = "5")] + pub url_suffix: ::prost::alloc::string::String, + /// Unix epoch seconds at which the URL expires. + #[prost(uint64, tag = "6")] + pub expires_at_epoch_seconds: u64, + /// Final path component used for Content-Disposition. + #[prost(string, tag = "7")] + pub file_name: ::prost::alloc::string::String, + /// File mode stored on the revision node. + #[prost(uint32, tag = "8")] + pub mode: u32, +} +impl ::prost::Name for RevisionFileDownloadResponse { + const NAME: &'static str = "RevisionFileDownloadResponse"; + const PACKAGE: &'static str = "lore.thin_client.v1"; + fn full_name() -> ::prost::alloc::string::String { + "lore.thin_client.v1.RevisionFileDownloadResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/lore.thin_client.v1.RevisionFileDownloadResponse".into() + } +} +/// Controls whether RevisionDiff compares two snapshots directly or computes +/// their merge-relative changes through a common ancestor. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum RevisionDiffMode { + /// Preserve the server's historical mode selection: pairwise for revisions + /// on the same branch (or at a branch point), merge otherwise. + Auto = 0, + /// Always compare the "from" snapshot directly with the "to" snapshot. + Pairwise = 1, + /// Always resolve a common ancestor and compute a three-way merge diff. + Merge = 2, +} +impl RevisionDiffMode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Auto => "REVISION_DIFF_MODE_AUTO", + Self::Pairwise => "REVISION_DIFF_MODE_PAIRWISE", + Self::Merge => "REVISION_DIFF_MODE_MERGE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "REVISION_DIFF_MODE_AUTO" => Some(Self::Auto), + "REVISION_DIFF_MODE_PAIRWISE" => Some(Self::Pairwise), + "REVISION_DIFF_MODE_MERGE" => Some(Self::Merge), + _ => None, + } + } +} /// Generated client implementations. pub mod thin_client_service_client { #![allow( @@ -805,8 +942,8 @@ pub mod thin_client_service_client { )] use tonic::codegen::*; use tonic::codegen::http::Uri; - /// Thin-client presentation helpers — describe / tree / diff RPCs that are - /// convenience helpers for clients that lack local cache or compute ( web UIs). + /// Thin-client presentation helpers — describe / tree / diff / download RPCs + /// for clients that lack local cache or compute (web UIs). /// Sits alongside the baseline lore.revision.v1.RevisionService and shares the /// same backing revision-graph state. #[derive(Debug, Clone)] @@ -1012,6 +1149,38 @@ pub mod thin_client_service_client { ); self.inner.server_streaming(req, path, codec).await } + /// Resolve one file at an immutable revision and create a short-lived URL + /// that streams the reconstructed logical bytes through the Lore HTTP + /// server. The returned URL never exposes storage fragment URLs. + pub async fn revision_file_download( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/lore.thin_client.v1.ThinClientService/RevisionFileDownload", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "lore.thin_client.v1.ThinClientService", + "RevisionFileDownload", + ), + ); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -1080,9 +1249,19 @@ pub mod thin_client_service_server { tonic::Response, tonic::Status, >; + /// Resolve one file at an immutable revision and create a short-lived URL + /// that streams the reconstructed logical bytes through the Lore HTTP + /// server. The returned URL never exposes storage fragment URLs. + async fn revision_file_download( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } - /// Thin-client presentation helpers — describe / tree / diff RPCs that are - /// convenience helpers for clients that lack local cache or compute ( web UIs). + /// Thin-client presentation helpers — describe / tree / diff / download RPCs + /// for clients that lack local cache or compute (web UIs). /// Sits alongside the baseline lore.revision.v1.RevisionService and shares the /// same backing revision-graph state. #[derive(Debug)] @@ -1348,6 +1527,55 @@ pub mod thin_client_service_server { }; Box::pin(fut) } + "/lore.thin_client.v1.ThinClientService/RevisionFileDownload" => { + #[allow(non_camel_case_types)] + struct RevisionFileDownloadSvc(pub Arc); + impl< + T: ThinClientService, + > tonic::server::UnaryService + for RevisionFileDownloadSvc { + type Response = super::RevisionFileDownloadResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::revision_file_download( + &inner, + request, + ) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RevisionFileDownloadSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => { Box::pin(async move { let mut response = http::Response::new( diff --git a/lore-proto/tests/v1_revision.rs b/lore-proto/tests/v1_revision.rs index e195eab3d..f848ff8b4 100644 --- a/lore-proto/tests/v1_revision.rs +++ b/lore-proto/tests/v1_revision.rs @@ -1,6 +1,6 @@ // SPDX-FileCopyrightText: 2026 Epic Games, Inc. // SPDX-License-Identifier: MIT -//! Smoke test verifying `lore.revision.v1` carries the 8 RPCs' request / +//! Smoke test verifying `lore.revision.v1` carries the RPC request / //! response messages. use lore_proto::lore::revision::v1::BranchCreateRequest; @@ -11,15 +11,30 @@ use lore_proto::lore::revision::v1::BranchGetRequest; use lore_proto::lore::revision::v1::BranchGetResponse; use lore_proto::lore::revision::v1::BranchListRequest; use lore_proto::lore::revision::v1::BranchListResponse; +use lore_proto::lore::revision::v1::BranchMergeConflict; +use lore_proto::lore::revision::v1::BranchMergeRequest; +use lore_proto::lore::revision::v1::BranchMergeResolution; +use lore_proto::lore::revision::v1::BranchMergeResponse; +use lore_proto::lore::revision::v1::BranchMergeSide; use lore_proto::lore::revision::v1::BranchMetadataGetRequest; use lore_proto::lore::revision::v1::BranchMetadataGetResponse; use lore_proto::lore::revision::v1::BranchMetadataSetRequest; use lore_proto::lore::revision::v1::BranchMetadataSetResponse; use lore_proto::lore::revision::v1::BranchPushRequest; use lore_proto::lore::revision::v1::BranchPushResponse; +use lore_proto::lore::revision::v1::RevisionCreateDeletePath; +use lore_proto::lore::revision::v1::RevisionCreateDirectory; +use lore_proto::lore::revision::v1::RevisionCreateMetadataEntry; +use lore_proto::lore::revision::v1::RevisionCreateMovePath; +use lore_proto::lore::revision::v1::RevisionCreateOperation; +use lore_proto::lore::revision::v1::RevisionCreatePutFile; +use lore_proto::lore::revision::v1::RevisionCreateRequest; +use lore_proto::lore::revision::v1::RevisionCreateResponse; use lore_proto::lore::revision::v1::RevisionListRequest; use lore_proto::lore::revision::v1::RevisionListResponse; use lore_proto::lore::revision::v1::branch_get_request::Query as BranchGetQuery; +use lore_proto::lore::revision::v1::branch_merge_resolution::Resolution as BranchMergeResolutionKind; +use lore_proto::lore::revision::v1::revision_create_operation::Op as RevisionCreateOp; use lore_proto::lore::revision::v1::revision_list_request::Start as RevisionListStart; #[test] @@ -34,12 +49,16 @@ fn v1_revision_request_response_types_default() { let _ = BranchListResponse::default(); let _ = BranchPushRequest::default(); let _ = BranchPushResponse::default(); + let _ = BranchMergeRequest::default(); + let _ = BranchMergeResponse::default(); let _ = BranchMetadataGetRequest::default(); let _ = BranchMetadataGetResponse::default(); let _ = BranchMetadataSetRequest::default(); let _ = BranchMetadataSetResponse::default(); let _ = RevisionListRequest::default(); let _ = RevisionListResponse::default(); + let _ = RevisionCreateRequest::default(); + let _ = RevisionCreateResponse::default(); } /// Field-shape regression net: destructuring each message + naming each @@ -84,6 +103,33 @@ fn v1_revision_field_shapes() { message: _, } = BranchPushResponse::default(); + let BranchMergeRequest { + request_id: _, + branch_id_target: _, + revision_signature_target: _, + branch_id_source: _, + revision_signature_source: _, + commit_message: _, + resolutions: _, + } = BranchMergeRequest::default(); + let BranchMergeResolution { + conflict_id: _, + resolution: _, + } = BranchMergeResolution::default(); + let _ = BranchMergeResolutionKind::Side(BranchMergeSide::Target as i32); + let BranchMergeConflict { + conflict_id: _, + path_source: _, + path_target: _, + } = BranchMergeConflict::default(); + let BranchMergeResponse { + outcome: _, + revision_signature: _, + revision_number: _, + revision_signature_base: _, + unresolved_conflicts: _, + } = BranchMergeResponse::default(); + let BranchMetadataGetRequest { id: _ } = BranchMetadataGetRequest::default(); let BranchMetadataGetResponse { metadata: _ } = BranchMetadataGetResponse::default(); let BranchMetadataSetRequest { @@ -101,4 +147,38 @@ fn v1_revision_field_shapes() { signature_forward: _, signature_backward: _, } = RevisionListResponse::default(); + + let RevisionCreateRequest { + request_id: _, + branch_id: _, + revision_signature_base: _, + commit_message: _, + metadata: _, + operations: _, + } = RevisionCreateRequest::default(); + let RevisionCreateMetadataEntry { + key: _, + value: _, + format: _, + } = RevisionCreateMetadataEntry::default(); + let RevisionCreateOperation { op: _ } = RevisionCreateOperation::default(); + let _ = RevisionCreateOp::PutFile(Default::default()); + let _ = RevisionCreateOp::CreateDirectory(Default::default()); + let _ = RevisionCreateOp::DeletePath(Default::default()); + let _ = RevisionCreateOp::MovePath(Default::default()); + let RevisionCreatePutFile { + path: _, + mode: _, + address: _, + } = RevisionCreatePutFile::default(); + let RevisionCreateDirectory { path: _, mode: _ } = RevisionCreateDirectory::default(); + let RevisionCreateDeletePath { path: _ } = RevisionCreateDeletePath::default(); + let RevisionCreateMovePath { + source: _, + destination: _, + } = RevisionCreateMovePath::default(); + let RevisionCreateResponse { + revision_signature: _, + revision_number: _, + } = RevisionCreateResponse::default(); } diff --git a/lore-proto/tests/v1_thin_client.rs b/lore-proto/tests/v1_thin_client.rs index 8628a62de..e3a7f59ca 100644 --- a/lore-proto/tests/v1_thin_client.rs +++ b/lore-proto/tests/v1_thin_client.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2026 Epic Games, Inc. // SPDX-License-Identifier: MIT //! Smoke test verifying `lore.thin_client.v1` carries the model types and -//! the 4 RPCs' request / response messages. +//! the RPC request / response messages. use lore_proto::lore::thin_client::v1::Action; use lore_proto::lore::thin_client::v1::ContentDiffChunkResponse; @@ -16,8 +16,11 @@ use lore_proto::lore::thin_client::v1::MetadataType; use lore_proto::lore::thin_client::v1::NodeType; use lore_proto::lore::thin_client::v1::Revision; use lore_proto::lore::thin_client::v1::RevisionDiffHeader; +use lore_proto::lore::thin_client::v1::RevisionDiffMode; use lore_proto::lore::thin_client::v1::RevisionDiffRequest; use lore_proto::lore::thin_client::v1::RevisionDiffResponse; +use lore_proto::lore::thin_client::v1::RevisionFileDownloadRequest; +use lore_proto::lore::thin_client::v1::RevisionFileDownloadResponse; use lore_proto::lore::thin_client::v1::RevisionInfoRequest; use lore_proto::lore::thin_client::v1::RevisionInfoResponse; use lore_proto::lore::thin_client::v1::RevisionTreeHeader; @@ -29,6 +32,7 @@ use lore_proto::lore::thin_client::v1::revision::Parent as RevisionParent; use lore_proto::lore::thin_client::v1::revision_diff_request::QueryFrom as RevisionDiffQueryFrom; use lore_proto::lore::thin_client::v1::revision_diff_request::QueryTo as RevisionDiffQueryTo; use lore_proto::lore::thin_client::v1::revision_diff_response::Payload as RevisionDiffPayload; +use lore_proto::lore::thin_client::v1::revision_file_download_request::Query as RevisionFileDownloadQuery; use lore_proto::lore::thin_client::v1::revision_info_request::Query as RevisionInfoQuery; use lore_proto::lore::thin_client::v1::revision_tree_request::Query as RevisionTreeQuery; use lore_proto::lore::thin_client::v1::revision_tree_response::Payload as RevisionTreePayload; @@ -49,6 +53,7 @@ fn v1_thin_client_model_types_default() { assert_eq!(NodeType::Directory as i32, 0); assert_eq!(Action::Keep as i32, 0); assert_eq!(MetadataType::Address as i32, 0); + assert_eq!(RevisionDiffMode::Auto as i32, 0); } #[test] @@ -61,6 +66,8 @@ fn v1_thin_client_service_types_default() { let _ = RevisionTreeRequest::default(); let _ = RevisionTreeResponse::default(); let _ = RevisionTreeHeader::default(); + let _ = RevisionFileDownloadRequest::default(); + let _ = RevisionFileDownloadResponse::default(); } /// Field-shape regression net: destructuring each message + naming each @@ -107,6 +114,7 @@ fn v1_thin_client_field_shapes() { let DiffConflict { change_from: _, change_to: _, + conflict_id: _, } = DiffConflict::default(); let DiffPartition { index: _, @@ -116,6 +124,7 @@ fn v1_thin_client_field_shapes() { path: _, node_type: _, address: _, + last_changed_revision_signature: _, size: _, mode: _, tracking: _, @@ -155,6 +164,7 @@ fn v1_thin_client_field_shapes() { query_from: _, query_to: _, autoresolve: _, + mode: _, } = RevisionDiffRequest::default(); let _ = RevisionDiffQueryFrom::IdentifierFrom(Default::default()); let _ = RevisionDiffQueryFrom::SignatureFrom(Default::default()); @@ -167,6 +177,7 @@ fn v1_thin_client_field_shapes() { signature_to: _, identifier_base: _, signature_base: _, + mode: _, } = RevisionDiffHeader::default(); let RevisionDiffResponse { payload: _ } = RevisionDiffResponse::default(); let _ = RevisionDiffPayload::Header(Default::default()); @@ -189,4 +200,25 @@ fn v1_thin_client_field_shapes() { let RevisionTreeResponse { payload: _ } = RevisionTreeResponse::default(); let _ = RevisionTreePayload::Header(Default::default()); let _ = RevisionTreePayload::Node(Default::default()); + + // RevisionFileDownload + let RevisionFileDownloadRequest { + query: _, + path: _, + ttl_seconds: _, + content_type: _, + inline: _, + } = RevisionFileDownloadRequest::default(); + let _ = RevisionFileDownloadQuery::Identifier(Default::default()); + let _ = RevisionFileDownloadQuery::Signature(Default::default()); + let RevisionFileDownloadResponse { + revision: _, + repository_resolved: _, + address: _, + size: _, + url_suffix: _, + expires_at_epoch_seconds: _, + file_name: _, + mode: _, + } = RevisionFileDownloadResponse::default(); } diff --git a/lore-revision/src/branch.rs b/lore-revision/src/branch.rs index 0dd0c4019..ad98cc1e1 100644 --- a/lore-revision/src/branch.rs +++ b/lore-revision/src/branch.rs @@ -72,6 +72,7 @@ use crate::node::Node; use crate::node::NodeBlock; use crate::node::NodeFlags; use crate::node::NodeIDExt; +use crate::path_merge::PathMergeRule; use crate::repository; use crate::repository::RepositoryContext; use crate::repository::RepositoryWriteToken; @@ -2553,36 +2554,62 @@ pub async fn diff3( graft_view: Option>, tx: mpsc::Sender>, ) -> Result { - Box::pin(diff3_with_source_cap( + Box::pin(diff3_with_options( repository, source_branch, source_revision, target_branch, target_revision, - path, - include_same, - auto_resolve, - None, - None, - graft_view, + Diff3Options { + path, + include_same, + auto_resolve, + graft_view, + ..Default::default() + }, tx, )) .await } -#[allow(clippy::too_many_arguments)] -pub async fn diff3_with_source_cap( +/// Optional behavior for streaming branch 3-way diffs. +#[derive(Clone, Debug)] +pub struct Diff3Options<'a> { + pub path: Option, + pub include_same: bool, + pub auto_resolve: bool, + /// Abort when source's diff produces more than this many items. + pub source_cap: Option, + /// Permit count for parallel history-walk conflict checks. + pub history_walk_concurrency: Option, + /// Ordered per-path merge rules used to suppress matching merge inputs. + pub path_merge_rules: &'a [PathMergeRule], + /// View used to identify untouched out-of-view subtrees that can be grafted. + /// `None` disables graft adoption. + pub graft_view: Option>, +} + +impl Default for Diff3Options<'_> { + fn default() -> Self { + Self { + path: None, + include_same: false, + auto_resolve: false, + source_cap: None, + history_walk_concurrency: None, + path_merge_rules: &[], + graft_view: None, + } + } +} + +pub async fn diff3_with_options( repository: Arc, source_branch: BranchId, source_revision: Hash, target_branch: BranchId, target_revision: Hash, - path: Option, - include_same: bool, - auto_resolve: bool, - source_cap: Option, - history_walk_concurrency: Option, - graft_view: Option>, + options: Diff3Options<'_>, tx: mpsc::Sender>, ) -> Result { lore_info!( @@ -2619,16 +2646,20 @@ pub async fn diff3_with_source_cap( }; let (inner_tx, mut inner_rx) = mpsc::channel::>(256); - let mut driver = std::pin::pin!(revision::diff3_with_source_cap( + let auto_resolve = options.auto_resolve; + let mut driver = std::pin::pin!(revision::diff3_with_options( repository.clone(), base_revision, source_revision, target_revision, - path, - include_same, - source_cap, - history_walk_concurrency, - graft_view, + revision::Diff3Options { + path: options.path, + include_same: options.include_same, + source_cap: options.source_cap, + history_walk_concurrency: options.history_walk_concurrency, + path_merge_rules: options.path_merge_rules, + graft_view: options.graft_view, + }, inner_tx, )); loop { @@ -3063,16 +3094,18 @@ pub async fn diff3_collect( include_same: bool, auto_resolve: bool, ) -> Result { - diff3_collect_with_graft( + diff3_collect_with_options( repository, source_branch, source_revision, target_branch, target_revision, - path, - include_same, - auto_resolve, - None, + Diff3Options { + path, + include_same, + auto_resolve, + ..Default::default() + }, ) .await } @@ -3092,18 +3125,40 @@ pub async fn diff3_collect_with_graft( include_same: bool, auto_resolve: bool, graft_view: Option>, +) -> Result { + diff3_collect_with_options( + repository, + source_branch, + source_revision, + target_branch, + target_revision, + Diff3Options { + path, + include_same, + auto_resolve, + graft_view, + ..Default::default() + }, + ) + .await +} + +pub async fn diff3_collect_with_options( + repository: Arc, + source_branch: BranchId, + source_revision: Hash, + target_branch: BranchId, + target_revision: Hash, + options: Diff3Options<'_>, ) -> Result { let (summary, items) = crate::util::collect_stream::collect_stream_with_summary(|tx| { - diff3( + diff3_with_options( repository, source_branch, source_revision, target_branch, target_revision, - path, - include_same, - auto_resolve, - graft_view, + options, tx, ) }) diff --git a/lore-revision/src/branch/merge.rs b/lore-revision/src/branch/merge.rs index 336c57b56..89182cf46 100644 --- a/lore-revision/src/branch/merge.rs +++ b/lore-revision/src/branch/merge.rs @@ -35,6 +35,7 @@ use crate::interface::LoreEvent; use crate::interface::LoreFileAction; use crate::interface::LoreString; use crate::link; +use crate::lore::Address; use crate::lore::BranchId; use crate::lore::Hash; use crate::lore::RepositoryId; @@ -54,6 +55,9 @@ use crate::node::NodeFlags; use crate::node::NodeID; use crate::node::NodeLink; use crate::path::emit_path_ignore; +use crate::path_merge::PathMergePolicy; +pub use crate::path_merge::PathMergeRule; +pub use crate::path_merge::PathMergeStrategy; use crate::repository::RepositoryContext; use crate::repository::RepositoryWriteToken; use crate::revision::DiffResult; @@ -396,6 +400,9 @@ pub struct MergeStartOptions { pub no_commit: bool, /// Which repositories to include in the merge. pub scope: MergeScope, + /// Ordered per-path merge rules. The most-specific match wins; equal + /// specificity uses the later rule. + pub path_merge_rules: Vec, } /// The revisions a merge's three-way diff ran between, other than the target. @@ -433,6 +440,57 @@ pub struct ConflictRealizeContext { pub conflicts: Arc>, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DiffChangeSide { + Source, + Target, + Unknown, +} + +fn diff_change_side(change: &NodeChange, source: Hash, target: Hash) -> DiffChangeSide { + let revision = change.to.state.revision(); + if revision == source { + DiffChangeSide::Source + } else if revision == target { + DiffChangeSide::Target + } else { + DiffChangeSide::Unknown + } +} + +fn apply_path_merge_rules(diff: &mut DiffResult, rules: &[PathMergeRule]) { + let policy = PathMergePolicy::new(rules); + if policy.is_empty() { + return; + } + + let source = diff.source; + let target = diff.target; + let change_count = diff.changes.len(); + diff.changes.retain(|change| { + let strategy = policy.strategy_for_change(change); + match strategy { + PathMergeStrategy::Merge => true, + PathMergeStrategy::KeepTarget | PathMergeStrategy::Exclude => { + diff_change_side(change, source, target) != DiffChangeSide::Source + } + } + }); + + let conflict_count = diff.conflicts.len(); + diff.conflicts.retain(|(source_change, target_change)| { + policy.should_merge_conflict(source_change, target_change) + }); + + let suppressed_changes = change_count.saturating_sub(diff.changes.len()); + let suppressed_conflicts = conflict_count.saturating_sub(diff.conflicts.len()); + if suppressed_changes > 0 || suppressed_conflicts > 0 { + lore_debug!( + "Path merge strategy suppressed {suppressed_changes} source changes and {suppressed_conflicts} conflicts" + ); + } +} + async fn merge_repository( repository: Arc, token: &RepositoryWriteToken, @@ -440,6 +498,7 @@ async fn merge_repository( current_branch: BranchId, current_signature: Hash, state_current: Arc, + path_merge_rules: &[PathMergeRule], ) -> Result { let latest_merge = branch::load_latest(repository.clone(), source_branch) .await @@ -568,18 +627,22 @@ async fn merge_repository( // Diff over the full tree. A view-scoped diff drops the other branch's // out-of-view changes and leaves this branch divergent from it. - let diff = Box::pin(branch::diff3_collect_with_graft( + let diff = Box::pin(branch::diff3_collect_with_options( full_tree_context(&repository), source_branch, revision, current_branch, current_signature, - None, /* No path */ - true, /* Include identical changes for merge tracking */ - false, /* Do not autoresolve, this is done later */ - // The view decides which subtrees are out of view. The walk stays - // full-tree. - Some(repository.filter.clone()), + branch::Diff3Options { + path: None, + include_same: true, /* Include identical changes for merge tracking */ + auto_resolve: false, /* Do not autoresolve, this is done later */ + path_merge_rules, + // The view decides which subtrees are out of view. The walk stays + // full-tree. + graft_view: Some(repository.filter.clone()), + ..Default::default() + }, )) .await .forward::("running diff3 for merge")?; @@ -600,6 +663,7 @@ async fn merge_repository( diff, state_current, MergeType::BranchMerge, + path_merge_rules, // If this merge is reconciling a remote LATEST with local LATEST of a branch // reverse the parent order in order to keep the remote history as the main // history line shows in CLI output and other places @@ -721,6 +785,7 @@ pub async fn merge_start( current_branch, state_current.revision(), state_current, + &options.path_merge_rules, ) .await?; @@ -905,6 +970,7 @@ async fn merge_start_link( link_branch, link_reference.signature, link_state, + &options.path_merge_rules, ) .await .forward_with::(|| format!("merging link {link_path}"))?; @@ -1254,6 +1320,7 @@ async fn merge_start_all( eligible.resolved_branch, eligible.link_reference.signature, link_state, + &options.path_merge_rules, ) .await .forward_with::(|| format!("merging link {}", eligible.link_path))?; @@ -1497,6 +1564,7 @@ async fn finalize_main_merge( current_branch, state_current.revision(), state_current, + &options.path_merge_rules, ) .await?; @@ -1884,6 +1952,7 @@ pub async fn apply_diff( mut diff: DiffResult, state_current: Arc, merge_type: MergeType, + path_merge_rules: &[PathMergeRule], reverse_parents: bool, target_branch: BranchId, ) -> Result { @@ -1923,6 +1992,10 @@ pub async fn apply_diff( }); } + if matches!(merge_type, MergeType::BranchMerge) { + apply_path_merge_rules(&mut diff, path_merge_rules); + } + // A graft covers a whole subtree the target branch never touched. Nothing // inside it needs a three-way merge, so it skips per-file verify and // realize. @@ -3948,7 +4021,7 @@ async fn merge_into_link( .forward::("serializing state")?; // Collect and push fragments - let fragments = state::collect_new_fragments( + let mut fragments = state::collect_new_fragments( repository.clone(), state_branch.clone(), state_new.clone(), @@ -3956,6 +4029,9 @@ async fn merge_into_link( ) .await .forward::("collecting new fragments")?; + if !state_new.parent_other().is_zero() { + fragments.push(Address::zero_context_hash(state_new.parent_other())); + } let mut revision = signature; let mut revision_number = state_new.revision_number(); @@ -4307,7 +4383,7 @@ pub async fn merge_into( state_branch.revision(), state_new.revision() ); - let fragments = state::collect_new_fragments( + let mut fragments = state::collect_new_fragments( repository.clone(), state_branch.clone(), state_new.clone(), @@ -4315,6 +4391,9 @@ pub async fn merge_into( ) .await .forward::("collecting new fragments")?; + if !state_new.parent_other().is_zero() { + fragments.push(Address::zero_context_hash(state_new.parent_other())); + } let mut revision = signature; let mut revision_number = state_new.revision_number(); diff --git a/lore-revision/src/commit.rs b/lore-revision/src/commit.rs index 2e935a97b..f0261ebef 100644 --- a/lore-revision/src/commit.rs +++ b/lore-revision/src/commit.rs @@ -1386,6 +1386,373 @@ pub async fn store_branch_latest_and_make_current( Ok(()) } +/// Commit an in-memory revision tree state as a new revision on `branch`, +/// atomically advancing the branch's local tip. +/// +/// The memory-based counterpart of [`commit`], serving the +/// `lore_revision_tree_*` surface: same tip-collision check, delta block, +/// history weave, state serialization, and branch-advance path as the +/// file-system-based commit — with the working-tree walk replaced by a +/// freeze pass over the staged nodes. Node content addresses were supplied +/// by the caller through the revision tree write verbs and are recorded +/// as-is; no bytes move here. +/// +/// `pending_metadata` carries the caller's accumulated `metadata_set` edits +/// (a caller-set commit message travels on the `message` key and is +/// preserved). `deleted` carries the delta entries recorded when deleted +/// subtrees were discarded from the child chains at verb time — the freeze +/// pass can no longer observe those nodes. The new revision derives from +/// the state's loaded revision; when the branch tip has advanced past it +/// the commit fails with `BranchAdvanced` before any state is touched. +/// +/// On success the state's signature is the newly-committed revision, so the +/// caller's handle behaves as if freshly loaded from it. +pub async fn commit_tree( + repository: Arc, + token: &RepositoryWriteToken, + state: Arc, + pending_metadata: Metadata, + branch: BranchId, + deleted: Vec, +) -> Result { + let context = execution_context(); + let globals = context.globals(); + let parent_revision = state.revision(); + + // Same early tip-collision check as the file-system-based commit; the + // mutable store lock keeps the advance itself atomic regardless of + // check placement. + let branch_latest = branch::load_latest(repository.clone(), branch) + .await + .unwrap_or_default(); + if !globals.force() && !branch_latest.is_zero() && branch_latest != parent_revision { + return Err(BranchAdvanced.into()); + } + + let signature = construct_tree_revision( + repository.clone(), + token, + state.clone(), + pending_metadata, + branch, + deleted, + ) + .await?; + + if !globals.dry_run() { + branch::store_latest( + repository.clone(), + branch, + parent_revision, + signature, + BranchLatestStatus::Divergent, + ) + .await + .forward::("Failed to store current branch latest")?; + } + + event::LoreEvent::RevisionCommitRevision(LoreRevisionCommitRevisionEventData { + repository: repository.id, + branch, + revision: signature, + revision_number: state.revision_number(), + parent: state.parent_self(), + parent_other: state.parent_other(), + }) + .send(); + + Ok(signature) +} + +/// Construct and serialize an immutable revision from an in-memory tree. +/// +/// This performs the freeze, directory rehash, delta/history weave, metadata, +/// and immutable state serialization portions of [`commit_tree`]. It never +/// reads or updates branch latest. Server-side callers use this boundary to +/// construct content-addressed state before the authoritative BranchPush +/// protection, verification, hook, and compare-and-swap path publishes it. +/// Unpublished revisions may remain unreachable when publication loses a +/// race; that is safe content-addressed-storage behavior. +pub async fn construct_tree_revision( + repository: Arc, + token: &RepositoryWriteToken, + state: Arc, + pending_metadata: Metadata, + branch: BranchId, + deleted: Vec, +) -> Result { + construct_tree_revision_with_parent( + repository, + token, + state, + pending_metadata, + branch, + Hash::default(), + deleted, + false, + ) + .await +} + +/// Construct a normal two-parent merge revision from an in-memory target +/// tree. Unlike [`construct_tree_revision`], an unchanged tree is valid: the +/// second parent records that the source lineage was considered and merged. +#[allow(clippy::too_many_arguments)] +pub async fn construct_merge_revision( + repository: Arc, + token: &RepositoryWriteToken, + state: Arc, + pending_metadata: Metadata, + branch: BranchId, + source_revision: Hash, + deleted: Vec, +) -> Result { + if source_revision.is_zero() { + return Err(CommitError::internal( + "merge source revision must be non-zero", + )); + } + construct_tree_revision_with_parent( + repository, + token, + state, + pending_metadata, + branch, + source_revision, + deleted, + true, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn construct_tree_revision_with_parent( + repository: Arc, + token: &RepositoryWriteToken, + state: Arc, + pending_metadata: Metadata, + branch: BranchId, + parent_other: Hash, + deleted: Vec, + allow_unchanged_tree: bool, +) -> Result { + let context = execution_context(); + let globals = context.globals(); + let parent_revision = state.revision(); + + // An untouched handle has nothing to freeze: edits dirty the state and + // deletes leave recorded entries, so a clean state with no carried + // deletes is a no-op commit. + if !allow_unchanged_tree && !globals.force() && deleted.is_empty() && !state.is_dirty() { + return Err(NothingStaged.into()); + } + + // The pending metadata may carry a caller-set message; hoist it so + // prepare_commit_metadata records branch, timestamp, and authorship + // around it instead of overwriting it with an empty string. + let message = pending_metadata + .get_string(metadata::MESSAGE) + .unwrap_or_default() + .to_string(); + let metadata = prepare_commit_metadata( + repository.clone(), + pending_metadata, + branch, + message, + None, + None, + None, + ) + .await?; + + // Freeze pass: fold the deleted subtrees' recorded entries and every + // staged node into the delta, clearing leaf change flags as it goes + // (directories are cleared by the rehash). + let delta = Arc::new(parking_lot::RwLock::new(BytesMut::new())); + { + let mut writer = delta.write(); + for entry in deleted.iter() { + writer.extend_from_slice(entry.as_bytes()); + } + } + commit_tree_freeze(repository.clone(), state.clone(), ROOT_NODE, delta.clone()).await?; + + if !allow_unchanged_tree && !globals.force() { + let read = delta.read(); + if read.count::() == 0 { + return Err(NothingStaged.into()); + } + } + + // Derive the new revision from the loaded parent. The weave assigns the + // revision number; the metadata hash lands after it serializes below. + state.set_parent_self(parent_revision); + state.set_parent_other(parent_other); + + // Same graceful-drain pattern as commit_staged_revision: run all + // write-producing work under the tracker, then ALWAYS drain so no + // background leader outlives this function. + let tracker = immutable::commit_write_tracker(false); + let work_tracker = tracker.clone(); + let work_repository = repository.clone(); + let work_state = state.clone(); + let work_result: Result = async move { + let repository = work_repository; + let state = work_state; + + rehash_directory(repository.clone(), state.clone(), ROOT_NODE).await?; + + state + .update_tree_root_hash(repository.clone()) + .await + .forward::("Failed to update tree root hash")?; + + // Same identical-tree guard as the file-system-based commit: edits + // that reproduce the parent's tree byte for byte are a no-op. + if !allow_unchanged_tree + && !execution_context().globals().force() + && !parent_revision.is_zero() + { + let state_parent = State::deserialize(repository.clone(), parent_revision) + .await + .forward_with::(|| { + format!("Failed to deserialize revision state {parent_revision}") + })?; + let tree_staged = state + .tree(repository.clone()) + .await + .forward::("Failed to read revision tree data")?; + let tree_parent = state_parent + .tree(repository.clone()) + .await + .forward::("Failed to read revision tree data")?; + if tree_staged.hash_root == tree_parent.hash_root { + lore_debug!( + "Tree {} in memory-based commit is identical to tree {} in parent revision {}", + tree_staged.hash_root, + tree_parent.hash_root, + parent_revision, + ); + return Err(NothingStaged.into()); + } + } + + generate_delta_block( + repository.clone(), + state.clone(), + delta, + work_tracker.clone(), + ) + .await?; + + let metadata_hash = metadata + .serialize_with_tracker(repository.clone(), Some(work_tracker)) + .await + .forward::("Failed to write commit metadata")?; + state.set_metadata_hash(metadata_hash); + + weave_history(repository.clone(), state.clone()).await?; + + state.reset_merge_conflict_flags(); + state.clear_link_merge_state(); + + let signature = state + .serialize(repository.clone(), token) + .await + .forward::("Failed to serialize revision state")?; + + Ok(signature) + } + .await; + + // ALWAYS drain, regardless of the work outcome — dropping the tracker + // with live leaders would abort them mid-write. + let drain_result = tracker.await_all().await; + let signature = match work_result { + Ok(signature) => { + drain_result + .forward::("Background fragment upload task failed during commit")?; + signature + } + Err(work_err) => return Err(work_err), + }; + + Ok(signature) +} + +/// Freeze one directory of the memory-based revision tree: record a delta +/// entry for every staged node and clear the change flags on staged leaves +/// (files and links) so the rehash observes a settled tree. Directories +/// keep their staged bits — [`rehash_directory`] consumes and clears those. +/// Only staged directories are descended into: marking a node propagates +/// the base staged bit to every ancestor, so a clean directory cannot hold +/// staged descendants. Deleted subtrees were already discarded at verb +/// time; a staged-delete node surviving to commit is a broken invariant, +/// not input. +fn commit_tree_freeze( + repository: Arc, + state: Arc, + node_id: NodeID, + delta: Arc>, +) -> Pin> + Send>> { + Box::pin(async move { + let node = state + .node(repository.clone(), node_id) + .await + .forward::("Failed deserializing state block")?; + debug_assert!(node.is_directory()); + + if node.is_staged() { + delta_add(delta.clone(), node_id, node.flags); + } + + let mut children = + StateNodeChildrenIterator::new(state.clone(), repository.clone(), node_id) + .await + .forward::("Failed deserializing state block")?; + while let Some((child_node_id, child_node)) = children + .next() + .await + .forward::("Failed deserializing state block")? + { + if child_node.is_staged_delete() { + return Err(CommitError::internal( + "Deleted node remains in the memory-based revision tree", + )); + } + if child_node.is_directory() { + if child_node.is_staged() { + commit_tree_freeze( + repository.clone(), + state.clone(), + child_node_id, + delta.clone(), + ) + .await?; + } + } else if child_node.is_staged() { + delta_add(delta.clone(), child_node_id, child_node.flags); + let block_index = NodeBlock::index(child_node_id); + let block = state + .block(repository.clone(), block_index) + .await + .forward::("Failed deserializing state block")?; + let dirtied = { + let mut block_writer = block.write(); + let node = block_writer.node(Node::index(child_node_id)); + node.clear_all_change_flags(); + block_writer.mark_dirty() + }; + if dirtied { + state.block_modified(block, block_index); + state.mark_dirty(); + } + } + } + Ok(()) + }) +} + #[allow(clippy::too_many_arguments)] pub(crate) async fn commit_files_and_rehash( repository: Arc, diff --git a/lore-revision/src/event.rs b/lore-revision/src/event.rs index 515490408..dd8bb33e1 100644 --- a/lore-revision/src/event.rs +++ b/lore-revision/src/event.rs @@ -435,6 +435,8 @@ pub enum LoreErrorCode { Internal = -1, /// The backing store is overloaded; the caller should retry later. SlowDown = 31, + /// The target branch tip advanced past the caller's parent revision. + BranchAdvanced = 41, } // cbindgen cannot evaluate a const in a discriminant position — it drops the @@ -446,6 +448,8 @@ const _: () = assert!(LoreErrorCode::Internal as i32 == lore_error_set::Internal const _: () = assert!(LoreErrorCode::InvalidArguments as i32 == lore_base::error::InvalidArguments::FFI_CODE); const _: () = assert!(LoreErrorCode::SlowDown as i32 == lore_base::error::SlowDown::FFI_CODE); +const _: () = + assert!(LoreErrorCode::BranchAdvanced as i32 == lore_base::error::BranchAdvanced::FFI_CODE); const _: () = assert!(LoreErrorCode::AddressNotFound as i32 == lore_base::error::AddressNotFound::FFI_CODE); diff --git a/lore-revision/src/immutable.rs b/lore-revision/src/immutable.rs index bd59ee0ce..647e01bcf 100644 --- a/lore-revision/src/immutable.rs +++ b/lore-revision/src/immutable.rs @@ -251,15 +251,55 @@ pub async fn read_stream( address: Address, range: Option>, options: ReadOptions, - sender: Sender>, + sender: Sender>, ) -> Result { + let range = range.map(|range| range.start as u64..range.end as u64); + let (_, streamed) = read_stream_range(repository, address, range, options, sender).await?; + Ok(streamed.end - streamed.start) +} + +pub async fn read_stream_range( + repository: Arc, + address: Address, + range: Option>, + options: ReadOptions, + sender: Sender>, +) -> Result<(u64, Range), ImmutableError> { let store = repository.immutable_store(); let partition = repository.id; let session = resolve_session(&repository); - lore_storage::read_stream(store, partition, address, range, options, sender, session) - .await - .map(|(_fragment, streamed)| streamed.end - streamed.start) - .forward("reading immutable data") + let range = range + .map(|range| { + let start = usize::try_from(range.start) + .map_err(|_| ImmutableError::internal("stream range start is too large"))?; + let end = usize::try_from(range.end) + .map_err(|_| ImmutableError::internal("stream range end is too large"))?; + Ok::<_, ImmutableError>(start..end) + }) + .transpose()?; + let (storage_sender, mut storage_receiver) = tokio::sync::mpsc::channel(sender.max_capacity()); + let (fragment, streamed) = lore_storage::read_stream( + store, + partition, + address, + range, + options, + storage_sender, + session, + ) + .await + .forward::("reading immutable data")?; + lore_spawn!(async move { + while let Some(item) = storage_receiver.recv().await { + let item = item.map_err(|error| { + ImmutableError::internal_with_context(error, "reading immutable stream") + }); + if sender.send(item).await.is_err() { + break; + } + } + }); + Ok((fragment.size_content, streamed)) } /// Read the given data range from a fragment which can be a large data set diff --git a/lore-revision/src/lib.rs b/lore-revision/src/lib.rs index db4d6f69f..d9a3ccff5 100644 --- a/lore-revision/src/lib.rs +++ b/lore-revision/src/lib.rs @@ -34,17 +34,20 @@ pub use lore_base; pub mod lore; pub mod merge; pub mod merge_carry; +pub mod merge_resolution; pub mod metadata; pub mod nametable; pub mod node; pub mod notification; pub mod path; +pub mod path_merge; pub mod progress; pub mod proto; pub mod protocol; pub mod relay; pub mod repository; pub mod revision; +pub mod revision_tree; pub mod runtime; pub mod stage; pub mod state; diff --git a/lore-revision/src/merge_resolution.rs b/lore-revision/src/merge_resolution.rs new file mode 100644 index 000000000..33cf2411c --- /dev/null +++ b/lore-revision/src/merge_resolution.rs @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT + +//! Stable merge-conflict identities and whole-side conflict resolution. +//! +//! This module is deliberately independent of gRPC. Callers can compute a +//! [`MergePlan`] from Lore's normal three-way [`DiffResult`], present its +//! stable conflict ids through any UI, then resolve each conflict to the +//! target or source side before applying the returned changes to the target +//! state. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +use lore_base::types::Hash; + +use crate::change::Flags; +use crate::change::NodeChange; +use crate::revision::DiffResult; + +/// Complete side to retain for a conflict. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConflictSide { + /// Preserve the target state. Because merge changes are applied onto the + /// target tree, this requires no tree edit. + Target, + /// Apply the source-side change to the target tree. + Source, +} + +/// One conflict in a three-way merge plan. +#[derive(Clone, Debug)] +pub struct MergeConflict { + pub id: Hash, + pub source: NodeChange, + pub target: NodeChange, +} + +/// A reusable, immutable description of a three-way merge. +#[derive(Debug)] +pub struct MergePlan { + pub base: Hash, + pub source: Hash, + pub target: Hash, + pub changes: Vec, + pub conflicts: Vec, +} + +/// Result of applying a partial set of whole-side decisions to a plan. +#[derive(Debug)] +pub struct ResolvedMerge { + /// Conflict-free edits to apply to the target tree. + pub changes: Vec, + /// Conflicts that still require a decision. + pub unresolved: Vec, + /// Supplied ids that do not occur in this merge plan. + pub unknown_resolution_ids: Vec, +} + +impl MergePlan { + /// Attach stable ids to a three-way diff result. + pub fn new(result: DiffResult) -> Self { + // revision::diff3 also emits target-only changes so presentation + // callers can describe the complete joined diff. A merge is applied + // onto that target already, so only source-side edits belong in the + // executable plan. + let changes = result + .changes + .into_iter() + .filter(|change| change.to.state.revision() == result.source) + .collect(); + let conflicts = result + .conflicts + .into_iter() + .map(|(source, target)| MergeConflict { + id: conflict_id(result.base, result.source, result.target, &source, &target), + source, + target, + }) + .collect(); + Self { + base: result.base, + source: result.source, + target: result.target, + changes, + conflicts, + } + } + + /// Apply any supplied target/source choices. A target choice is omitted + /// from `changes`, because the edits are applied to the target state. A + /// source choice contributes the source-side change with merge-conflict + /// flags removed. + pub fn resolve(&self, resolutions: &BTreeMap) -> ResolvedMerge { + let known: BTreeSet = self.conflicts.iter().map(|conflict| conflict.id).collect(); + let mut changes = self.changes.clone(); + let mut unresolved = Vec::new(); + + for conflict in &self.conflicts { + match resolutions.get(&conflict.id) { + Some(ConflictSide::Target) => {} + Some(ConflictSide::Source) => { + let mut source = conflict.source.clone(); + source.flags = if source.flags.contains(Flags::Modify) { + Flags::Modify + } else { + Flags::None + }; + changes.push(source); + } + None => unresolved.push(conflict.clone()), + } + } + + let unknown_resolution_ids = resolutions + .keys() + .filter(|id| !known.contains(id)) + .copied() + .collect(); + ResolvedMerge { + changes, + unresolved, + unknown_resolution_ids, + } + } +} + +/// Compute a stable id for a conflict under pinned base/source/target tips. +/// Stream-local partition indices and node ids are intentionally excluded. +pub fn conflict_id( + base: Hash, + source_revision: Hash, + target_revision: Hash, + source: &NodeChange, + target: &NodeChange, +) -> Hash { + let mut bytes = Vec::with_capacity(256); + bytes.extend_from_slice(b"lore.merge-conflict.v1\0"); + bytes.extend_from_slice(base.data()); + bytes.extend_from_slice(source_revision.data()); + bytes.extend_from_slice(target_revision.data()); + append_change(&mut bytes, source); + append_change(&mut bytes, target); + Hash::hash_buffer(&bytes) +} + +fn append_change(bytes: &mut Vec, change: &NodeChange) { + bytes.extend_from_slice(&(change.action as u16).to_le_bytes()); + append_string(bytes, change.path.as_str()); + match change.from_path.as_ref() { + Some(path) => { + bytes.push(1); + append_string(bytes, path.as_str()); + } + None => bytes.push(0), + } + append_state(bytes, &change.from); + append_state(bytes, &change.to); +} + +fn append_state(bytes: &mut Vec, state: &crate::change::NodeChangeState) { + bytes.extend_from_slice(state.repository.id.data()); + bytes.extend_from_slice(state.address.hash.data()); + bytes.extend_from_slice(state.address.context.data()); + bytes.extend_from_slice(&state.flags.bits().to_le_bytes()); +} + +fn append_string(bytes: &mut Vec, value: &str) { + bytes.extend_from_slice(&(value.len() as u64).to_le_bytes()); + bytes.extend_from_slice(value.as_bytes()); +} diff --git a/lore-revision/src/path_merge.rs b/lore-revision/src/path_merge.rs new file mode 100644 index 000000000..0f353cf4b --- /dev/null +++ b/lore-revision/src/path_merge.rs @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT + +use crate::change::NodeChange; +use crate::util::path::RelativePath; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum PathMergeStrategy { + #[default] + Merge, + KeepTarget, + Exclude, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct PathMergeRule { + /// Repository-relative path the rule applies to. A rule matches the path + /// itself and descendants below it. + pub path: RelativePath, + pub strategy: PathMergeStrategy, +} + +#[derive(Clone, Copy, Debug)] +pub struct PathMergePolicy<'a> { + rules: &'a [PathMergeRule], +} + +impl Default for PathMergePolicy<'_> { + fn default() -> Self { + Self { rules: &[] } + } +} + +impl<'a> PathMergePolicy<'a> { + pub fn new(rules: &'a [PathMergeRule]) -> Self { + Self { rules } + } + + pub fn is_empty(self) -> bool { + self.rules.is_empty() + } + + pub fn strategy_for_change(self, change: &NodeChange) -> PathMergeStrategy { + self.rules + .iter() + .enumerate() + .filter(|(_, rule)| path_merge_rule_matches_change(rule, change)) + .max_by_key(|(index, rule)| (rule.path.len(), *index)) + .map(|(_, rule)| rule.strategy) + .unwrap_or_default() + } + + pub fn strategy_for_conflict( + self, + source_change: &NodeChange, + target_change: &NodeChange, + ) -> PathMergeStrategy { + self.rules + .iter() + .enumerate() + .filter(|(_, rule)| { + path_merge_rule_matches_change(rule, source_change) + || path_merge_rule_matches_change(rule, target_change) + }) + .max_by_key(|(index, rule)| (rule.path.len(), *index)) + .map(|(_, rule)| rule.strategy) + .unwrap_or_default() + } + + pub fn should_merge_change(self, change: &NodeChange) -> bool { + self.strategy_for_change(change) == PathMergeStrategy::Merge + } + + pub fn should_merge_conflict( + self, + source_change: &NodeChange, + target_change: &NodeChange, + ) -> bool { + self.strategy_for_conflict(source_change, target_change) == PathMergeStrategy::Merge + } +} + +fn path_merge_rule_matches_change(rule: &PathMergeRule, change: &NodeChange) -> bool { + rule.path.overlaps(&change.path) + || change + .from_path + .as_ref() + .is_some_and(|from_path| rule.path.overlaps(from_path)) +} diff --git a/lore-revision/src/repository.rs b/lore-revision/src/repository.rs index 50da51129..6c59f9bfa 100644 --- a/lore-revision/src/repository.rs +++ b/lore-revision/src/repository.rs @@ -1304,7 +1304,7 @@ pub enum RepositoryAccess { } pub const DOT_URC: &str = ".urc"; -pub const DOT_LORE: &str = ".lore"; +pub const DOT_LORE: &str = ".spacesync"; pub const ID: &str = "id"; pub const INSTANCE: &str = "instance"; pub const CONFIG: &str = "config.toml"; diff --git a/lore-revision/src/repository/clone.rs b/lore-revision/src/repository/clone.rs index e98613ced..1ba6bbad0 100644 --- a/lore-revision/src/repository/clone.rs +++ b/lore-revision/src/repository/clone.rs @@ -332,6 +332,24 @@ pub struct CloneWorkItem { pub repository_path: RepositoryPath, } +async fn cache_repository_metadata( + repository: Arc, + metadata: &repository::RepositoryMetadata, + expected_hash: Hash, +) -> Result<(), CloneError> { + let cached_hash = repository::metadata_store(repository.clone(), metadata.clone()) + .await + .forward::("Failed to cache repository metadata")?; + if cached_hash != expected_hash { + return Err(CloneError::internal(format!( + "Cloned repository metadata hash mismatch: expected {expected_hash}, got {cached_hash}" + ))); + } + repository::metadata_store_hash(repository, expected_hash) + .await + .forward::("Failed to cache repository metadata hash") +} + /// Shared context for dependency-driven discovery across all block workers. struct DependencyDiscoverContext { /// Tags to filter dependency edges by. Empty means follow all edges. @@ -895,6 +913,7 @@ pub async fn clone( file: Some(FileConfig::default()), }; + let repository_metadata_hash = repository_data.metadata; let repository_metadata = { // Dummy repository context just to be able to load the repository // metadata from the remote @@ -918,7 +937,7 @@ pub async fn clone( repository.set_disable_upload(true); - repository::metadata(repository, repository_data.metadata) + repository::metadata(repository, repository_metadata_hash) .await .forward::("Repository not found")? }; @@ -1025,6 +1044,18 @@ pub async fn clone( repository.set_disable_upload(true); + // Clone initially deserializes repository metadata through a temporary + // remote-backed context. Persist the same content-addressed fragment and + // mutable pointer in the real local context so later branch pushes can + // collect every repository fragment without requiring another remote + // metadata read. + cache_repository_metadata( + repository.clone(), + &repository_metadata, + repository_metadata_hash, + ) + .await?; + // Prune stale instances in the background when using a shared store. // AbortOnDropHandle ensures the task is cancelled if clone fails early. let prune_task = if options @@ -2178,6 +2209,72 @@ mod tests { RelativePath::new_from_initial_path(path).expect("Relative path") } + async fn metadata_test_context(path: &Path) -> Arc { + let (immutable, mutable) = repository::create_client_memory_stores() + .await + .expect("create in-memory stores"); + let token = RepositoryWriteToken::acquire(path).await; + Arc::new( + RepositoryContext::new(RepositoryContextCreationArgs { + path: Some(path.to_path_buf()), + immutable_store: immutable, + mutable_store: mutable, + id: RepositoryId::default(), + instance_id: crate::instance::InstanceId::default(), + remote: Err(lore_transport::ProtocolError::from(crate::errors::NoRemote)), + filter: Arc::default(), + format: RepositoryFormat::Lore, + filesystem_provider: None, + }) + .with_write_token(token), + ) + } + + #[tokio::test] + async fn clone_metadata_cache_is_available_without_a_remote() { + let execution = Arc::new(crate::interface::ExecutionContext::new_client_with_user_id( + crate::interface::LoreGlobalArgs::default(), + crate::relay::EventDispatcher::no_dispatch(), + "test-user".to_owned(), + )); + lore_base::runtime::LORE_CONTEXT + .scope(execution, async { + let source_dir = tempfile::tempdir().expect("source temp dir"); + let source = metadata_test_context(source_dir.path()).await; + let metadata = repository::RepositoryMetadata { + name: "clone-metadata-test".to_owned(), + description: "cached for later pushes".to_owned(), + default_branch: Default::default(), + default_branch_name: "main".to_owned(), + creator: "test-user".to_owned(), + created: 42, + }; + let expected_hash = repository::metadata_store(source, metadata.clone()) + .await + .expect("serialize source metadata"); + + let clone_dir = tempfile::tempdir().expect("clone temp dir"); + let cloned = metadata_test_context(clone_dir.path()).await; + cache_repository_metadata(cloned.clone(), &metadata, expected_hash) + .await + .expect("cache cloned metadata"); + + assert_eq!( + repository::metadata_hash(cloned.clone()) + .await + .expect("read cached metadata hash"), + expected_hash + ); + assert_eq!( + repository::metadata(cloned, expected_hash) + .await + .expect("deserialize cached metadata"), + metadata + ); + }) + .await; + } + #[tokio::test] async fn ensure_parent_dir_creates_missing_ancestors() { let (temp, operation) = create_operation().await; diff --git a/lore-revision/src/repository/create.rs b/lore-revision/src/repository/create.rs index aa96ad2c3..5a84f55d0 100644 --- a/lore-revision/src/repository/create.rs +++ b/lore-revision/src/repository/create.rs @@ -113,6 +113,8 @@ pub struct CreateOptions { pub id: Option, // Repository description pub description: Option, + // Name of the default branch + pub default_branch_name: Option, // Whether to use the shared store and options configuring it if desired pub shared_store_options: Option, } @@ -172,8 +174,16 @@ pub async fn create_with_metadata( let id = options.id.unwrap_or_else(|| uuid::Uuid::now_v7().into()); - // TODO(mjansson): Make this configurable in arguments and command line - let branch_name = branch::DEFAULT_DEFAULT_NAME; + let branch_name = options + .default_branch_name + .as_deref() + .filter(|name| !name.is_empty()) + .unwrap_or(branch::DEFAULT_DEFAULT_NAME); + if !branch::is_valid_name(branch_name) { + return Err(CreateError::from(InvalidArguments { + reason: format!("invalid default branch name: {branch_name}"), + })); + } // Use the hash of the default name for the main branch ID to make it // easily distinguishable in logs even in the ID form. let branch = if branch_name == branch::DEFAULT_DEFAULT_NAME { diff --git a/lore-revision/src/revision.rs b/lore-revision/src/revision.rs index c3340c18e..8f4a167dc 100644 --- a/lore-revision/src/revision.rs +++ b/lore-revision/src/revision.rs @@ -44,6 +44,8 @@ use crate::metadata::Metadata; use crate::node::NodeFileMetadata; use crate::node::NodeFileMetadataBlock; use crate::node::NodeIDExt; +use crate::path_merge::PathMergePolicy; +use crate::path_merge::PathMergeRule; use crate::repository::RepositoryContext; use crate::state; use crate::state::State; @@ -216,6 +218,18 @@ fn filter_from_source_changes(source_changes: &[NodeChange]) -> Option { Some(filter) } +async fn state_has_path( + repository: Arc, + state: Arc, + path: &RelativePath, +) -> Result { + match state.find_node_link(repository, path.as_str()).await { + Ok(link) => Ok(link.is_valid_or_root()), + Err(err) if err.is_node_not_found() => Ok(false), + Err(err) => Err(err), + } +} + /// Threshold above which `diff3_collect` skips building a source-derived /// path filter for target's walk. Tuned to bound filter-construction /// cost; above this the unfiltered walk is cheaper. @@ -243,8 +257,8 @@ const SOURCE_FILTER_THRESHOLD: usize = 10_000; /// (`Move + from_path`, directory-delete overlap, history-walk /// resolution) before emitting. History-walk runs in parallel via /// a semaphore-bounded `JoinSet` (see the -/// `history_walk_concurrency` parameter of -/// `diff3_with_source_cap`, defaulting to +/// `history_walk_concurrency` option of +/// `diff3_with_options`, defaulting to /// `DEFAULT_HISTORY_WALK_CONCURRENCY`). /// /// Returns a `Diff3Summary` carrying the base / source / target @@ -259,16 +273,16 @@ pub async fn diff3( include_same: bool, tx: mpsc::Sender>, ) -> Result { - diff3_with_source_cap( + diff3_with_options( repository, base, source, target, - path, - include_same, - None, - None, - None, + Diff3Options { + path, + include_same, + ..Default::default() + }, tx, ) .await @@ -282,31 +296,47 @@ pub async fn diff3( /// deserialised revision blob). pub const DEFAULT_HISTORY_WALK_CONCURRENCY: usize = 24; +/// Optional behavior for streaming revision 3-way diffs. +#[derive(Clone, Debug)] +pub struct Diff3Options<'a> { + pub path: Option, + pub include_same: bool, + /// Abort with `StateError::Oversized` when source's diff produces more + /// than this many items. `None` keeps the diff unbounded. + pub source_cap: Option, + /// Permit count for the parallel history-walk semaphore. `None` falls back + /// to `DEFAULT_HISTORY_WALK_CONCURRENCY`. + pub history_walk_concurrency: Option, + /// Ordered per-path merge rules used to suppress matching merge inputs. + pub path_merge_rules: &'a [PathMergeRule], + /// View used to identify untouched out-of-view subtrees that can be grafted. + /// `None` disables graft adoption. + pub graft_view: Option>, +} + +impl Default for Diff3Options<'_> { + fn default() -> Self { + Self { + path: None, + include_same: false, + source_cap: None, + history_walk_concurrency: None, + path_merge_rules: &[], + graft_view: None, + } + } +} + /// `diff3` with optional tunables. -/// -/// * `source_cap` — abort with `StateError::Oversized` when source's -/// `diff_collect` produces more than `n` items. The error message -/// includes the cap and the produced count. Bounds peak memory for -/// callers that need a ceiling. Library callers (filesystem diff, -/// merge, capi, CLI) pass `None` via `diff3` to stay unbounded. -/// * `history_walk_concurrency` — permit count for the semaphore -/// gating parallel `is_last_change_merged` history walks. Passing -/// `None` falls back to `DEFAULT_HISTORY_WALK_CONCURRENCY`. -#[allow(clippy::too_many_arguments)] -pub async fn diff3_with_source_cap( +pub async fn diff3_with_options( repository: Arc, base: Hash, source: Hash, target: Hash, - path: Option, - include_same: bool, - source_cap: Option, - history_walk_concurrency: Option, - // Decides which subtrees the view excludes and may be adopted whole. - // `None` disables adoption. - graft_view: Option>, + options: Diff3Options<'_>, tx: mpsc::Sender>, ) -> Result { + let path_merge_policy = PathMergePolicy::new(options.path_merge_rules); let (state_base, state_source, state_target) = join!( State::deserialize(repository.clone(), base), State::deserialize(repository.clone(), source), @@ -344,10 +374,10 @@ pub async fn diff3_with_source_cap( let source_walker_repo = repository.clone(); let source_walker_state_base = state_base.clone(); let source_walker_state_source = state_source.clone(); - let source_walker_path = path.clone(); + let source_walker_path = options.path.clone(); // Give the source walk the target tree, so it can adopt whole subtrees // the target never touched instead of descending. - let source_walker_graft = graft_view.map(|view| { + let source_walker_graft = options.graft_view.clone().map(|view| { Arc::new(state::GraftOracle::new( repository.clone(), state_target.clone(), @@ -386,8 +416,11 @@ pub async fn diff3_with_source_cap( if is_file_id_only_churn { continue; } + if !path_merge_policy.should_merge_change(&change) { + continue; + } source_changes.push(change); - if let Some(cap) = source_cap + if let Some(cap) = options.source_cap && source_changes.len() > cap { oversized = true; @@ -404,7 +437,7 @@ pub async fn diff3_with_source_cap( return Err(StateError::from(Oversized { context: format!( "source-side diff change count exceeds configured limit of {}", - source_cap.unwrap_or(0) + options.source_cap.unwrap_or(0) ), })); } @@ -423,7 +456,20 @@ pub async fn diff3_with_source_cap( lore_info!("Sorting {} source changes", source_changes.len()); change::sort_by_path(&mut source_changes); - let target_filter = if source_changes.len() < SOURCE_FILTER_THRESHOLD + let target_paths = if !path_merge_policy.is_empty() { + let mut paths = Vec::with_capacity(source_changes.len()); + for change in &source_changes { + paths.push(change.path.clone()); + if let Some(from_path) = change.from_path.as_ref() { + paths.push(from_path.clone()); + } + } + Some(RelativePath::dedup_to_supersets(paths)) + } else { + None + }; + let target_filter = if path_merge_policy.is_empty() + && source_changes.len() < SOURCE_FILTER_THRESHOLD && let Some(filter) = filter_from_source_changes(&source_changes) { Arc::new(filter) @@ -436,21 +482,48 @@ pub async fn diff3_with_source_cap( let (target_tx, mut target_rx) = mpsc::channel::>(256); let walker_repo = target_repository.clone(); let walker_state_base = state_base.clone(); - let walker_path = path.clone(); + let walker_state_target = state_target.clone(); + let walker_path = options.path.clone(); + let walker_paths = target_paths.clone(); let walker = lore_spawn!(async move { let mut sink = state::ChangeSink::Channel(&target_tx); - state::diff( - walker_repo.clone(), - walker_state_base, - walker_repo, - state_target, - walker_path, - // Adoption is a source-side decision. - None, - &mut sink, - FilterMode::View, - ) - .await + if let Some(paths) = walker_paths { + for path in paths { + let exists_in_base = + state_has_path(walker_repo.clone(), walker_state_base.clone(), &path).await?; + let exists_in_target = + state_has_path(walker_repo.clone(), walker_state_target.clone(), &path).await?; + if !exists_in_base && !exists_in_target { + continue; + } + Box::pin(state::diff( + walker_repo.clone(), + walker_state_base.clone(), + walker_repo.clone(), + walker_state_target.clone(), + Some(path), + // Adoption is a source-side decision. + None, + &mut sink, + FilterMode::View, + )) + .await?; + } + Ok(()) + } else { + state::diff( + walker_repo.clone(), + walker_state_base, + walker_repo, + walker_state_target, + walker_path, + // Adoption is a source-side decision. + None, + &mut sink, + FilterMode::View, + ) + .await + } }); // Join loop: target streams in; source lives in the sorted Vec. @@ -471,6 +544,9 @@ pub async fn diff3_with_source_cap( if is_file_id_only_churn { continue; } + if !path_merge_policy.should_merge_change(&target_change) { + continue; + } match source_changes.binary_search_by(|c| c.path.as_str().cmp(target_change.path.as_str())) { Ok(idx) => { @@ -481,7 +557,7 @@ pub async fn diff3_with_source_cap( sc.flags = change::Flags::Conflict; target_change.flags = change::Flags::Conflict; joined_conflicts.push((sc, target_change)); - } else if include_same + } else if options.include_same && (joined_changes.is_empty() || joined_changes[joined_changes.len() - 1].path != source_change.path) { @@ -493,7 +569,6 @@ pub async fn diff3_with_source_cap( } } } - match walker.await { Ok(Ok(())) => {} Ok(Err(err)) => return Err(err), @@ -537,7 +612,9 @@ pub async fn diff3_with_source_cap( // code is designed to tolerate today (e.g. a task aborted mid-write // to a downstream channel could surface a partial value), so let // them run to completion. - let permits = history_walk_concurrency.unwrap_or(DEFAULT_HISTORY_WALK_CONCURRENCY); + let permits = options + .history_walk_concurrency + .unwrap_or(DEFAULT_HISTORY_WALK_CONCURRENCY); let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(permits)); let mut history_walks: JoinSet> = JoinSet::new(); let mut outcomes: Vec = Vec::with_capacity(joined_conflicts.len()); @@ -672,13 +749,6 @@ pub async fn diff3_with_source_cap( Ok(summary) } -/// Resolves `Move + other-change-at-from-path` interactions per the -/// pre-streaming rules: -/// - `Move(X→Y) + Delete(X)` → conflict (divergent move). -/// - Pure rename `Move(X→Y, content unchanged) + Modify(X)` → -/// absorb the modify into the rename (the rename carries the -/// modified content). -/// - `Move(X→Y, content changed) + Modify(X)` → conflict (both /// branches modified content differently). fn apply_move_from_path_pass( changes: &mut Vec, diff --git a/lore-revision/src/revision/cherry_pick.rs b/lore-revision/src/revision/cherry_pick.rs index 45fc08141..7e0feed9e 100644 --- a/lore-revision/src/revision/cherry_pick.rs +++ b/lore-revision/src/revision/cherry_pick.rs @@ -209,6 +209,7 @@ pub async fn cherry_pick( diff, state_current.clone(), MergeType::CherryPick, + &[], false, current_branch, ) diff --git a/lore-revision/src/revision/revert.rs b/lore-revision/src/revision/revert.rs index ad42ffa09..de59e4e0a 100644 --- a/lore-revision/src/revision/revert.rs +++ b/lore-revision/src/revision/revert.rs @@ -209,6 +209,7 @@ pub async fn revert( diff, state_current.clone(), MergeType::Revert, + &[], false, current_branch, ) diff --git a/lore-revision/src/revision/sync.rs b/lore-revision/src/revision/sync.rs index 6dc1af771..12df06c23 100644 --- a/lore-revision/src/revision/sync.rs +++ b/lore-revision/src/revision/sync.rs @@ -534,6 +534,7 @@ pub async fn sync( message: String::new(), no_commit: false, scope: merge::MergeScope::MainOnly, + path_merge_rules: Vec::new(), }; let revision_staged = Box::pin(merge::merge_start( repository.clone(), diff --git a/lore-revision/src/revision_tree.rs b/lore-revision/src/revision_tree.rs new file mode 100644 index 000000000..c663f079f --- /dev/null +++ b/lore-revision/src/revision_tree.rs @@ -0,0 +1,694 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +//! Working-tree-free revision tree edits shared by the SDK handle verbs and +//! server-side changeset handlers. + +use std::pin::Pin; +use std::sync::Arc; + +use lore_base::error::InvalidArguments; +use lore_base::types::Address; +use lore_error_set::prelude::*; +use lore_storage::hash::hash_string; + +use crate::errors::StateErrors; +use crate::event::EventError; +use crate::interface::LoreError; +use crate::interface::LoreNodeType; +use crate::node::Node; +use crate::node::NodeBlock; +use crate::node::NodeDelta; +use crate::node::NodeFlags; +use crate::node::NodeID; +use crate::node::NodeIDExt; +use crate::node::ROOT_NODE; +use crate::node::SiblingCycleGuard; +use crate::repository::RepositoryContext; +use crate::state; +use crate::state::State; + +#[error_set] +pub enum RevisionTreeEditError { + InvalidArguments, +} + +impl EventError for RevisionTreeEditError { + fn translated(&self) -> LoreError { + match self { + RevisionTreeEditError::InvalidArguments(_) => LoreError::InvalidArguments, + RevisionTreeEditError::Internal(_) => LoreError::Internal, + } + } + + fn inner(&self) -> String { + self.to_string() + } +} + +fn invalid(reason: impl Into) -> RevisionTreeEditError { + RevisionTreeEditError::from(InvalidArguments { + reason: reason.into(), + }) +} + +/// Add a file, empty directory, or link beneath `parent_node_id` and mark it +/// staged. `kind` uses the public [`LoreNodeType`] numeric encoding. +pub async fn add_node( + state: Arc, + repository: Arc, + parent_node_id: NodeID, + name: &[u8], + kind: u32, + mode: u16, + size: u64, + address: Address, +) -> Result { + if !parent_node_id.is_valid_or_root_node_id() { + return Err(invalid("parent node id is invalid")); + } + + let name = std::str::from_utf8(name).map_err(|_| invalid("name is not valid UTF-8"))?; + if name.is_empty() { + return Err(invalid("name is empty")); + } + if name.contains('/') || name.contains('\\') { + return Err(invalid("name must not contain path separators")); + } + + let flags = if kind == LoreNodeType::File as u32 { + NodeFlags::File + } else if kind == LoreNodeType::Directory as u32 { + NodeFlags::NoFlags + } else if kind == LoreNodeType::Link as u32 { + NodeFlags::Link + } else { + return Err(invalid("kind is not a valid node type")); + }; + + if kind == LoreNodeType::Directory as u32 + && (size != 0 || !address.hash.is_zero() || !address.context.is_zero()) + { + return Err(invalid( + "a directory takes no size or address; both are computed at commit", + )); + } + if kind == LoreNodeType::Link as u32 { + if size != 0 { + return Err(invalid("a link takes no size")); + } + if address.hash.is_zero() { + return Err(invalid("a link requires a target revision in address.hash")); + } + } + + let parent = state + .node(repository.clone(), parent_node_id) + .await + .map_err(|_| invalid("parent node id is unknown"))?; + if parent.is_discarded() { + return Err(invalid("parent node id resolves to a deleted node")); + } + if !parent.is_directory() { + return Err(invalid( + "parent node is not a directory in the handle's own tree", + )); + } + if parent_node_id != ROOT_NODE { + let parent_name = state + .node_name_clone(repository.clone(), parent_node_id) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "State::node_name_clone") + })?; + if parent_name.is_empty() { + return Err(invalid("parent node id does not resolve to a named node")); + } + } + + let name_hash = hash_string(name); + match state + .find_subnode(repository.clone(), parent_node_id, name_hash) + .await + { + Ok(_) => return Err(invalid("name already exists under the parent")), + Err(error) if error.is_node_not_found() => {} + Err(error) => { + return Err(RevisionTreeEditError::internal_with_context( + error, + "State::find_subnode", + )); + } + } + + let node = Node { + flags: flags.bits(), + name_hash, + mode, + size, + address, + ..Default::default() + }; + let node_id = state + .node_add(repository.clone(), parent_node_id, node, name) + .await + .map_err(|error| RevisionTreeEditError::internal_with_context(error, "State::node_add"))?; + state + .node_mark(repository, node_id, NodeFlags::StagedAdd, true) + .await + .map_err(|error| RevisionTreeEditError::internal_with_context(error, "State::node_mark"))?; + Ok(node_id) +} + +/// Create exactly one empty directory. Missing ancestors are not created. +pub async fn create_directory( + state: Arc, + repository: Arc, + parent_node_id: NodeID, + name: &[u8], + mode: u16, +) -> Result { + add_node( + state, + repository, + parent_node_id, + name, + LoreNodeType::Directory as u32, + mode, + 0, + Address::default(), + ) + .await +} + +/// Replace a file node's content hash, mode, and logical size while preserving +/// its existing file identity context. +pub async fn modify_file( + state: Arc, + repository: Arc, + node_id: NodeID, + mode: u16, + size: u64, + address: Address, +) -> Result { + if !node_id.is_valid_node_id() { + return Err(invalid("node id is invalid")); + } + + let block_index = NodeBlock::index(node_id); + let node_index = Node::index(node_id); + let block = state + .block(repository.clone(), block_index) + .await + .map_err(|_| invalid("node id is unknown"))?; + let node = block.node(node_index); + if node.is_discarded() { + return Err(invalid("node id resolves to a deleted node")); + } + if !node.is_file() { + return Err(invalid("node is not a leaf (file) node")); + } + if node.name_hash == 0 { + return Err(invalid("node id does not resolve to a named node")); + } + if !address.context.is_zero() && address.context != node.address.context { + return Err(invalid("address context does not match the node's file id")); + } + + let file_id = node.address.context; + let mark_flags = if node.is_staged_add() { + NodeFlags::StagedAdd + } else { + NodeFlags::StagedModify + }; + let dirtied = { + let mut block_writer = block.write(); + let node = block_writer.node(node_index); + node.address.hash = address.hash; + node.address.context = file_id; + node.mode = mode; + node.size = size; + block_writer.mark_dirty() + }; + if dirtied { + state.block_modified(block, block_index); + state.mark_dirty(); + } + state + .node_mark(repository, node_id, mark_flags, true) + .await + .map_err(|error| RevisionTreeEditError::internal_with_context(error, "State::node_mark"))?; + Ok(node_id) +} + +fn mark_delete_subtree( + state: Arc, + repository: Arc, + node_id: NodeID, +) -> Pin> + Send>> { + Box::pin(async move { + let node = state.node(repository.clone(), node_id).await?; + if node.is_staged_delete() { + return Ok(()); + } + state + .node_mark(repository.clone(), node_id, NodeFlags::StagedDelete, true) + .await?; + if node.is_directory() { + let mut child_node_iter = node.child(); + let mut cycle = SiblingCycleGuard::new(node_id); + while let Some(child_node_id) = child_node_iter { + mark_delete_subtree(state.clone(), repository.clone(), child_node_id).await?; + let child_node = state.node(repository.clone(), child_node_id).await?; + child_node.walk_step(child_node_id, node_id, &mut cycle)?; + child_node_iter = child_node.sibling(); + } + } + Ok(()) + }) +} + +/// Delete a node and its in-tree descendants, returning the delta entries that +/// must be carried into `commit_tree` after the nodes are discarded. +pub async fn delete_node( + state: Arc, + repository: Arc, + node_id: NodeID, +) -> Result, RevisionTreeEditError> { + if !node_id.is_valid_node_id() { + return Err(invalid("node id is invalid (the root cannot be deleted)")); + } + let node = state + .node(repository.clone(), node_id) + .await + .map_err(|_| invalid("node id is unknown"))?; + if node.name_hash == 0 { + return Err(invalid("node id does not resolve to a named node")); + } + if node.is_discarded() { + return Err(invalid("node id resolves to a deleted node")); + } + + mark_delete_subtree(state.clone(), repository.clone(), node_id) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "mark delete subtree") + })?; + + let deltas = Arc::new(parking_lot::RwLock::new(Vec::new())); + let recorder = deltas.clone(); + let handler = move |discarded_node_id: NodeID, flags: u16| { + recorder + .write() + .push(NodeDelta::from_node_and_flags(discarded_node_id, flags)); + }; + + if node.is_directory() { + let mut child_node_iter = node.child(); + let mut cycle = SiblingCycleGuard::new(node_id); + while let Some(child_node_id) = child_node_iter { + let child_node = state + .node(repository.clone(), child_node_id) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "read child node") + })?; + child_node + .walk_step(child_node_id, node_id, &mut cycle) + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "walk child chain") + })?; + let next_sibling = child_node.sibling(); + state::node_discard_nopatch( + state.clone(), + repository.clone(), + child_node_id, + true, + true, + handler.clone(), + ) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "discard subtree node") + })?; + child_node_iter = next_sibling; + } + } + + state::node_discard_patch(state, repository, node_id, handler) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "discard deleted node") + })?; + let result = deltas.read().clone(); + Ok(result) +} + +fn mark_children_moved( + state: Arc, + repository: Arc, + parent_node: NodeID, + move_flag: NodeFlags, +) -> Pin> + Send>> { + Box::pin(async move { + let children = state.node_children(repository.clone(), parent_node).await?; + for child_id in children { + let child_node = state.node(repository.clone(), child_id).await?; + let child_flag = if child_node.is_staged_add() { + NodeFlags::StagedAdd + } else { + move_flag + }; + state + .node_mark(repository.clone(), child_id, child_flag, false) + .await?; + if child_node.is_directory() { + mark_children_moved(state.clone(), repository.clone(), child_id, move_flag).await?; + } + } + Ok(()) + }) +} + +/// Reparent and/or rename a node while preserving its file identity. +pub async fn move_node( + state: Arc, + repository: Arc, + node_id: NodeID, + destination_parent_id: NodeID, + dst_name: &[u8], +) -> Result { + if !node_id.is_valid_node_id() { + return Err(invalid("node id is invalid (the root cannot be moved)")); + } + if !destination_parent_id.is_valid_or_root_node_id() { + return Err(invalid("destination parent node id is invalid")); + } + let dst_name = std::str::from_utf8(dst_name) + .map_err(|_| invalid("destination name is not valid UTF-8"))?; + if dst_name.is_empty() { + return Err(invalid("destination name is empty")); + } + if dst_name.contains('/') || dst_name.contains('\\') { + return Err(invalid("destination name must not contain path separators")); + } + + let node = state + .node(repository.clone(), node_id) + .await + .map_err(|_| invalid("node id is unknown"))?; + if node.name_hash == 0 { + return Err(invalid("node id does not resolve to a named node")); + } + if node.is_discarded() { + return Err(invalid("node id resolves to a deleted node")); + } + + let destination = state + .node(repository.clone(), destination_parent_id) + .await + .map_err(|_| invalid("destination parent node id is unknown"))?; + if destination.is_discarded() { + return Err(invalid("destination parent id resolves to a deleted node")); + } + if !destination.is_directory() { + return Err(invalid( + "destination parent is not a directory in the handle's own tree", + )); + } + if destination_parent_id != ROOT_NODE { + let name = state + .node_name_clone(repository.clone(), destination_parent_id) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "State::node_name_clone") + })?; + if name.is_empty() { + return Err(invalid( + "destination parent id does not resolve to a named node", + )); + } + } + + let mut ancestor = destination_parent_id; + while ancestor.is_valid_node_id() { + if ancestor == node_id { + return Err(invalid("destination lives inside the moved subtree")); + } + let ancestor_node = state + .node(repository.clone(), ancestor) + .await + .map_err(|_| invalid("destination parent chain is unreadable"))?; + ancestor = ancestor_node.parent; + } + + let dst_name_hash = hash_string(dst_name); + if node.parent == destination_parent_id && node.name_hash == dst_name_hash { + return Err(invalid("source and destination are the same")); + } + match state + .find_subnode(repository.clone(), destination_parent_id, dst_name_hash) + .await + { + Ok(existing) if existing != node_id => { + return Err(invalid("destination name already exists under the parent")); + } + Ok(_) => {} + Err(error) if error.is_node_not_found() => {} + Err(error) => { + return Err(RevisionTreeEditError::internal_with_context( + error, + "State::find_subnode", + )); + } + } + + let block_index = NodeBlock::index(node_id); + let node_index = Node::index(node_id); + let block = state + .block(repository.clone(), block_index) + .await + .map_err(|error| RevisionTreeEditError::internal_with_context(error, "read node block"))?; + let mut node = block.node(node_index); + + if node.parent != destination_parent_id { + let old_parent_id = node.parent; + let old_parent_block_index = NodeBlock::index(old_parent_id); + let old_parent_node_index = Node::index(old_parent_id); + let old_parent_block = state + .block(repository.clone(), old_parent_block_index) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "read old parent block") + })?; + let old_parent = old_parent_block.node(old_parent_node_index); + if old_parent.child == node_id { + let dirtied = { + let mut block_writer = old_parent_block.write(); + block_writer.node(old_parent_node_index).child = node.sibling; + block_writer.mark_dirty() + }; + if dirtied { + state.block_modified(old_parent_block, old_parent_block_index); + state.mark_dirty(); + } + } else { + let mut found = false; + let mut child_id = old_parent.child().unwrap_or_default(); + let mut cycle = SiblingCycleGuard::new(old_parent_id); + while child_id.is_valid_node_id() { + let child = state + .node(repository.clone(), child_id) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context( + error, + "walk old sibling chain", + ) + })?; + child + .walk_step(child_id, old_parent_id, &mut cycle) + .map_err(|error| { + RevisionTreeEditError::internal_with_context( + error, + "walk old sibling chain", + ) + })?; + let Some(sibling) = child.sibling() else { + break; + }; + if sibling == node_id { + let child_block_index = NodeBlock::index(child_id); + let child_node_index = Node::index(child_id); + let child_block = state + .block(repository.clone(), child_block_index) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context( + error, + "read sibling block", + ) + })?; + let dirtied = { + let mut block_writer = child_block.write(); + block_writer.node(child_node_index).sibling = node.sibling; + block_writer.mark_dirty() + }; + if dirtied { + state.block_modified(child_block, child_block_index); + state.mark_dirty(); + } + found = true; + break; + } + child_id = sibling; + } + if !found { + return Err(RevisionTreeEditError::internal( + "node not found in its parent's child chain", + )); + } + } + + let dst_block_index = NodeBlock::index(destination_parent_id); + let dst_node_index = Node::index(destination_parent_id); + let dst_block = state + .block(repository.clone(), dst_block_index) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "read destination block") + })?; + let sibling_node_id = dst_block.node(dst_node_index).child; + let dirtied = { + let mut block_writer = dst_block.write(); + block_writer.node(dst_node_index).child = node_id; + block_writer.mark_dirty() + }; + if dirtied { + state.block_modified(dst_block, dst_block_index); + state.mark_dirty(); + } + node.sibling = sibling_node_id; + } + + node.parent = destination_parent_id; + if node.name_hash != dst_name_hash { + block + .deserialize_nametable(repository.clone()) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "deserialize name table") + })?; + node.name_hash = dst_name_hash; + let (name_offset, name_length) = { + let mut block_writer = block.write(); + block_writer.node_name_store(dst_name, node.name_offset, node.name_length) + } + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "store renamed node name") + })?; + node.name_offset = name_offset; + node.name_length = name_length; + } + + let dirtied = { + let mut block_writer = block.write(); + *block_writer.node(node_index) = node; + block_writer.mark_dirty() + }; + if dirtied { + state.block_modified(block, block_index); + state.mark_dirty(); + } + + let move_flag = if node.is_staged_add() { + NodeFlags::StagedAdd + } else { + NodeFlags::StagedMove + }; + state + .node_mark(repository.clone(), node_id, move_flag, true) + .await + .map_err(|error| RevisionTreeEditError::internal_with_context(error, "State::node_mark"))?; + if node.is_directory() { + mark_children_moved(state, repository, node_id, move_flag) + .await + .map_err(|error| { + RevisionTreeEditError::internal_with_context(error, "mark children moved") + })?; + } + Ok(node_id) +} + +#[cfg(test)] +mod tests { + use lore_base::types::Context; + use lore_base::types::Partition; + use lore_storage::immutable_store::ImmutableStore; + use lore_storage::local::immutable_store::ImmutableStoreCreateOptions; + use lore_storage::local::immutable_store::ImmutableStoreSettings; + use lore_storage::local::immutable_store::create as create_immutable; + use lore_storage::local::mutable_store::LocalMutableStore; + use lore_storage::local::mutable_store::MutableStoreSettings; + use lore_storage::mutable_store::MutableStore; + + use super::*; + use crate::node::INVALID_NODE; + + async fn fixture() -> (Arc, Arc) { + let immutable = create_immutable( + Option::::None, + ImmutableStoreCreateOptions::none(), + false, + ImmutableStoreSettings::default(), + ) + .await + .expect("in-memory immutable store"); + let mutable: Arc = Arc::new( + LocalMutableStore::new( + Option::<&std::path::Path>::None, + MutableStoreSettings::default(), + immutable.clone(), + ) + .await + .expect("in-memory mutable store"), + ); + let immutable: Arc = immutable; + let repository = Arc::new(RepositoryContext::new_server_context( + immutable, + mutable, + Partition::from(Context::from([0x44; 16])), + )); + let state = State::deserialize(repository.clone(), Default::default()) + .await + .expect("empty state"); + (state, repository) + } + + #[tokio::test] + async fn create_directory_adds_a_staged_empty_node() { + let (state, repository) = fixture().await; + let node_id = create_directory( + state.clone(), + repository.clone(), + ROOT_NODE, + b"empty", + 0o755, + ) + .await + .expect("create directory"); + let node = state + .node(repository, node_id) + .await + .expect("directory node"); + assert!(node.is_directory()); + assert!(node.is_staged_add()); + assert!(!node.child.is_valid_node_id()); + assert_eq!(node.size, 0); + assert_eq!(node.address, Address::default()); + } + + #[test] + fn invalid_node_constant_remains_reserved() { + assert!(!INVALID_NODE.is_valid_or_root_node_id()); + } +} diff --git a/lore-revision/src/state.rs b/lore-revision/src/state.rs index 0ad26dffc..20126c4a3 100644 --- a/lore-revision/src/state.rs +++ b/lore-revision/src/state.rs @@ -5,6 +5,7 @@ pub mod dump; mod sink; use core::str; +use std::collections::HashSet; use std::future::Future; use std::io::Write; use std::mem::size_of; @@ -4660,6 +4661,7 @@ pub struct TreePath { pub path: RelativePath, pub address: Option
, pub flags: NodeFlags, + pub last_changed_revision: Option, pub size: u64, pub mode: u64, /// True when a link node tracks its parent's branch; false for pinned @@ -4718,6 +4720,7 @@ pub async fn gather_tree_paths( 0, max_depth, 0, + None, can_read, &mut paths, ) @@ -4734,6 +4737,7 @@ async fn enumerate_children( depth: usize, max_depth: usize, link_depth: usize, + inherited_last_changed_revision: Option, can_read: CanReadRepository, result: &mut Vec, ) -> Result<(), StateError> { @@ -4747,6 +4751,21 @@ async fn enumerate_children( } .into()); } + let changed_nodes = if inherited_last_changed_revision.is_some() { + Arc::new(HashSet::new()) + } else { + let delta = state + .delta_block(repository.clone()) + .await? + .to_aligned::(); + Arc::new( + delta + .as_type_slice::() + .iter() + .map(|entry| entry.node) + .collect(), + ) + }; let mut cycle = SiblingCycleGuard::new(parent_node_id); gather_tree_paths_node_recurse( state, @@ -4757,6 +4776,8 @@ async fn enumerate_children( depth, max_depth, link_depth, + changed_nodes, + inherited_last_changed_revision, can_read, result, &mut cycle, @@ -4794,6 +4815,8 @@ async fn gather_tree_paths_node( depth: usize, max_depth: usize, link_depth: usize, + changed_nodes: Arc>, + inherited_last_changed_revision: Option, can_read: CanReadRepository, result: &mut Vec, cycle: &mut SiblingCycleGuard, @@ -4826,6 +4849,30 @@ async fn gather_tree_paths_node( } else { NodeFlags::NoFlags }; + let last_changed_revision = if node.is_directory() { + None + } else if let Some(revision) = inherited_last_changed_revision { + Some(revision) + } else if changed_nodes.contains(&node_id) { + Some(state.revision()) + } else { + let metadata_node = node::node_to_file_metadata(node_id); + let metadata_block = state + .block_file_metadata( + repository.clone(), + NodeFileMetadataBlock::index(metadata_node), + ) + .await?; + let revision = metadata_block + .read() + .node(NodeFileMetadata::index(metadata_node)) + .revision[0]; + Some(if revision.is_zero() { + state.revision() + } else { + revision + }) + }; // An unresolvable link reference falls back to pinned. let tracking = if node.is_link() { let link = node.linked_node(); @@ -4840,6 +4887,7 @@ async fn gather_tree_paths_node( path: node_path.clone(), address, flags, + last_changed_revision, size: node.size, mode: node.mode as u64, tracking, @@ -4857,6 +4905,8 @@ async fn gather_tree_paths_node( depth + 1, max_depth, link_depth, + changed_nodes.clone(), + inherited_last_changed_revision, can_read, result, &mut child_cycle, @@ -4881,6 +4931,7 @@ async fn gather_tree_paths_node( depth + 1, max_depth, link_depth + 1, + last_changed_revision, can_read, result, ) @@ -4919,6 +4970,8 @@ fn gather_tree_paths_node_recurse<'a>( depth: usize, max_depth: usize, link_depth: usize, + changed_nodes: Arc>, + inherited_last_changed_revision: Option, can_read: CanReadRepository, result: &'a mut Vec, cycle: &'a mut SiblingCycleGuard, @@ -4935,6 +4988,8 @@ fn gather_tree_paths_node_recurse<'a>( depth, max_depth, link_depth, + changed_nodes.clone(), + inherited_last_changed_revision, can_read.clone(), result, cycle, @@ -5725,6 +5780,10 @@ pub async fn diff( .await .unwrap_or(NodeLink::invalid()); + if !from_link.is_valid_or_root() && !to_link.is_valid_or_root() { + return Ok(()); + } + let mut repository_from = repository_from; let state_from = if !from_link.repository.is_zero() && from_link.repository != repository_from.id @@ -9416,7 +9475,31 @@ pub async fn apply_tree_changes( target_state: Arc, changes: &[NodeChange], ) -> Result<(), StateError> { + apply_tree_changes_inner(repository, target_state, changes, false) + .await + .map(|_| ()) +} + +/// Commit-ready variant of [`apply_tree_changes`]. Deleted and moved-away +/// nodes are discarded from the target tree and returned as delta entries, +/// allowing [`crate::commit::construct_merge_revision`] to freeze and rehash +/// the resulting tree into a normal immutable revision. +pub async fn apply_tree_changes_for_commit( + repository: Arc, + target_state: Arc, + changes: &[NodeChange], +) -> Result, StateError> { + apply_tree_changes_inner(repository, target_state, changes, true).await +} + +async fn apply_tree_changes_inner( + repository: Arc, + target_state: Arc, + changes: &[NodeChange], + discard_deletes: bool, +) -> Result, StateError> { let stats = Arc::new(crate::stage::StageStats::default()); + let mut deleted = Vec::new(); // Process deletes first, in reverse path order (deepest paths first) so that // children are deleted before parent directories @@ -9436,7 +9519,19 @@ pub async fn apply_tree_changes( Err(err) => return Err(err), }; - if node_link.is_valid() { + if node_link.is_valid() && discard_deletes { + deleted.extend( + crate::revision_tree::delete_node( + target_state.clone(), + repository.clone(), + node_link.node, + ) + .await + .map_err(|error| { + StateError::internal_with_context(error, "discarding merge deletion") + })?, + ); + } else if node_link.is_valid() { crate::stage::stage_delete( repository.clone(), target_state.clone(), @@ -9469,7 +9564,19 @@ pub async fn apply_tree_changes( Err(err) => return Err(err), }; - if node_link.is_valid() { + if node_link.is_valid() && discard_deletes { + deleted.extend( + crate::revision_tree::delete_node( + target_state.clone(), + repository.clone(), + node_link.node, + ) + .await + .map_err(|error| { + StateError::internal_with_context(error, "discarding merge move source") + })?, + ); + } else if node_link.is_valid() { crate::stage::stage_delete( repository.clone(), target_state.clone(), @@ -9508,7 +9615,7 @@ pub async fn apply_tree_changes( .forward::("Node not found")?; } - Ok(()) + Ok(deleted) } #[cfg(test)] diff --git a/lore-revision/tests/diff3_streaming.rs b/lore-revision/tests/diff3_streaming.rs index 5f0fd197d..8d4f8a3f6 100644 --- a/lore-revision/tests/diff3_streaming.rs +++ b/lore-revision/tests/diff3_streaming.rs @@ -489,6 +489,7 @@ mod tests { message: "merge main into feature (sparse view)".to_string(), no_commit: false, scope: lore_revision::branch::merge::MergeScope::MainOnly, + path_merge_rules: Vec::new(), }, ) .await @@ -592,6 +593,7 @@ mod tests { message: message.to_string(), no_commit: false, scope: lore_revision::branch::merge::MergeScope::MainOnly, + path_merge_rules: Vec::new(), }, ) .await @@ -862,7 +864,7 @@ mod tests { /// walk runs. /// /// Build a fixture where source produces more changes than the - /// configured cap allows. The streaming `diff3_with_source_cap` + /// configured cap allows. The streaming `diff3_with_options` /// must error out with `BranchError::Oversized` before any target /// work happens. The v1 handler uses `is_oversized()` to map the /// failure to `Status::resource_exhausted`. @@ -903,25 +905,26 @@ mod tests { fixture.write_file("target_only.txt", b"target\n"); let target_revision = fixture.stage_and_commit("target add 1").await; - // Drive the streaming branch::diff3_with_source_cap + // Drive the streaming branch::diff3_with_options // with a cap below source's 3-change count. The cap // fires inside revision::diff3 and the error wraps as // BranchError::Diff at the branch layer. let (tx, mut rx) = tokio::sync::mpsc::channel::< Result, >(8); - let producer = Box::pin(branch::diff3_with_source_cap( + let producer = Box::pin(branch::diff3_with_options( fixture.repository.clone(), source_branch, source_revision, target_branch, target_revision, - None, - false, // include_same - false, // auto_resolve - Some(1), - None, // history_walk_concurrency: default - None, // graft_view: no grafting + branch::Diff3Options { + include_same: false, + auto_resolve: false, + source_cap: Some(1), + history_walk_concurrency: None, + ..Default::default() + }, tx, )); // Drain any items the producer emits before erroring. @@ -1223,6 +1226,7 @@ mod tests { message: "merge target into source".to_string(), no_commit: false, scope: lore_revision::branch::merge::MergeScope::MainOnly, + path_merge_rules: Vec::new(), }, ) .await diff --git a/lore-revision/tests/immutable.rs b/lore-revision/tests/immutable.rs index 2396bf6b0..44ccefc65 100644 --- a/lore-revision/tests/immutable.rs +++ b/lore-revision/tests/immutable.rs @@ -1149,8 +1149,7 @@ mod tests { .expect("Failed writing to store"); // Read via stream and collect all buffers - let (tx, mut rx) = - tokio::sync::mpsc::channel::>(64); + let (tx, mut rx) = tokio::sync::mpsc::channel(64); let options = immutable::read_options_from_repository(&repository); let content_length = immutable::read_stream(repository.clone(), address, None, options, tx) @@ -1167,6 +1166,32 @@ mod tests { assert_eq!(reassembled.len(), payload.len()); assert_eq!(reassembled.as_slice(), payload.as_ref()); + + // A cross-fragment range streams only the selected logical + // bytes, including partial first/last leaves. + let requested = 12_345u64..1_876_543u64; + let (tx, mut rx) = tokio::sync::mpsc::channel(64); + let options = immutable::read_options_from_repository(&repository); + let (full_length, normalized) = immutable::read_stream_range( + repository.clone(), + address, + Some(requested.clone()), + options, + tx, + ) + .await + .expect("read_stream_range failed"); + assert_eq!(full_length, payload_size as u64); + assert_eq!(normalized, requested); + + let mut ranged = Vec::new(); + while let Some(chunk) = rx.recv().await { + ranged.extend_from_slice(chunk.expect("range stream item failed").as_ref()); + } + assert_eq!( + ranged.as_slice(), + &payload[requested.start as usize..requested.end as usize] + ); }) .await; } diff --git a/lore-revision/tests/repository_format.rs b/lore-revision/tests/repository_format.rs index 26d20f428..466450e4e 100644 --- a/lore-revision/tests/repository_format.rs +++ b/lore-revision/tests/repository_format.rs @@ -51,7 +51,7 @@ mod tests { fn detect_lore_directory() { let dir = std::env::temp_dir().join("lore-test-detect-lore"); let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(dir.join(".lore")).expect("create .lore dir"); + std::fs::create_dir_all(dir.join(".spacesync")).expect("create .lore dir"); let format = RepositoryFormat::detect(&dir); assert!(matches!(format, RepositoryFormat::Lore)); @@ -76,7 +76,7 @@ mod tests { let dir = std::env::temp_dir().join("lore-test-detect-both"); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(dir.join(".urc")).expect("create .urc dir"); - std::fs::create_dir_all(dir.join(".lore")).expect("create .lore dir"); + std::fs::create_dir_all(dir.join(".spacesync")).expect("create .lore dir"); let format = RepositoryFormat::detect(&dir); assert!(matches!(format, RepositoryFormat::Urc)); @@ -103,12 +103,12 @@ mod tests { let _ = std::fs::remove_dir_all(&base); let nested = base.join("a").join("b"); std::fs::create_dir_all(&nested).expect("create nested dirs"); - std::fs::create_dir_all(base.join(".lore")).expect("create .lore dir"); + std::fs::create_dir_all(base.join(".spacesync")).expect("create .lore dir"); // Walk up from nested, looking for .lore or .urc let mut current = nested.as_path(); let found = loop { - if current.join(".urc").is_dir() || current.join(".lore").is_dir() { + if current.join(".urc").is_dir() || current.join(".spacesync").is_dir() { break Some(current.to_path_buf()); } match current.parent() { @@ -131,7 +131,7 @@ mod tests { let mut current = nested.as_path(); let found = loop { - if current.join(".urc").is_dir() || current.join(".lore").is_dir() { + if current.join(".urc").is_dir() || current.join(".spacesync").is_dir() { break Some(current.to_path_buf()); } match current.parent() { @@ -150,7 +150,7 @@ mod tests { let dir = std::env::temp_dir().join("lore-test-ignore-fallback"); let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(dir.join(".lore")).expect("create .lore dir"); + std::fs::create_dir_all(dir.join(".spacesync")).expect("create .lore dir"); // Write a .urcignore with a pattern (no .loreignore present) std::fs::write(dir.join(".urcignore"), "secret.txt\n").expect("write .urcignore"); @@ -170,7 +170,7 @@ mod tests { let dir = std::env::temp_dir().join("lore-test-ignore-prefer"); let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(dir.join(".lore")).expect("create .lore dir"); + std::fs::create_dir_all(dir.join(".spacesync")).expect("create .lore dir"); // Both files present — .loreignore should win std::fs::write(dir.join(".loreignore"), "a.txt\nb.txt\n").expect("write .loreignore"); diff --git a/lore-server/Cargo.toml b/lore-server/Cargo.toml index 5dd743360..46ef54e26 100644 --- a/lore-server/Cargo.toml +++ b/lore-server/Cargo.toml @@ -18,6 +18,7 @@ bytes = { workspace = true } clap = { workspace = true, features = ["env"] } config = { workspace = true } dashmap = { workspace = true } +diffy = { workspace = true } enum_dispatch = { workspace = true } futures = { workspace = true } hex = { workspace = true } @@ -27,6 +28,8 @@ opentelemetry = { workspace = true } opentelemetry-semantic-conventions = { workspace = true } parking_lot = { workspace = true } pin-project = { workspace = true } +postgres = { workspace = true } +prost = { workspace = true } prost-types = { workspace = true } quinn = { workspace = true } rand = { workspace = true } @@ -43,7 +46,7 @@ sysinfo = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-metrics = { workspace = true } -tokio-util = { workspace = true } +tokio-util = { workspace = true, features = ["io"] } tokio-stream = { workspace = true } toml = { workspace = true } tonic = { workspace = true } diff --git a/lore-server/config/dev-local.toml b/lore-server/config/dev-local.toml index c86409dd5..067293c9f 100644 --- a/lore-server/config/dev-local.toml +++ b/lore-server/config/dev-local.toml @@ -29,14 +29,14 @@ mode = "aws" # TLS — use certs baked into the Docker image # ============================================================================= -[server.quic.certificate] -cert_file = "/etc/lore/certs/server.crt" -pkey_file = "/etc/lore/certs/server.key" +# [server.quic.certificate] +# cert_file = "/etc/lore/certs/server.crt" +# pkey_file = "/etc/lore/certs/server.key" -[server.quic_internal.certificate] -cert_file = "/etc/lore/certs/server.crt" -pkey_file = "/etc/lore/certs/server.key" -cert_chain = "/etc/lore/certs/ca.crt" +# [server.quic_internal.certificate] +# cert_file = "/etc/lore/certs/server.crt" +# pkey_file = "/etc/lore/certs/server.key" +# cert_chain = "/etc/lore/certs/ca.crt" # Increase handler timeouts for slow emulated storage [server.quic] @@ -58,8 +58,8 @@ presigned_url_hmac_key = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbe # ============================================================================= [plugins.aws.immutable_store] -s3_bucket = "lore-immutable-store-dev" -s3_endpoint_url = "http://minio:9000" +s3_bucket = "lore-dev" +s3_endpoint_url = "http://127.0.0.1:9000" s3_region = "us-east-1" s3_force_path_style = true dynamodb_fragments_table = "lore-fragments-dev" @@ -72,13 +72,13 @@ timeout_millis = 120000 [plugins.aws.mutable_store] dynamodb_table = "lore-mutable-store-dev" -dynamodb_endpoint_url = "http://dynamodb:9090" +dynamodb_endpoint_url = "https://127.0.0.1:8000" dynamodb_region = "us-east-1" timeout_millis = 120000 [plugins.aws.lock_store] dynamodb_table = "lore-locks-dev" -dynamodb_endpoint_url = "http://dynamodb:9090" +dynamodb_endpoint_url = "https://127.0.0.1:8000" dynamodb_region = "us-east-1" timeout_millis = 120000 @@ -89,17 +89,17 @@ timeout_millis = 120000 [telemetry] additional_labels = { region = "local" } -[telemetry.exporter] -endpoint = "http://grafana:4317" -queue_size = 8192 -timeout = 5000 +# [telemetry.exporter] +# endpoint = "http://grafana:4317" +# queue_size = 8192 +# timeout = 5000 [telemetry.logger] enable_otlp = false format = "json" [telemetry.logger.output] -file = "/var/log/lore/server.log" +file = "./server.log" [telemetry.traces] sample_rate = 1.0 diff --git a/lore-server/config/local.toml b/lore-server/config/local.toml index ebe4c4d68..9b83844d4 100644 --- a/lore-server/config/local.toml +++ b/lore-server/config/local.toml @@ -63,6 +63,7 @@ flush_delay_seconds = 10 # immutable_store.s3_bucket = "lore-immutable-store-local" # immutable_store.s3_endpoint_url = "http://127.0.0.1:9000" # immutable_store.s3_slow_operation_threshold_millis = 2000 +# immutable_store.direct_downloads_enabled = false # force clients through the Lore server # immutable_store.dynamodb_endpoint_url = "http://127.0.0.1:9090" # immutable_store.dynamodb_fragments_table = "lore-fragments-local" # immutable_store.dynamodb_metadata_table = "lore-fragment-metadata-local" diff --git a/lore-server/src/grpc/handlers/repository_create.rs b/lore-server/src/grpc/handlers/repository_create.rs index ad3988d33..a0d290450 100644 --- a/lore-server/src/grpc/handlers/repository_create.rs +++ b/lore-server/src/grpc/handlers/repository_create.rs @@ -144,6 +144,21 @@ fn validate_create_input( Ok(()) } +pub(crate) fn ensure_default_branch_name_matches( + repository_id: RepositoryId, + existing_default_branch_name: &str, + requested_default_branch_name: &str, +) -> Result<(), Status> { + if existing_default_branch_name == requested_default_branch_name { + return Ok(()); + } + + Err(Status::already_exists(format!( + "Repository {repository_id} already exist with default branch \ + {existing_default_branch_name} which does not match {requested_default_branch_name}" + ))) +} + #[allow(clippy::too_many_arguments)] async fn repository_create( repository: Arc, @@ -181,6 +196,12 @@ async fn repository_create( .await { return if data.name == name { + ensure_default_branch_name_matches( + data.id, + data.default_branch_name.as_str(), + default_branch_name, + )?; + info!( "Repository {} already exist with name {}, early out create successful", repository.id, data.name @@ -222,6 +243,12 @@ async fn repository_create( .await { return if data.id == repository.id { + ensure_default_branch_name_matches( + data.id, + data.default_branch_name.as_str(), + default_branch_name, + )?; + info!( "Repository {} already exist with id {}, early out create successful", name, data.id @@ -311,6 +338,7 @@ async fn repository_create( Ok(RepositoryData { id: repository.id, name: name.to_string(), + default_branch_name: default_branch_name.to_string(), metadata, }) } @@ -442,6 +470,26 @@ mod tests { } } + mod retry_idempotency { + use super::*; + + #[test] + fn accepts_matching_default_branch_name() { + ensure_default_branch_name_matches(RepositoryId::default(), "main", "main") + .expect("matching branch names should pass"); + } + + #[test] + fn rejects_mismatched_default_branch_name() { + let err = ensure_default_branch_name_matches(RepositoryId::default(), "main", "trunk") + .expect_err("mismatched branch names should reject retry"); + + assert_eq!(err.code(), Code::AlreadyExists); + assert!(err.message().contains("default branch main")); + assert!(err.message().contains("does not match trunk")); + } + } + mod repository_create_auth_resource_tests { use lore_proto::rebac::CreateResourceResponse; use lore_proto::rebac::DeleteResourceRequest; diff --git a/lore-server/src/grpc/handlers/repository_query.rs b/lore-server/src/grpc/handlers/repository_query.rs index d5fbd3420..a627df359 100644 --- a/lore-server/src/grpc/handlers/repository_query.rs +++ b/lore-server/src/grpc/handlers/repository_query.rs @@ -142,6 +142,7 @@ pub async fn repository_query_id( Ok(RepositoryData { id, name: metadata.name, + default_branch_name: metadata.default_branch_name, metadata: metadata_hash, }) } @@ -200,6 +201,7 @@ pub async fn repository_query_name( Ok(RepositoryData { id, name: metadata.name, + default_branch_name: metadata.default_branch_name, metadata: metadata_hash, }) } diff --git a/lore-server/src/grpc/repository/v1/repository_create.rs b/lore-server/src/grpc/repository/v1/repository_create.rs index 04c0c4c85..28045fd7c 100644 --- a/lore-server/src/grpc/repository/v1/repository_create.rs +++ b/lore-server/src/grpc/repository/v1/repository_create.rs @@ -31,6 +31,7 @@ use crate::grpc::forwarded_requests::CallerContext; use crate::grpc::forwarded_requests::ForwardedRequests; use crate::grpc::get_user_id; use crate::grpc::get_write_token; +use crate::grpc::handlers::repository_create::ensure_default_branch_name_matches; use crate::grpc::handlers::repository_create::repository_create_auth_resource; use crate::grpc::hook_error_to_status; use crate::grpc::warn_error_to_status; @@ -247,6 +248,12 @@ async fn repository_create_inner( repository_load_id(repository.clone(), repository.id, None, None).await { return if metadata.name == name { + ensure_default_branch_name_matches( + repository.id, + metadata.default_branch_name.as_str(), + default_branch_name, + )?; + info!( "Repository {} already exist with name {}, early out create successful", repository.id, metadata.name @@ -277,6 +284,12 @@ async fn repository_create_inner( repository_load_name(repository.clone(), name, None, None).await { return if id == repository.id { + ensure_default_branch_name_matches( + id, + metadata.default_branch_name.as_str(), + default_branch_name, + )?; + info!( "Repository {} already exist with id {}, early out create successful", name, id diff --git a/lore-server/src/grpc/revision/v1/branch_merge.rs b/lore-server/src/grpc/revision/v1/branch_merge.rs new file mode 100644 index 000000000..aa9848bd6 --- /dev/null +++ b/lore-server/src/grpc/revision/v1/branch_merge.rs @@ -0,0 +1,903 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT + +//! Atomic, server-authored branch merges for the revision v1 API. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::sync::Arc; + +use lore_base::runtime::LORE_CONTEXT; +use lore_base::types::BranchId; +use lore_base::types::Context; +use lore_base::types::Hash; +use lore_proto::lore::revision::v1 as revision_v1; +use lore_proto::lore::revision::v1::branch_merge_resolution::Resolution; +use lore_revision::branch; +use lore_revision::branch::Diff3Options; +use lore_revision::merge_resolution::ConflictSide; +use lore_revision::merge_resolution::MergePlan; +use lore_revision::metadata; +use lore_revision::metadata::Metadata; +use lore_revision::notification::NotificationSender; +use lore_revision::repository::RepositoryContext; +use lore_revision::state; +use lore_revision::state::State; +use lore_telemetry::InstrumentProvider; +use prost::Message; +use tonic::Request; +use tonic::Response; +use tonic::Status; + +use super::branch_push::publish_revision; +use super::idempotency; +use crate::grpc::extract_correlation_id; +use crate::grpc::get_authorization; +use crate::grpc::get_repository; +use crate::grpc::get_user_id; +use crate::grpc::get_write_token; +use crate::grpc::handlers::branch_push::extract_client_ip; +use crate::grpc::server::RevisionListAcceleration; +use crate::hooks::HookDispatcher; +use crate::util::setup_execution; + +const MAX_REQUEST_BYTES: usize = 4 * 1024 * 1024; +const MAX_RESOLUTIONS: usize = 100_000; +const MAX_SOURCE_CHANGES: usize = 100_000; + +enum Attempt { + Published(revision_v1::BranchMergeResponse), + NotPublished(revision_v1::BranchMergeResponse), +} + +#[allow(clippy::too_many_arguments)] +#[tracing::instrument(name = "BranchMerge::v1::handle", skip_all)] +pub async fn handler( + request: Request, + immutable_store: Arc, + mutable_store: Arc, + notification: Arc, + hook_dispatcher: &HookDispatcher, + history_step_size: u64, + acceleration: RevisionListAcceleration, + instrument_provider: &impl InstrumentProvider, +) -> Result, Status> { + let repository_id = get_repository(request.metadata())?; + let correlation_id = extract_correlation_id(&request).unwrap_or_default(); + let user_id = get_user_id(request.extensions()); + let bypass_protection = get_authorization(request.extensions()) + .ok() + .and_then(|token| token.is_service_account) + .unwrap_or_default(); + let client_ip = extract_client_ip(&request).map(|value| value.to_string()); + let req = request.into_inner(); + + if req.encoded_len() > MAX_REQUEST_BYTES { + return Err(Status::resource_exhausted(format!( + "BranchMerge encoded request byte limit exceeded: {} > {MAX_REQUEST_BYTES}", + req.encoded_len() + ))); + } + if req.resolutions.len() > MAX_RESOLUTIONS { + return Err(Status::resource_exhausted(format!( + "BranchMerge resolution count limit exceeded: {} > {MAX_RESOLUTIONS}", + req.resolutions.len() + ))); + } + + let request_id = Context::from(uuid_v7(req.request_id.as_ref(), "request_id")?); + let target_branch = branch_id(req.branch_id_target.as_ref(), "branch_id_target")?; + let source_branch = branch_id(req.branch_id_source.as_ref(), "branch_id_source")?; + if target_branch == source_branch { + return Err(Status::invalid_argument( + "branch_id_source and branch_id_target must differ", + )); + } + let expected_target = nonzero_hash( + req.revision_signature_target.as_ref(), + "revision_signature_target", + )?; + let expected_source = nonzero_hash( + req.revision_signature_source.as_ref(), + "revision_signature_source", + )?; + let resolutions = parse_resolutions(&req.resolutions)?; + let request_digest = Hash::hash_buffer(req.encode_to_vec().as_slice()); + + let reservation = match idempotency::begin( + "BranchMerge", + immutable_store.clone(), + mutable_store.clone(), + repository_id, + target_branch, + request_id, + request_digest, + ) + .await? + { + idempotency::Start::Completed { + revision, + revision_number, + } => { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store, + mutable_store, + repository_id, + )); + let base = resolve_base( + repository, + source_branch, + expected_source, + target_branch, + expected_target, + ) + .await?; + return Ok(Response::new(revision_v1::BranchMergeResponse { + outcome: revision_v1::BranchMergeOutcome::Merged as i32, + revision_signature: revision.into(), + revision_number, + revision_signature_base: base.into(), + unresolved_conflicts: Vec::new(), + })); + } + idempotency::Start::Reserved(reservation) => reservation, + }; + + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository_id, + )); + let execution = setup_execution(module_path!(), correlation_id.clone(), user_id.clone()); + let outcome = LORE_CONTEXT + .scope(execution, async { + ensure_tip(repository.clone(), target_branch, expected_target, "target").await?; + ensure_tip(repository.clone(), source_branch, expected_source, "source").await?; + + let diff = branch::diff3_collect_with_options( + repository.clone(), + source_branch, + expected_source, + target_branch, + expected_target, + Diff3Options { + auto_resolve: false, + source_cap: Some(MAX_SOURCE_CHANGES), + ..Default::default() + }, + ) + .await + .map_err(branch_error)?; + if diff.base.is_zero() { + return Err(Status::failed_precondition( + "BranchMerge: source and target have no common ancestor", + )); + } + + let plan = MergePlan::new(diff); + let resolved = plan.resolve(&resolutions); + if !resolved.unknown_resolution_ids.is_empty() { + return Err(Status::invalid_argument(format!( + "resolution contains {} conflict id(s) that do not belong to this merge", + resolved.unknown_resolution_ids.len() + ))); + } + + let target_state = load_state(repository.clone(), expected_target, "target").await?; + if !resolved.unresolved.is_empty() { + return Ok(Attempt::NotPublished(revision_v1::BranchMergeResponse { + outcome: revision_v1::BranchMergeOutcome::Conflicted as i32, + revision_signature: expected_target.into(), + revision_number: target_state.revision_number(), + revision_signature_base: plan.base.into(), + unresolved_conflicts: resolved + .unresolved + .iter() + .map(|conflict| revision_v1::BranchMergeConflict { + conflict_id: conflict.id.into(), + path_source: conflict.source.path.to_string(), + path_target: conflict.target.path.to_string(), + }) + .collect(), + })); + } + + if plan.changes.is_empty() && plan.conflicts.is_empty() { + return Ok(Attempt::NotPublished(revision_v1::BranchMergeResponse { + outcome: revision_v1::BranchMergeOutcome::AlreadyUpToDate as i32, + revision_signature: expected_target.into(), + revision_number: target_state.revision_number(), + revision_signature_base: plan.base.into(), + unresolved_conflicts: Vec::new(), + })); + } + + // A source branch can advance independently while the immutable + // merge plan is being computed. Honor both optimistic guards at + // the final publication boundary. + ensure_tip(repository.clone(), source_branch, expected_source, "source").await?; + ensure_tip(repository.clone(), target_branch, expected_target, "target").await?; + + let deleted = state::apply_tree_changes_for_commit( + repository.clone(), + target_state.clone(), + &resolved.changes, + ) + .await + .map_err(|error| { + Status::internal(format!("failed to apply merge tree changes: {error}")) + })?; + + let mut pending_metadata = if target_state.metadata_hash().is_zero() { + Metadata::new() + } else { + Metadata::deserialize(repository.clone(), target_state.metadata_hash()) + .await + .map_err(|error| { + Status::internal(format!("failed to load target metadata: {error}")) + })? + }; + pending_metadata + .set_string( + metadata::MERGED_BY, + if user_id.is_empty() { + "server" + } else { + &user_id + }, + ) + .map_err(|error| { + Status::internal(format!("failed to set merge metadata: {error}")) + })?; + pending_metadata + .set_string(metadata::MESSAGE, &req.commit_message) + .map_err(|error| { + Status::internal(format!("failed to set merge message: {error}")) + })?; + let revision = lore_revision::commit::construct_merge_revision( + repository.clone(), + &get_write_token(), + target_state, + pending_metadata, + target_branch, + expected_source, + deleted, + ) + .await + .map_err(|error| { + Status::internal(format!("failed to construct merge revision: {error}")) + })?; + let published = publish_revision( + repository, + target_branch, + revision, + bypass_protection, + false, + false, + client_ip, + correlation_id, + user_id, + notification, + hook_dispatcher, + history_step_size, + acceleration, + instrument_provider, + ) + .await? + .into_inner(); + + Ok(Attempt::Published(revision_v1::BranchMergeResponse { + outcome: revision_v1::BranchMergeOutcome::Merged as i32, + revision_signature: published.revision_signature, + revision_number: published.revision_number, + revision_signature_base: plan.base.into(), + unresolved_conflicts: Vec::new(), + })) + }) + .await; + + match outcome { + Ok(Attempt::Published(response)) => { + idempotency::complete( + immutable_store, + mutable_store, + repository_id, + &reservation, + nonzero_hash(response.revision_signature.as_ref(), "published revision")?, + response.revision_number, + ) + .await?; + Ok(Response::new(response)) + } + Ok(Attempt::NotPublished(response)) => { + idempotency::release(mutable_store, repository_id, &reservation).await; + Ok(Response::new(response)) + } + Err(status) => { + idempotency::release(mutable_store, repository_id, &reservation).await; + Err(status) + } + } +} + +fn parse_resolutions( + values: &[revision_v1::BranchMergeResolution], +) -> Result, Status> { + let mut result = BTreeMap::new(); + let mut duplicate_ids = BTreeSet::new(); + for value in values { + let id = nonzero_hash(value.conflict_id.as_ref(), "resolution.conflict_id")?; + let side = match value.resolution { + Some(Resolution::Side(value)) => match revision_v1::BranchMergeSide::try_from(value) { + Ok(revision_v1::BranchMergeSide::Target) => ConflictSide::Target, + Ok(revision_v1::BranchMergeSide::Source) => ConflictSide::Source, + _ => { + return Err(Status::invalid_argument( + "resolution.side must be TARGET or SOURCE", + )); + } + }, + None => { + return Err(Status::invalid_argument( + "BranchMergeResolution.resolution must be set", + )); + } + }; + if result.insert(id, side).is_some() { + duplicate_ids.insert(id); + } + } + if !duplicate_ids.is_empty() { + return Err(Status::invalid_argument(format!( + "duplicate resolution for {} conflict id(s)", + duplicate_ids.len() + ))); + } + Ok(result) +} + +async fn ensure_tip( + repository: Arc, + branch_id: BranchId, + expected: Hash, + label: &str, +) -> Result<(), Status> { + let current = branch::load_latest(repository, branch_id) + .await + .map_err(|error| Status::not_found(format!("{label} branch was not found: {error}")))?; + if current != expected { + return Err(Status::failed_precondition(format!( + "{label} branch advanced; current latest: {current}" + ))); + } + Ok(()) +} + +async fn resolve_base( + repository: Arc, + source_branch: BranchId, + source: Hash, + target_branch: BranchId, + target: Hash, +) -> Result { + let base = branch::resolve_diff3_base(repository, source_branch, source, target_branch, target) + .await + .map_err(branch_error)?; + if base.is_zero() { + return Err(Status::failed_precondition( + "BranchMerge: source and target have no common ancestor", + )); + } + Ok(base) +} + +async fn load_state( + repository: Arc, + revision: Hash, + label: &str, +) -> Result, Status> { + State::deserialize(repository, revision) + .await + .map_err(|error| { + if error.is_address_not_found() || error.is_payload_not_found() || error.is_not_found() + { + Status::not_found(format!("{label} revision was not found: {error}")) + } else { + Status::internal(format!("failed to load {label} revision: {error}")) + } + }) +} + +fn branch_error(error: branch::BranchError) -> Status { + if error.is_divergent() { + Status::failed_precondition(error.to_string()) + } else if error.is_max_history_search_depth() || error.is_oversized() { + Status::resource_exhausted(error.to_string()) + } else { + Status::internal(error.to_string()) + } +} + +fn uuid_v7(bytes: &[u8], field: &str) -> Result { + let value = uuid::Uuid::from_slice(bytes) + .map_err(|_| Status::invalid_argument(format!("{field} must be exactly 16 UUID bytes")))?; + if value.get_version_num() != 7 { + return Err(Status::invalid_argument(format!( + "{field} must be a UUIDv7" + ))); + } + Ok(value) +} + +fn branch_id(bytes: &[u8], field: &str) -> Result { + if bytes.len() != 16 { + return Err(Status::invalid_argument(format!( + "{field} must be exactly 16 bytes" + ))); + } + let value = BranchId::from(bytes); + if value.is_zero() { + return Err(Status::invalid_argument(format!( + "{field} must be non-zero" + ))); + } + Ok(value) +} + +fn nonzero_hash(bytes: &[u8], field: &str) -> Result { + if bytes.len() != 32 { + return Err(Status::invalid_argument(format!( + "{field} must be exactly 32 bytes" + ))); + } + let value = Hash::from(bytes); + if value.is_zero() { + return Err(Status::invalid_argument(format!( + "{field} must be non-zero" + ))); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use lore_base::types::Address; + use lore_base::types::BranchPoint; + use lore_base::types::Partition; + use lore_proto::lore::revision::v1::revision_create_operation::Op; + use lore_revision::branch::DEFAULT_HISTORY_STEP_SIZE; + use lore_storage::options::WriteOptions; + use opentelemetry::KeyValue; + use tonic::metadata::BinaryMetadataValue; + + use super::*; + use crate::grpc::revision::v1::revision_create; + use crate::grpc::revision::v1::revision_create::RevisionCreateLimits; + use crate::notification::testing::MockNotificationSender; + use crate::store::test_store_create; + + struct TestInstruments; + + impl InstrumentProvider for TestInstruments { + fn namespace(&self) -> &'static str { + "test" + } + + fn labels(&self) -> &[KeyValue] { + &[] + } + } + + async fn create_branch( + repository: &Arc, + branch_id: BranchId, + name: &str, + stack: Vec, + ) { + branch::create( + repository.clone(), + &get_write_token(), + branch_id, + name, + branch::default_category(), + "tester", + 1, + stack, + false, + false, + ) + .await + .expect("create branch"); + } + + async fn content( + immutable: Arc, + repository: Partition, + file_id: Context, + value: &'static [u8], + ) -> Address { + lore_storage::write_content( + immutable, + repository, + file_id, + Bytes::from_static(value), + WriteOptions::default(), + None, + None, + None, + ) + .await + .expect("write content") + .address + } + + fn put(path: &str, address: Address) -> revision_v1::RevisionCreateOperation { + revision_v1::RevisionCreateOperation { + op: Some(Op::PutFile(revision_v1::RevisionCreatePutFile { + path: path.into(), + mode: 0o644, + address: Some(address.into()), + })), + } + } + + fn delete(path: &str) -> revision_v1::RevisionCreateOperation { + revision_v1::RevisionCreateOperation { + op: Some(Op::DeletePath(revision_v1::RevisionCreateDeletePath { + path: path.into(), + })), + } + } + + fn revision_request( + repository: Partition, + branch_id: BranchId, + base: Hash, + operations: Vec, + ) -> Request { + let mut request = Request::new(revision_v1::RevisionCreateRequest { + request_id: Bytes::copy_from_slice(uuid::Uuid::now_v7().as_bytes()), + branch_id: branch_id.into(), + revision_signature_base: base.into(), + commit_message: "test edit".into(), + metadata: Vec::new(), + operations, + }); + request.metadata_mut().insert_bin( + lore_transport::grpc::REPOSITORY_ID_KEY, + BinaryMetadataValue::from_bytes(repository.data()), + ); + request + } + + async fn commit( + repository: Partition, + branch_id: BranchId, + base: Hash, + operations: Vec, + immutable: Arc, + mutable: Arc, + notification: Arc, + hooks: &HookDispatcher, + ) -> Hash { + let response = revision_create::handler( + revision_request(repository, branch_id, base, operations), + immutable, + mutable, + notification, + hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &TestInstruments, + ) + .await + .expect("commit revision") + .into_inner(); + Hash::from(response.revision_signature) + } + + fn merge_request( + repository: Partition, + request_id: uuid::Uuid, + target_branch: BranchId, + target: Hash, + source_branch: BranchId, + source: Hash, + resolutions: Vec, + ) -> Request { + let mut request = Request::new(revision_v1::BranchMergeRequest { + request_id: Bytes::copy_from_slice(request_id.as_bytes()), + branch_id_target: target_branch.into(), + revision_signature_target: target.into(), + branch_id_source: source_branch.into(), + revision_signature_source: source.into(), + commit_message: "merge feature".into(), + resolutions, + }); + request.metadata_mut().insert_bin( + lore_transport::grpc::REPOSITORY_ID_KEY, + BinaryMetadataValue::from_bytes(repository.data()), + ); + request + } + + fn choose(id: Bytes, side: revision_v1::BranchMergeSide) -> revision_v1::BranchMergeResolution { + revision_v1::BranchMergeResolution { + conflict_id: id, + resolution: Some(Resolution::Side(side as i32)), + } + } + + #[tokio::test] + async fn conflicts_are_non_mutating_and_mixed_choices_publish_idempotently() { + let repository_id = Partition::from(uuid::Uuid::now_v7()); + let main = BranchId::from(uuid::Uuid::now_v7()); + let feature = BranchId::from(uuid::Uuid::now_v7()); + let (immutable, mutable, execution) = test_store_create().await.expect("stores"); + let mut notifications = MockNotificationSender::new(); + notifications + .expect_branch_pushed() + .times(4) + .returning(|_, _, _, _, _| ()); + let notifications = Arc::new(notifications); + let hooks = HookDispatcher::empty(); + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable.clone(), + mutable.clone(), + repository_id, + )); + create_branch(&repository, main, "main", Vec::new()).await; + + let file_a = Context::from(uuid::Uuid::now_v7()); + let file_b = Context::from(uuid::Uuid::now_v7()); + let file_c = Context::from(uuid::Uuid::now_v7()); + let base_a = content(immutable.clone(), repository_id, file_a, b"base-a").await; + let base_b = content(immutable.clone(), repository_id, file_b, b"base-b").await; + let base_c = content(immutable.clone(), repository_id, file_c, b"base-c").await; + let base = commit( + repository_id, + main, + Hash::default(), + vec![ + put("a.txt", base_a), + put("b.txt", base_b), + put("c.txt", base_c), + ], + immutable.clone(), + mutable.clone(), + notifications.clone(), + &hooks, + ) + .await; + create_branch( + &repository, + feature, + "feature", + vec![BranchPoint { + branch: main, + revision: base, + }], + ) + .await; + + let source_a = content(immutable.clone(), repository_id, file_a, b"source-a").await; + let source_b = content(immutable.clone(), repository_id, file_b, b"source-b").await; + let source = commit( + repository_id, + feature, + base, + vec![ + put("a.txt", source_a), + put("b.txt", source_b), + delete("c.txt"), + ], + immutable.clone(), + mutable.clone(), + notifications.clone(), + &hooks, + ) + .await; + + let target_a = content(immutable.clone(), repository_id, file_a, b"target-a").await; + let target_b = content(immutable.clone(), repository_id, file_b, b"target-b").await; + let target_c = content(immutable.clone(), repository_id, file_c, b"target-c").await; + let target = commit( + repository_id, + main, + base, + vec![ + put("a.txt", target_a), + put("b.txt", target_b), + put("c.txt", target_c), + ], + immutable.clone(), + mutable.clone(), + notifications.clone(), + &hooks, + ) + .await; + + let stale_error = handler( + merge_request( + repository_id, + uuid::Uuid::now_v7(), + main, + base, + feature, + source, + Vec::new(), + ), + immutable.clone(), + mutable.clone(), + notifications.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + &TestInstruments, + ) + .await + .expect_err("stale target must fail"); + assert_eq!(stale_error.code(), tonic::Code::FailedPrecondition); + + let request_id = uuid::Uuid::now_v7(); + let conflicted = handler( + merge_request( + repository_id, + request_id, + main, + target, + feature, + source, + Vec::new(), + ), + immutable.clone(), + mutable.clone(), + notifications.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + &TestInstruments, + ) + .await + .expect("conflict response") + .into_inner(); + assert_eq!( + conflicted.outcome, + revision_v1::BranchMergeOutcome::Conflicted as i32 + ); + assert_eq!(conflicted.unresolved_conflicts.len(), 3); + assert_eq!( + branch::load_latest(repository.clone(), main).await.unwrap(), + target + ); + + let resolutions = conflicted + .unresolved_conflicts + .iter() + .map(|conflict| { + let side = if conflict.path_source == "a.txt" || conflict.path_source == "c.txt" + { + revision_v1::BranchMergeSide::Source + } else { + revision_v1::BranchMergeSide::Target + }; + choose(conflict.conflict_id.clone(), side) + }) + .collect::>(); + let request = || { + merge_request( + repository_id, + request_id, + main, + target, + feature, + source, + resolutions.clone(), + ) + }; + let merged = handler( + request(), + immutable.clone(), + mutable.clone(), + notifications.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + &TestInstruments, + ) + .await + .expect("resolved merge") + .into_inner(); + assert_eq!( + merged.outcome, + revision_v1::BranchMergeOutcome::Merged as i32 + ); + let revision = Hash::from(&merged.revision_signature); + let state = State::deserialize(repository.clone(), revision) + .await + .expect("merge state"); + assert_eq!(state.parent_self(), target); + assert_eq!(state.parent_other(), source); + let a = state + .find_node_link(repository.clone(), "a.txt") + .await + .expect("a.txt"); + let b = state + .find_node_link(repository.clone(), "b.txt") + .await + .expect("b.txt"); + assert_eq!( + state + .node(repository.clone(), a.node) + .await + .unwrap() + .address, + source_a + ); + assert_eq!( + state + .node(repository.clone(), b.node) + .await + .unwrap() + .address, + target_b + ); + assert!( + state + .find_node_link(repository.clone(), "c.txt") + .await + .is_err() + ); + + let retry = handler( + request(), + immutable.clone(), + mutable.clone(), + notifications.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + &TestInstruments, + ) + .await + .expect("idempotent retry") + .into_inner(); + assert_eq!(retry, merged); + + let already = handler( + merge_request( + repository_id, + uuid::Uuid::now_v7(), + main, + revision, + feature, + source, + Vec::new(), + ), + immutable.clone(), + mutable.clone(), + notifications, + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + &TestInstruments, + ) + .await + .expect("source is already merged") + .into_inner(); + assert_eq!( + already.outcome, + revision_v1::BranchMergeOutcome::AlreadyUpToDate as i32 + ); + assert_eq!( + branch::load_latest(repository, main).await.unwrap(), + revision + ); + })) + .await; + } +} diff --git a/lore-server/src/grpc/revision/v1/branch_push.rs b/lore-server/src/grpc/revision/v1/branch_push.rs index 185296b74..7b5ab0551 100644 --- a/lore-server/src/grpc/revision/v1/branch_push.rs +++ b/lore-server/src/grpc/revision/v1/branch_push.rs @@ -102,116 +102,154 @@ pub async fn handler( mutable_store, repository_id, )); - let repository_id: RepositoryId = repository.id; let execution = setup_execution(module_path!(), correlation_id.clone(), user_id.clone()); LORE_CONTEXT - .scope(execution, async move { - let mut ctx_builder = HookContext::builder() - .correlation_id(correlation_id.clone()) - .hook_point(HookPoint::BranchPush) - .repository(repository_id) - .user(user_id.clone()) - .branch(branch_id) - .revision(revision); - - if let Some(ip) = client_ip { - ctx_builder = ctx_builder.metadata("client_ip", ip); - } - - let mut hook_ctx = ctx_builder.build(); - - hook_dispatcher - .dispatch_pre(HookPoint::BranchPush, &hook_ctx) - .map_err(hook_error_to_status)?; - - ensure_branch_pushable(repository.clone(), branch_id).await?; - - let PushResult { - success, - fast_forward_merged, - revision: resulting_revision, - revision_number, - } = push( - repository.clone(), + .scope( + execution, + publish_revision( + repository, branch_id, revision, bypass_protection, force, fast_forward_merge, + client_ip, + correlation_id, + user_id, + notification, + hook_dispatcher, history_step_size, acceleration, - ) - .await?; - - instrument_provider - .counter("num_branches_pushed") - .add(1, &[]); - - if !success { - let detail = if fast_forward_merge { - format!("Fast-forward merge has conflicts; branch latest: {resulting_revision}") - } else { - format!( - "Branch push is not a fast-forward; branch latest: {resulting_revision}" - ) - }; - debug!( - {BRANCH_ID} = %branch_id, - branch_latest = %resulting_revision, - fast_forward_merge, - "Branch push rejected", - ); - return Err(Status::failed_precondition(detail)); - } + instrument_provider, + ), + ) + .await +} - lore_spawn!({ - let user_id = user_id.clone(); - async move { - notification - .branch_pushed( - repository_id, - branch_id, - &user_id, - resulting_revision, - revision_number, - ) - .instrument(span!(Level::DEBUG, "publish_notification")) - .await; - } - .in_current_span() - }); +/// Shared authoritative publication orchestration for BranchPush and +/// server-constructed revisions. The low-level CAS helper is intentionally +/// kept behind this function so callers cannot skip protection, hooks, +/// fragment verification, notifications, or response hooks. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn publish_revision( + repository: Arc, + branch_id: BranchId, + revision: Hash, + bypass_protection: bool, + force: bool, + fast_forward_merge: bool, + client_ip: Option, + correlation_id: String, + user_id: String, + notification: Arc, + hook_dispatcher: &HookDispatcher, + history_step_size: u64, + acceleration: crate::grpc::server::RevisionListAcceleration, + instrument_provider: &impl InstrumentProvider, +) -> Result, Status> { + let repository_id: RepositoryId = repository.id; + let mut ctx_builder = HookContext::builder() + .correlation_id(correlation_id.clone()) + .hook_point(HookPoint::BranchPush) + .repository(repository_id) + .user(user_id.clone()) + .branch(branch_id) + .revision(revision); + + if let Some(ip) = client_ip { + ctx_builder = ctx_builder.metadata("client_ip", ip); + } - hook_ctx.set_revision_number(revision_number); - hook_dispatcher.spawn_post(HookPoint::BranchPush, hook_ctx); + let mut hook_ctx = ctx_builder.build(); - let message = dispatch_response_message( - hook_dispatcher, - &correlation_id, - &user_id, - repository_id, - branch_id, - resulting_revision, - repository.clone(), - ) - .await; - - debug!( - {BRANCH_ID} = %branch_id, - {REVISION} = %resulting_revision, - revision_number, - fast_forward_merged, - "Branch push response", - ); + hook_dispatcher + .dispatch_pre(HookPoint::BranchPush, &hook_ctx) + .map_err(hook_error_to_status)?; - Ok(Response::new(BranchPushResponse { - revision_signature: resulting_revision.into(), - revision_number, - fast_forward_merged, - message, - })) - }) - .await + ensure_branch_pushable(repository.clone(), branch_id).await?; + + let PushResult { + success, + fast_forward_merged, + revision: resulting_revision, + revision_number, + } = push( + repository.clone(), + branch_id, + revision, + bypass_protection, + force, + fast_forward_merge, + history_step_size, + acceleration, + ) + .await?; + + instrument_provider + .counter("num_branches_pushed") + .add(1, &[]); + + if !success { + let detail = if fast_forward_merge { + format!("Fast-forward merge has conflicts; branch latest: {resulting_revision}") + } else { + format!("Branch push is not a fast-forward; branch latest: {resulting_revision}") + }; + debug!( + {BRANCH_ID} = %branch_id, + branch_latest = %resulting_revision, + fast_forward_merge, + "Branch push rejected", + ); + return Err(Status::failed_precondition(detail)); + } + + lore_spawn!({ + let user_id = user_id.clone(); + async move { + notification + .branch_pushed( + repository_id, + branch_id, + &user_id, + resulting_revision, + revision_number, + ) + .instrument(span!(Level::DEBUG, "publish_notification")) + .await; + } + .in_current_span() + }); + + hook_ctx.set_revision_number(revision_number); + hook_dispatcher.spawn_post(HookPoint::BranchPush, hook_ctx); + + let message = dispatch_response_message( + hook_dispatcher, + &correlation_id, + &user_id, + repository_id, + branch_id, + resulting_revision, + repository.clone(), + ) + .await; + + debug!( + {BRANCH_ID} = %branch_id, + {REVISION} = %resulting_revision, + revision_number, + fast_forward_merged, + "Branch push response", + ); + + Ok(Response::new(BranchPushResponse { + revision_signature: resulting_revision.into(), + revision_number, + fast_forward_merged, + message, + })) } /// Returns `NotFound` for branch ids without metadata, and reinstates diff --git a/lore-server/src/grpc/revision/v1/idempotency.rs b/lore-server/src/grpc/revision/v1/idempotency.rs new file mode 100644 index 000000000..d0dc388de --- /dev/null +++ b/lore-server/src/grpc/revision/v1/idempotency.rs @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT + +//! Durable request-id reservations for server-authored revisions. + +use std::sync::Arc; +use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use bytes::Bytes; +use lore_base::types::Address; +use lore_base::types::BranchId; +use lore_base::types::Context; +use lore_base::types::Hash; +use lore_base::types::Partition; +use lore_storage::KeyType; +use lore_storage::options::ReadOptions; +use lore_storage::options::WriteOptions; +use serde::Deserialize; +use serde::Serialize; +use tonic::Status; + +const PENDING_TTL: Duration = Duration::from_secs(15 * 60); + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Record { + request_digest: Hash, + created_at_epoch_seconds: u64, + revision: Hash, + revision_number: u64, +} + +impl Record { + fn pending(request_digest: Hash) -> Self { + Self { + request_digest, + created_at_epoch_seconds: now(), + revision: Hash::default(), + revision_number: 0, + } + } + + fn is_complete(&self) -> bool { + !self.revision.is_zero() + } + + fn is_expired(&self) -> bool { + now().saturating_sub(self.created_at_epoch_seconds) >= PENDING_TTL.as_secs() + } +} + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +pub(crate) struct Reservation { + operation: &'static str, + key: Hash, + record_hash: Hash, + request_id: Context, + request_digest: Hash, +} + +pub(crate) enum Start { + Completed { + revision: Hash, + revision_number: u64, + }, + Reserved(Reservation), +} + +fn key(operation: &str, branch: BranchId, request_id: Context) -> Hash { + let mut bytes = Vec::with_capacity(operation.len() + 1 + 16 + 16); + bytes.extend_from_slice(operation.as_bytes()); + bytes.push(0); + bytes.extend_from_slice(branch.data()); + bytes.extend_from_slice(request_id.data()); + Hash::hash_buffer(&bytes) +} + +async fn store_record( + immutable_store: Arc, + repository: Partition, + request_id: Context, + record: &Record, +) -> Result { + let bytes = serde_json::to_vec(record) + .map_err(|error| Status::internal(format!("serialize idempotency record: {error}")))?; + let address = lore_storage::write_content( + immutable_store, + repository, + request_id, + Bytes::from(bytes), + WriteOptions::default().with_local_cache_priority(), + None, + None, + None, + ) + .await + .map_err(|error| Status::internal(format!("store idempotency record: {error}")))?; + Ok(address.address.hash) +} + +async fn load_record( + immutable_store: Arc, + repository: Partition, + request_id: Context, + record_hash: Hash, +) -> Result { + let (_, bytes) = lore_storage::read( + immutable_store, + repository, + Address { + hash: record_hash, + context: request_id, + }, + None, + ReadOptions::default(), + None, + ) + .await + .map_err(|error| Status::internal(format!("load idempotency record: {error}")))?; + serde_json::from_slice(bytes.as_ref()) + .map_err(|error| Status::internal(format!("decode idempotency record: {error}"))) +} + +pub(crate) async fn begin( + operation: &'static str, + immutable_store: Arc, + mutable_store: Arc, + repository: Partition, + branch: BranchId, + request_id: Context, + request_digest: Hash, +) -> Result { + let key = key(operation, branch, request_id); + for _ in 0..4 { + let current = match mutable_store + .clone() + .load(repository, key, KeyType::Untyped) + .await + { + Ok(hash) => hash, + Err(error) if error.is_address_not_found() => Hash::default(), + Err(error) => { + return Err(Status::internal(format!( + "load {operation} idempotency pointer: {error}" + ))); + } + }; + + if !current.is_zero() { + let record = + load_record(immutable_store.clone(), repository, request_id, current).await?; + if record.request_digest != request_digest { + return Err(Status::already_exists( + "request_id was already used with different request content", + )); + } + if record.is_complete() { + return Ok(Start::Completed { + revision: record.revision, + revision_number: record.revision_number, + }); + } + if !record.is_expired() { + return Err(Status::aborted(format!( + "an identical {operation} request is still in progress; retry later" + ))); + } + } + + let pending_hash = store_record( + immutable_store.clone(), + repository, + request_id, + &Record::pending(request_digest), + ) + .await?; + let observed = mutable_store + .clone() + .compare_and_swap(repository, key, current, pending_hash, KeyType::Untyped) + .await + .map_err(|error| { + Status::internal(format!("reserve {operation} request_id: {error}")) + })?; + if observed == current { + return Ok(Start::Reserved(Reservation { + operation, + key, + record_hash: pending_hash, + request_id, + request_digest, + })); + } + } + Err(Status::aborted(format!( + "{operation} request_id reservation changed concurrently; retry" + ))) +} + +pub(crate) async fn release( + mutable_store: Arc, + repository: Partition, + reservation: &Reservation, +) { + let _ = mutable_store + .compare_and_swap( + repository, + reservation.key, + reservation.record_hash, + Hash::default(), + KeyType::Untyped, + ) + .await; +} + +pub(crate) async fn complete( + immutable_store: Arc, + mutable_store: Arc, + repository: Partition, + reservation: &Reservation, + revision: Hash, + revision_number: u64, +) -> Result<(), Status> { + let completed_hash = store_record( + immutable_store, + repository, + reservation.request_id, + &Record { + request_digest: reservation.request_digest, + created_at_epoch_seconds: now(), + revision, + revision_number, + }, + ) + .await?; + let observed = mutable_store + .compare_and_swap( + repository, + reservation.key, + reservation.record_hash, + completed_hash, + KeyType::Untyped, + ) + .await + .map_err(|error| { + Status::internal(format!( + "complete {} idempotency record: {error}", + reservation.operation + )) + })?; + if observed != reservation.record_hash { + return Err(Status::internal(format!( + "{} idempotency reservation changed before completion", + reservation.operation + ))); + } + Ok(()) +} diff --git a/lore-server/src/grpc/revision/v1/mod.rs b/lore-server/src/grpc/revision/v1/mod.rs index 7f7448f32..67b164fa7 100644 --- a/lore-server/src/grpc/revision/v1/mod.rs +++ b/lore-server/src/grpc/revision/v1/mod.rs @@ -4,9 +4,12 @@ pub mod branch_create; pub mod branch_delete; pub mod branch_get; pub mod branch_list; +pub mod branch_merge; pub mod branch_metadata_get; pub mod branch_metadata_set; pub mod branch_push; mod branch_record; +mod idempotency; +pub mod revision_create; pub mod revision_list; pub mod service; diff --git a/lore-server/src/grpc/revision/v1/revision_create.rs b/lore-server/src/grpc/revision/v1/revision_create.rs new file mode 100644 index 000000000..99d162824 --- /dev/null +++ b/lore-server/src/grpc/revision/v1/revision_create.rs @@ -0,0 +1,1305 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::collections::BTreeMap; +use std::sync::Arc; + +use futures::StreamExt; +use futures::TryStreamExt; +use lore_base::runtime::LORE_CONTEXT; +use lore_base::types::Address; +use lore_base::types::BranchId; +use lore_base::types::Context; +use lore_base::types::Hash; +use lore_base::types::Partition; +use lore_proto::lore::revision::v1 as revision_v1; +use lore_proto::lore::revision::v1::revision_create_operation::Op; +use lore_revision::branch; +use lore_revision::commit::CommitError; +use lore_revision::commit::construct_tree_revision; +use lore_revision::interface::LoreNodeType; +use lore_revision::metadata::Metadata; +use lore_revision::node::NodeDelta; +use lore_revision::node::NodeID; +use lore_revision::node::ROOT_NODE; +use lore_revision::notification::NotificationSender; +use lore_revision::repository::RepositoryContext; +use lore_revision::revision_tree::RevisionTreeEditError; +use lore_revision::state::State; +use lore_storage::StoreMatch; +use lore_telemetry::InstrumentProvider; +use prost::Message; +use tonic::Request; +use tonic::Response; +use tonic::Status; + +use super::branch_push::publish_revision; +use super::idempotency; +use crate::grpc::extract_correlation_id; +use crate::grpc::get_authorization; +use crate::grpc::get_repository; +use crate::grpc::get_user_id; +use crate::grpc::get_write_token; +use crate::grpc::handlers::branch_push::extract_client_ip; +use crate::grpc::server::RevisionListAcceleration; +use crate::hooks::HookDispatcher; +use crate::util::setup_execution; + +#[derive(Clone, Copy, Debug)] +pub struct RevisionCreateLimits { + pub max_request_bytes: usize, + pub max_operations: usize, + pub max_metadata_entries: usize, + pub max_metadata_bytes: usize, + pub max_path_bytes: usize, +} + +impl Default for RevisionCreateLimits { + fn default() -> Self { + Self { + max_request_bytes: 4 * 1024 * 1024, + max_operations: 10_000, + max_metadata_entries: 256, + max_metadata_bytes: 256 * 1024, + max_path_bytes: 4096, + } + } +} + +fn resource_limit(name: &str, actual: usize, limit: usize) -> Status { + Status::resource_exhausted(format!( + "RevisionCreate {name} limit exceeded: {actual} > {limit}" + )) +} + +fn uuid_v7(bytes: &[u8], field: &str) -> Result { + let id = uuid::Uuid::from_slice(bytes) + .map_err(|error| Status::invalid_argument(format!("invalid {field}: {error}")))?; + if id.get_version_num() != 7 { + return Err(Status::invalid_argument(format!( + "{field} must be a UUIDv7" + ))); + } + Ok(id) +} + +fn fixed_context(bytes: &[u8], field: &str) -> Result { + if bytes.len() != 16 { + return Err(Status::invalid_argument(format!( + "{field} must contain exactly 16 bytes" + ))); + } + Ok(Context::from(bytes)) +} + +fn fixed_hash(bytes: &[u8], field: &str) -> Result { + if bytes.len() != 32 { + return Err(Status::invalid_argument(format!( + "{field} must contain exactly 32 bytes" + ))); + } + Ok(Hash::from(bytes)) +} + +fn address(value: &lore_proto::lore::model::v1::Address) -> Result { + let hash = fixed_hash(value.hash.as_ref(), "address.hash")?; + let context = fixed_context(value.context.as_ref(), "address.context")?; + if context.is_zero() { + return Err(Status::invalid_argument( + "address.context (file id) must be non-zero", + )); + } + Ok(Address { hash, context }) +} + +fn path_parts<'a>(path: &'a str, limits: RevisionCreateLimits) -> Result, Status> { + if path.is_empty() { + return Err(Status::invalid_argument("path must not be empty")); + } + if path.len() > limits.max_path_bytes { + return Err(resource_limit( + "path bytes", + path.len(), + limits.max_path_bytes, + )); + } + if path.starts_with('/') || path.ends_with('/') || path.contains('\\') { + return Err(Status::invalid_argument( + "path must be normalized, relative, and use '/' separators", + )); + } + let parts = path.split('/').collect::>(); + if parts + .iter() + .any(|part| part.is_empty() || *part == "." || *part == "..") + { + return Err(Status::invalid_argument( + "path contains an empty, '.' or '..' component", + )); + } + Ok(parts) +} + +async fn resolve_path( + state: Arc, + repository: Arc, + parts: &[&str], +) -> Result { + let mut node_id = ROOT_NODE; + for part in parts { + let parent = state + .node(repository.clone(), node_id) + .await + .map_err(|error| Status::internal(format!("failed to read path parent: {error}")))?; + if parent.is_discarded() || !parent.is_directory() { + return Err(Status::invalid_argument(format!( + "path component parent for '{part}' is not a directory" + ))); + } + node_id = match state + .find_subnode(repository.clone(), node_id, lore_storage::hash_string(part)) + .await + { + Ok(node_id) => node_id, + Err(error) if error.is_node_not_found() => { + return Err(Status::invalid_argument(format!( + "path component '{part}' does not exist" + ))); + } + Err(error) => { + return Err(Status::internal(format!( + "failed resolving path component '{part}': {error}" + ))); + } + }; + } + Ok(node_id) +} + +async fn resolve_parent( + state: Arc, + repository: Arc, + parts: &[&str], +) -> Result<(NodeID, String), Status> { + let (name, parents) = parts + .split_last() + .ok_or_else(|| Status::invalid_argument("path must not be empty"))?; + let parent = if parents.is_empty() { + ROOT_NODE + } else { + resolve_path(state, repository, parents).await? + }; + Ok((parent, (*name).to_string())) +} + +fn edit_status(error: RevisionTreeEditError) -> Status { + if error.is_invalid_arguments() { + Status::invalid_argument(error.to_string()) + } else { + Status::internal(error.to_string()) + } +} + +fn commit_status(error: CommitError) -> Status { + if error.is_nothing_staged() || error.is_invalid_arguments() { + Status::invalid_argument(error.to_string()) + } else if error.is_slow_down() { + Status::resource_exhausted(error.to_string()) + } else { + Status::internal(error.to_string()) + } +} + +fn metadata( + entries: &[revision_v1::RevisionCreateMetadataEntry], + commit_message: &str, +) -> Result { + let mut metadata = Metadata::default(); + for entry in entries { + if entry.key.is_empty() { + return Err(Status::invalid_argument("metadata key must not be empty")); + } + let result = match entry.format { + 1 => { + if entry.value.len() != 48 { + return Err(Status::invalid_argument( + "address metadata values must contain exactly 48 bytes", + )); + } + metadata.set_address( + &entry.key, + Address { + hash: Hash::from(&entry.value[..32]), + context: Context::from(&entry.value[32..]), + }, + ) + } + 2 => { + if entry.value.len() != 1 || entry.value[0] > 1 { + return Err(Status::invalid_argument( + "boolean metadata values must be one byte (0 or 1)", + )); + } + metadata.set_bool(&entry.key, entry.value[0] != 0) + } + 3 => { + let value = fixed_context(entry.value.as_ref(), "metadata context value")?; + metadata.set_context(&entry.key, value) + } + 4 => { + let value = fixed_hash(entry.value.as_ref(), "metadata hash value")?; + metadata.set_hash(&entry.key, value) + } + 5 => { + let bytes: [u8; 8] = entry.value.as_ref().try_into().map_err(|_| { + Status::invalid_argument("numeric metadata values must contain 8 bytes") + })?; + metadata.set_u64(&entry.key, u64::from_le_bytes(bytes)) + } + 6 => { + let value = std::str::from_utf8(entry.value.as_ref()).map_err(|_| { + Status::invalid_argument("string metadata value is not valid UTF-8") + })?; + metadata.set_string(&entry.key, value) + } + 255 => metadata.set_binary(&entry.key, entry.value.as_ref()), + _ => return Err(Status::invalid_argument("unsupported metadata format")), + }; + result.map_err(|error| Status::invalid_argument(error.to_string()))?; + } + metadata + .set_string(lore_revision::metadata::MESSAGE, commit_message) + .map_err(|error| Status::invalid_argument(error.to_string()))?; + Ok(metadata) +} + +async fn address_sizes( + immutable_store: Arc, + repository: Partition, + operations: &[revision_v1::RevisionCreateOperation], +) -> Result, Status> { + let mut addresses = BTreeMap::new(); + for operation in operations { + if let Some(Op::PutFile(put)) = &operation.op { + let value = put + .address + .as_ref() + .ok_or_else(|| Status::invalid_argument("PutFile.address is required"))?; + addresses.insert(address(value)?, 0); + } + } + + let loaded = futures::stream::iter(addresses.into_keys()) + .map(|address| { + let store = immutable_store.clone(); + async move { + if address.hash.is_zero() { + return Ok((address, 0)); + } + let query = store + .get_metadata(repository, address) + .await + .map_err(|error| { + Status::internal(format!("failed to query PutFile address: {error}")) + })?; + if query.match_made != StoreMatch::MatchFull { + return Err(Status::invalid_argument(format!( + "PutFile address {address} is not stored in this repository/context" + ))); + } + Ok((address, query.fragment.size_content)) + } + }) + .buffer_unordered(32) + .try_collect::>() + .await?; + Ok(loaded.into_iter().collect()) +} + +async fn apply_operations( + state: Arc, + repository: Arc, + operations: &[revision_v1::RevisionCreateOperation], + sizes: &BTreeMap, + limits: RevisionCreateLimits, +) -> Result, Status> { + let mut deleted = Vec::new(); + for operation in operations { + match operation + .op + .as_ref() + .ok_or_else(|| Status::invalid_argument("operation is missing its op"))? + { + Op::PutFile(put) => { + let parts = path_parts(&put.path, limits)?; + let proto_address = put + .address + .as_ref() + .ok_or_else(|| Status::invalid_argument("PutFile.address is required"))?; + let address = address(proto_address)?; + let size = *sizes + .get(&address) + .ok_or_else(|| Status::internal("PutFile address size was not loaded"))?; + let mode = u16::try_from(put.mode) + .map_err(|_| Status::invalid_argument("PutFile.mode exceeds u16"))?; + match resolve_path(state.clone(), repository.clone(), &parts).await { + Ok(node_id) => { + lore_revision::revision_tree::modify_file( + state.clone(), + repository.clone(), + node_id, + mode, + size, + address, + ) + .await + .map_err(edit_status)?; + } + Err(status) if status.code() == tonic::Code::InvalidArgument => { + let (parent, name) = + resolve_parent(state.clone(), repository.clone(), &parts).await?; + lore_revision::revision_tree::add_node( + state.clone(), + repository.clone(), + parent, + name.as_bytes(), + LoreNodeType::File as u32, + mode, + size, + address, + ) + .await + .map_err(edit_status)?; + } + Err(status) => return Err(status), + } + } + Op::CreateDirectory(create) => { + let parts = path_parts(&create.path, limits)?; + match resolve_path(state.clone(), repository.clone(), &parts).await { + Ok(_) => { + return Err(Status::invalid_argument(format!( + "path '{}' already exists", + create.path + ))); + } + Err(status) if status.code() == tonic::Code::InvalidArgument => {} + Err(status) => return Err(status), + } + let mode = u16::try_from(create.mode) + .map_err(|_| Status::invalid_argument("CreateDirectory.mode exceeds u16"))?; + let (parent, name) = + resolve_parent(state.clone(), repository.clone(), &parts).await?; + lore_revision::revision_tree::create_directory( + state.clone(), + repository.clone(), + parent, + name.as_bytes(), + mode, + ) + .await + .map_err(edit_status)?; + } + Op::DeletePath(delete) => { + let parts = path_parts(&delete.path, limits)?; + let node_id = resolve_path(state.clone(), repository.clone(), &parts).await?; + deleted.extend( + lore_revision::revision_tree::delete_node( + state.clone(), + repository.clone(), + node_id, + ) + .await + .map_err(edit_status)?, + ); + } + Op::MovePath(moved) => { + let source = path_parts(&moved.source, limits)?; + let destination = path_parts(&moved.destination, limits)?; + let node_id = resolve_path(state.clone(), repository.clone(), &source).await?; + let (parent, name) = + resolve_parent(state.clone(), repository.clone(), &destination).await?; + lore_revision::revision_tree::move_node( + state.clone(), + repository.clone(), + node_id, + parent, + name.as_bytes(), + ) + .await + .map_err(edit_status)?; + } + } + } + Ok(deleted) +} + +#[allow(clippy::too_many_arguments)] +#[tracing::instrument(name = "RevisionCreate::v1::handle", skip_all)] +pub async fn handler( + request: Request, + immutable_store: Arc, + mutable_store: Arc, + notification: Arc, + hook_dispatcher: &HookDispatcher, + history_step_size: u64, + acceleration: RevisionListAcceleration, + limits: RevisionCreateLimits, + instrument_provider: &impl InstrumentProvider, +) -> Result, Status> { + let repository_id = get_repository(request.metadata())?; + let correlation_id = extract_correlation_id(&request).unwrap_or_default(); + let user_id = get_user_id(request.extensions()); + let bypass_protection = get_authorization(request.extensions()) + .ok() + .and_then(|token| token.is_service_account) + .unwrap_or_default(); + let client_ip = extract_client_ip(&request).map(|value| value.to_string()); + let req = request.into_inner(); + + let encoded_len = req.encoded_len(); + if encoded_len > limits.max_request_bytes { + return Err(resource_limit( + "encoded request bytes", + encoded_len, + limits.max_request_bytes, + )); + } + if req.operations.is_empty() { + return Err(Status::invalid_argument( + "RevisionCreate requires at least one operation", + )); + } + if req.operations.len() > limits.max_operations { + return Err(resource_limit( + "operation count", + req.operations.len(), + limits.max_operations, + )); + } + if req.metadata.len() > limits.max_metadata_entries { + return Err(resource_limit( + "metadata count", + req.metadata.len(), + limits.max_metadata_entries, + )); + } + let metadata_bytes = req.metadata.iter().try_fold(0usize, |total, entry| { + total + .checked_add(entry.key.len()) + .and_then(|value| value.checked_add(entry.value.len())) + .ok_or_else(|| Status::resource_exhausted("metadata byte count overflow")) + })?; + if metadata_bytes > limits.max_metadata_bytes { + return Err(resource_limit( + "metadata bytes", + metadata_bytes, + limits.max_metadata_bytes, + )); + } + + let request_id = Context::from(uuid_v7(req.request_id.as_ref(), "request_id")?); + let branch_id = BranchId::from(fixed_context(req.branch_id.as_ref(), "branch_id")?); + if branch_id.is_zero() { + return Err(Status::invalid_argument("branch_id must be non-zero")); + } + let base_revision = fixed_hash( + req.revision_signature_base.as_ref(), + "revision_signature_base", + )?; + let request_digest = Hash::hash_buffer(req.encode_to_vec().as_slice()); + + let reservation = match idempotency::begin( + "RevisionCreate", + immutable_store.clone(), + mutable_store.clone(), + repository_id, + branch_id, + request_id, + request_digest, + ) + .await? + { + idempotency::Start::Completed { + revision, + revision_number, + } => { + return Ok(Response::new(revision_v1::RevisionCreateResponse { + revision_signature: revision.into(), + revision_number, + })); + } + idempotency::Start::Reserved(reservation) => reservation, + }; + + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository_id, + )); + let current_tip = branch::load_latest(repository.clone(), branch_id) + .await + .unwrap_or_default(); + if current_tip != base_revision { + idempotency::release(mutable_store, repository_id, &reservation).await; + return Err(Status::failed_precondition(format!( + "branch advanced; current latest: {current_tip}" + ))); + } + + let pending_metadata = match metadata(&req.metadata, &req.commit_message) { + Ok(metadata) => metadata, + Err(status) => { + idempotency::release(mutable_store, repository_id, &reservation).await; + return Err(status); + } + }; + let sizes = match address_sizes(immutable_store.clone(), repository_id, &req.operations).await { + Ok(sizes) => sizes, + Err(status) => { + idempotency::release(mutable_store, repository_id, &reservation).await; + return Err(status); + } + }; + let execution = setup_execution(module_path!(), correlation_id.clone(), user_id.clone()); + let outcome = LORE_CONTEXT + .scope(execution, async move { + let state = State::deserialize(repository.clone(), base_revision) + .await + .map_err(|error| { + if error.is_address_not_found() + || error.is_payload_not_found() + || error.is_not_found() + { + Status::not_found(format!("base revision was not found: {error}")) + } else { + Status::internal(format!("failed to deserialize base revision: {error}")) + } + })?; + let deleted = apply_operations( + state.clone(), + repository.clone(), + &req.operations, + &sizes, + limits, + ) + .await?; + let token = get_write_token(); + let revision = construct_tree_revision( + repository.clone(), + &token, + state, + pending_metadata, + branch_id, + deleted, + ) + .await + .map_err(commit_status)?; + + let published = publish_revision( + repository, + branch_id, + revision, + bypass_protection, + false, + false, + client_ip, + correlation_id, + user_id, + notification.clone(), + hook_dispatcher, + history_step_size, + acceleration, + instrument_provider, + ) + .await? + .into_inner(); + + Ok(Response::new(revision_v1::RevisionCreateResponse { + revision_signature: published.revision_signature, + revision_number: published.revision_number, + })) + }) + .await; + + match outcome { + Ok(response) => { + idempotency::complete( + immutable_store, + mutable_store, + repository_id, + &reservation, + fixed_hash( + response.get_ref().revision_signature.as_ref(), + "published revision", + )?, + response.get_ref().revision_number, + ) + .await?; + Ok(response) + } + Err(status) => { + idempotency::release(mutable_store, repository_id, &reservation).await; + Err(status) + } + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use lore_proto::lore::storage::v1::UploadContentHeader; + use lore_proto::lore::storage::v1::UploadContentRequest; + use lore_proto::lore::storage::v1::upload_content_request::Part as UploadPart; + use lore_proto::lore::thin_client::v1::RevisionTreeRequest; + use lore_proto::lore::thin_client::v1::revision_tree_request::Query; + use lore_proto::lore::thin_client::v1::revision_tree_response::Payload; + use lore_revision::branch::DEFAULT_HISTORY_STEP_SIZE; + use lore_revision::node::NodeIDExt; + use lore_storage::options::WriteOptions; + use lore_telemetry::InstrumentProvider; + use opentelemetry::KeyValue; + use tokio_stream::StreamExt; + use tonic::metadata::BinaryMetadataValue; + + use super::*; + use crate::hooks::HookDispatcher; + use crate::notification::testing::MockNotificationSender; + use crate::store::test_store_create; + + struct TestInstrumentProvider; + + impl InstrumentProvider for TestInstrumentProvider { + fn namespace(&self) -> &'static str { + "test" + } + + fn labels(&self) -> &[KeyValue] { + &[] + } + } + + async fn create_root_branch(repository: &Arc, branch_id: BranchId) { + let token = get_write_token(); + branch::create( + repository.clone(), + &token, + branch_id, + "main", + branch::default_category(), + "test-creator", + 1, + vec![], + false, + false, + ) + .await + .expect("create root branch"); + } + + fn request( + repository: Partition, + request_id: uuid::Uuid, + branch_id: BranchId, + base: Hash, + operations: Vec, + ) -> Request { + let mut request = Request::new(revision_v1::RevisionCreateRequest { + request_id: Bytes::copy_from_slice(request_id.as_bytes()), + branch_id: branch_id.into(), + revision_signature_base: base.into(), + commit_message: "browser edit".into(), + metadata: Vec::new(), + operations, + }); + request.metadata_mut().insert_bin( + lore_transport::grpc::REPOSITORY_ID_KEY, + BinaryMetadataValue::from_bytes(repository.data()), + ); + request + } + + fn service_account_request( + repository: Partition, + request_id: uuid::Uuid, + branch_id: BranchId, + base: Hash, + operations: Vec, + ) -> Request { + let mut request = request(repository, request_id, branch_id, base, operations); + request + .extensions_mut() + .insert(crate::auth::jwt::AuthorizationToken { + user_id: "service-bot".into(), + is_service_account: Some(true), + ..crate::auth::jwt::AuthorizationToken::default() + }); + request + } + + fn create_directory(path: &str) -> revision_v1::RevisionCreateOperation { + revision_v1::RevisionCreateOperation { + op: Some(Op::CreateDirectory(revision_v1::RevisionCreateDirectory { + path: path.into(), + mode: 0o755, + })), + } + } + + fn put_file(path: &str, address: Address) -> revision_v1::RevisionCreateOperation { + revision_v1::RevisionCreateOperation { + op: Some(Op::PutFile(revision_v1::RevisionCreatePutFile { + path: path.into(), + mode: 0o644, + address: Some(address.into()), + })), + } + } + + fn delete_path(path: &str) -> revision_v1::RevisionCreateOperation { + revision_v1::RevisionCreateOperation { + op: Some(Op::DeletePath(revision_v1::RevisionCreateDeletePath { + path: path.into(), + })), + } + } + + fn move_path(source: &str, destination: &str) -> revision_v1::RevisionCreateOperation { + revision_v1::RevisionCreateOperation { + op: Some(Op::MovePath(revision_v1::RevisionCreateMovePath { + source: source.into(), + destination: destination.into(), + })), + } + } + + #[tokio::test] + async fn revision_create_publishes_file_and_nested_empty_directory_idempotently() { + let repository_id = Partition::from(uuid::Uuid::now_v7()); + let branch_id = BranchId::from(uuid::Uuid::now_v7()); + let request_id = uuid::Uuid::now_v7(); + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("create stores"); + + let mut notification = MockNotificationSender::new(); + notification + .expect_branch_pushed() + .once() + .return_once(|_, _, _, _, _| ()); + let notification = Arc::new(notification); + let hooks = HookDispatcher::empty(); + let instruments = TestInstrumentProvider; + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository_id, + )); + create_root_branch(&repository, branch_id).await; + + let payload = Bytes::from_static(b"hello from a browser upload"); + let file_id = uuid::Uuid::now_v7(); + let uploaded = crate::grpc::storage::v1::upload_content::ingest( + futures::stream::iter(vec![ + Ok(UploadContentRequest { + part: Some(UploadPart::Header(UploadContentHeader { + file_id: Bytes::copy_from_slice(file_id.as_bytes()), + expected_size: Some(payload.len() as u64), + request_id: Bytes::copy_from_slice(uuid::Uuid::now_v7().as_bytes()), + })), + }), + Ok(UploadContentRequest { + part: Some(UploadPart::Chunk(payload)), + }), + ]), + immutable_store.clone(), + repository_id, + None, + ) + .await + .expect("stream file content"); + let file_address = Address::from(uploaded.address.expect("upload address")); + assert_eq!(uploaded.size, 27); + + let operations = vec![ + create_directory("docs"), + create_directory("docs/empty"), + create_directory("root-empty"), + put_file("docs/readme.txt", file_address), + ]; + let first = handler( + request( + repository_id, + request_id, + branch_id, + Hash::default(), + operations.clone(), + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect("create revision") + .into_inner(); + assert!(!first.revision_signature.is_empty()); + assert_eq!(first.revision_number, 1); + + let retry = handler( + request( + repository_id, + request_id, + branch_id, + Hash::default(), + operations, + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect("idempotent retry") + .into_inner(); + assert_eq!(retry, first); + + let conflict = handler( + request( + repository_id, + request_id, + branch_id, + Hash::default(), + vec![create_directory("different")], + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect_err("request id cannot be reused for different content"); + assert_eq!(conflict.code(), tonic::Code::AlreadyExists); + + let occupied = handler( + request( + repository_id, + uuid::Uuid::now_v7(), + branch_id, + fixed_hash(first.revision_signature.as_ref(), "revision").unwrap(), + vec![create_directory("docs")], + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect_err("occupied directory path"); + assert_eq!(occupied.code(), tonic::Code::InvalidArgument); + + let revision = fixed_hash(first.revision_signature.as_ref(), "revision").unwrap(); + let state = State::deserialize(repository.clone(), revision) + .await + .expect("load published revision"); + let docs = state + .find_subnode( + repository.clone(), + ROOT_NODE, + lore_storage::hash_string("docs"), + ) + .await + .expect("docs directory"); + let empty = state + .find_subnode(repository.clone(), docs, lore_storage::hash_string("empty")) + .await + .expect("empty directory"); + let empty = state.node(repository.clone(), empty).await.unwrap(); + assert!(empty.is_directory()); + assert!(!empty.child.is_valid_node_id()); + let file = state + .find_subnode( + repository.clone(), + docs, + lore_storage::hash_string("readme.txt"), + ) + .await + .expect("uploaded file"); + let file = state.node(repository, file).await.unwrap(); + assert_eq!(file.address, file_address); + assert_eq!(file.size, uploaded.size); + + let mut tree_request = Request::new(RevisionTreeRequest { + query: Some(Query::Signature(first.revision_signature.clone())), + path_prefix: None, + max_depth: None, + }); + tree_request.metadata_mut().insert_bin( + lore_transport::grpc::REPOSITORY_ID_KEY, + BinaryMetadataValue::from_bytes(repository_id.data()), + ); + let mut tree = crate::grpc::thinclient::v1::revision_tree::handler( + tree_request, + immutable_store, + mutable_store, + ) + .await + .expect("open thin-client tree") + .into_inner(); + let mut paths = Vec::new(); + while let Some(item) = tree.next().await { + if let Some(Payload::Node(node)) = item.expect("tree item").payload { + paths.push(node.path); + } + } + assert!(paths.iter().any(|path| path == "docs/empty")); + assert!(paths.iter().any(|path| path == "root-empty")); + assert!(paths.iter().any(|path| path == "docs/readme.txt")); + })) + .await; + } + + #[tokio::test] + async fn revision_create_applies_modify_move_and_delete_in_order() { + let repository_id = Partition::from(uuid::Uuid::now_v7()); + let branch_id = BranchId::from(uuid::Uuid::now_v7()); + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("create stores"); + let mut notification = MockNotificationSender::new(); + notification + .expect_branch_pushed() + .times(2) + .returning(|_, _, _, _, _| ()); + let notification = Arc::new(notification); + let hooks = HookDispatcher::empty(); + let instruments = TestInstrumentProvider; + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository_id, + )); + create_root_branch(&repository, branch_id).await; + let old_address = lore_storage::write_content( + immutable_store.clone(), + repository_id, + Context::from(uuid::Uuid::now_v7()), + Bytes::from_static(b"old"), + WriteOptions::default(), + None, + None, + None, + ) + .await + .unwrap() + .address; + let new_address = lore_storage::write_content( + immutable_store.clone(), + repository_id, + old_address.context, + Bytes::from_static(b"new content"), + WriteOptions::default(), + None, + None, + None, + ) + .await + .unwrap() + .address; + let new_fragment = immutable_store + .clone() + .get_metadata(repository_id, new_address) + .await + .unwrap() + .fragment; + + let first = handler( + request( + repository_id, + uuid::Uuid::now_v7(), + branch_id, + Hash::default(), + vec![ + create_directory("src"), + create_directory("src/empty"), + put_file("src/file.txt", old_address), + put_file("src/gone.txt", old_address), + ], + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect("initial revision") + .into_inner(); + let first_hash = fixed_hash(first.revision_signature.as_ref(), "first").unwrap(); + + let second = handler( + request( + repository_id, + uuid::Uuid::now_v7(), + branch_id, + first_hash, + vec![ + put_file("src/file.txt", new_address), + move_path("src/file.txt", "moved.txt"), + delete_path("src/gone.txt"), + delete_path("src/empty"), + ], + ), + immutable_store, + mutable_store, + notification, + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect("ordered edit revision") + .into_inner(); + assert_eq!(second.revision_number, 2); + + let second_hash = fixed_hash(second.revision_signature.as_ref(), "second").unwrap(); + let state = State::deserialize(repository.clone(), second_hash) + .await + .unwrap(); + let moved = state + .find_subnode( + repository.clone(), + ROOT_NODE, + lore_storage::hash_string("moved.txt"), + ) + .await + .expect("moved file"); + let moved = state.node(repository.clone(), moved).await.unwrap(); + assert_eq!(moved.address, new_address); + assert_eq!(moved.size, new_fragment.size_content); + let src = state + .find_subnode( + repository.clone(), + ROOT_NODE, + lore_storage::hash_string("src"), + ) + .await + .expect("src directory"); + assert!( + state + .find_subnode( + repository.clone(), + src, + lore_storage::hash_string("gone.txt") + ) + .await + .is_err() + ); + assert!( + state + .find_subnode(repository, src, lore_storage::hash_string("empty")) + .await + .is_err() + ); + })) + .await; + } + + #[tokio::test] + async fn revision_create_rejects_stale_base_missing_parent_and_operation_limit() { + let repository_id = Partition::from(uuid::Uuid::now_v7()); + let branch_id = BranchId::from(uuid::Uuid::now_v7()); + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("create stores"); + let notification = Arc::new(MockNotificationSender::new()); + let hooks = HookDispatcher::empty(); + let instruments = TestInstrumentProvider; + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository_id, + )); + create_root_branch(&repository, branch_id).await; + + let stale = handler( + request( + repository_id, + uuid::Uuid::now_v7(), + branch_id, + Hash::hash_buffer(b"not the current tip"), + vec![create_directory("dir")], + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect_err("stale base"); + assert_eq!(stale.code(), tonic::Code::FailedPrecondition); + assert!(stale.message().contains("current latest")); + + let missing_parent = handler( + request( + repository_id, + uuid::Uuid::now_v7(), + branch_id, + Hash::default(), + vec![create_directory("missing/child")], + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect_err("missing parent"); + assert_eq!(missing_parent.code(), tonic::Code::InvalidArgument); + + let limited = handler( + request( + repository_id, + uuid::Uuid::now_v7(), + branch_id, + Hash::default(), + vec![create_directory("dir")], + ), + immutable_store, + mutable_store, + notification, + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits { + max_operations: 0, + ..RevisionCreateLimits::default() + }, + &instruments, + ) + .await + .expect_err("operation limit"); + assert_eq!(limited.code(), tonic::Code::ResourceExhausted); + assert!(limited.message().contains("operation count")); + })) + .await; + } + + #[tokio::test] + async fn revision_create_uses_branch_protection_and_service_account_bypass() { + let repository_id = Partition::from(uuid::Uuid::now_v7()); + let branch_id = BranchId::from(uuid::Uuid::now_v7()); + let request_id = uuid::Uuid::now_v7(); + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("create stores"); + let mut notification = MockNotificationSender::new(); + notification + .expect_branch_pushed() + .once() + .return_once(|_, _, _, _, _| ()); + let notification = Arc::new(notification); + let hooks = HookDispatcher::empty(); + let instruments = TestInstrumentProvider; + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository_id, + )); + create_root_branch(&repository, branch_id).await; + branch::protect(repository.clone(), branch_id) + .await + .expect("protect branch"); + let operations = vec![create_directory("protected")]; + + let denied = handler( + request( + repository_id, + request_id, + branch_id, + Hash::default(), + operations.clone(), + ), + immutable_store.clone(), + mutable_store.clone(), + notification.clone(), + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect_err("protected branch"); + assert_eq!(denied.code(), tonic::Code::PermissionDenied); + assert!( + branch::load_latest(repository.clone(), branch_id) + .await + .unwrap_or_default() + .is_zero() + ); + + let published = handler( + service_account_request( + repository_id, + request_id, + branch_id, + Hash::default(), + operations, + ), + immutable_store, + mutable_store, + notification, + &hooks, + DEFAULT_HISTORY_STEP_SIZE, + RevisionListAcceleration::default(), + RevisionCreateLimits::default(), + &instruments, + ) + .await + .expect("service account bypass") + .into_inner(); + assert_eq!(published.revision_number, 1); + })) + .await; + } + + #[test] + fn revision_create_default_limits_are_changeset_not_file_limits() { + let limits = RevisionCreateLimits::default(); + assert_eq!(limits.max_request_bytes, 4 * 1024 * 1024); + assert_eq!(limits.max_operations, 10_000); + } +} diff --git a/lore-server/src/grpc/revision/v1/service.rs b/lore-server/src/grpc/revision/v1/service.rs index 8b3ebd242..128b6262c 100644 --- a/lore-server/src/grpc/revision/v1/service.rs +++ b/lore-server/src/grpc/revision/v1/service.rs @@ -12,12 +12,16 @@ use lore_proto::lore::revision::v1::BranchGetRequest; use lore_proto::lore::revision::v1::BranchGetResponse; use lore_proto::lore::revision::v1::BranchListRequest; use lore_proto::lore::revision::v1::BranchListResponse; +use lore_proto::lore::revision::v1::BranchMergeRequest; +use lore_proto::lore::revision::v1::BranchMergeResponse; use lore_proto::lore::revision::v1::BranchMetadataGetRequest; use lore_proto::lore::revision::v1::BranchMetadataGetResponse; use lore_proto::lore::revision::v1::BranchMetadataSetRequest; use lore_proto::lore::revision::v1::BranchMetadataSetResponse; use lore_proto::lore::revision::v1::BranchPushRequest; use lore_proto::lore::revision::v1::BranchPushResponse; +use lore_proto::lore::revision::v1::RevisionCreateRequest; +use lore_proto::lore::revision::v1::RevisionCreateResponse; use lore_proto::lore::revision::v1::RevisionListRequest; use lore_proto::lore::revision::v1::RevisionListResponse; use lore_proto::lore::revision::v1::revision_service_server::RevisionService; @@ -33,9 +37,12 @@ use super::branch_create; use super::branch_delete; use super::branch_get; use super::branch_list; +use super::branch_merge; use super::branch_metadata_get; use super::branch_metadata_set; use super::branch_push; +use super::revision_create; +use super::revision_create::RevisionCreateLimits; use super::revision_list; use crate::grpc::forwarded_requests::ForwardedRequests; use crate::grpc::timeout_grpc; @@ -78,6 +85,7 @@ pub struct LoreRevisionV1Service { rpc_timeout: Duration, instrument_provider: RevisionServiceInstrumentProvider, revision_list_instruments: RevisionListInstruments, + revision_create_limits: RevisionCreateLimits, } impl LoreRevisionV1Service { @@ -91,6 +99,7 @@ impl LoreRevisionV1Service { acceleration: crate::grpc::server::RevisionListAcceleration, forwarded_requests: Option>, rpc_timeout: Duration, + revision_create_limits: RevisionCreateLimits, ) -> Self { let instrument_provider = RevisionServiceInstrumentProvider; let seconds_in_one_day = 86400f64; @@ -124,6 +133,7 @@ impl LoreRevisionV1Service { rpc_timeout, instrument_provider, revision_list_instruments, + revision_create_limits, } } @@ -239,6 +249,26 @@ impl RevisionService for LoreRevisionV1Service { .await } + async fn branch_merge( + &self, + request: Request, + ) -> Result, Status> { + timeout_grpc( + self.rpc_timeout, + branch_merge::handler( + request, + self.immutable_store.clone(), + self.mutable_store.clone(), + self.notification.clone(), + &self.hook_dispatcher, + self.history_step_size, + self.acceleration, + &self.instrument_provider, + ), + ) + .await + } + async fn branch_metadata_get( &self, request: Request, @@ -286,6 +316,27 @@ impl RevisionService for LoreRevisionV1Service { ) .await } + + async fn revision_create( + &self, + request: Request, + ) -> Result, Status> { + timeout_grpc( + self.rpc_timeout, + revision_create::handler( + request, + self.immutable_store.clone(), + self.mutable_store.clone(), + self.notification.clone(), + &self.hook_dispatcher, + self.history_step_size, + self.acceleration, + self.revision_create_limits, + &self.instrument_provider, + ), + ) + .await + } } #[cfg(test)] diff --git a/lore-server/src/grpc/server.rs b/lore-server/src/grpc/server.rs index c4d2da334..4a52ebb90 100644 --- a/lore-server/src/grpc/server.rs +++ b/lore-server/src/grpc/server.rs @@ -61,6 +61,7 @@ use crate::grpc::thinclient::LoreThinClientV1Service; use crate::grpc::tower::grpc_response_trace::GrpcResponseTraceLayer; use crate::grpc::tower::tracing::LoreTracingLayer; use crate::hooks::HookDispatcher; +use crate::http::server::PresignConfig; use crate::legacy::rpc::environment_service_server::EnvironmentServiceServer; use crate::legacy::rpc::repository_service_server::RepositoryServiceServer; use crate::legacy::rpc::revision_service_server::RevisionServiceServer; @@ -133,6 +134,19 @@ pub struct FeatureSettings { /// holds an `Arc` over a deserialised revision blob; /// the wall-clock benefit saturates well below 64. pub revision_diff_history_walk_concurrency: Option, + /// Optional operational quota for one raw `UploadContent` stream. The + /// streaming writer is memory-bounded without this cap; deployments set it + /// to control storage/bandwidth consumption. Enforced against bytes read, + /// not only the caller's declared size. + pub upload_content_max_bytes: Option, + /// RevisionCreate changeset limits. These bound the decoded request and + /// in-memory `State` retained until publication; they are independent of + /// the raw UploadContent file-size quota. + pub revision_create_max_request_bytes: Option, + pub revision_create_max_operations: Option, + pub revision_create_max_metadata_entries: Option, + pub revision_create_max_metadata_bytes: Option, + pub revision_create_max_path_bytes: Option, } /// Toggles for `RevisionList` acceleration features. Resolved once at @@ -454,6 +468,7 @@ impl GrpcServerBuilder { service_settings: Option, user_agent_filter: Arc, forwarded_requests: Option>, + presign_config: Option, ) -> GrpcServerBuilder { GrpcServerBuilder(MaybeJwtVerifier { environment: self.0.environment, @@ -473,6 +488,7 @@ impl GrpcServerBuilder { service_settings, user_agent_filter, forwarded_requests, + presign_config, }) } } @@ -495,6 +511,7 @@ pub struct MaybeJwtVerifier { service_settings: Option, user_agent_filter: Arc, forwarded_requests: Option>, + presign_config: Option, } impl GrpcServerBuilder { @@ -523,7 +540,8 @@ impl GrpcServerBuilder { self.0.immutable_store.clone(), self.0.local_store.clone(), self.0.mutable_store.clone(), - ); + ) + .with_upload_content_max_bytes(self.0.feature.upload_content_max_bytes); let history_step_size = self .0 .feature @@ -549,6 +567,37 @@ impl GrpcServerBuilder { acceleration, self.0.forwarded_requests.clone(), rpc_timeout, + { + let defaults = + crate::grpc::revision::v1::revision_create::RevisionCreateLimits::default(); + crate::grpc::revision::v1::revision_create::RevisionCreateLimits { + max_request_bytes: self + .0 + .feature + .revision_create_max_request_bytes + .unwrap_or(defaults.max_request_bytes), + max_operations: self + .0 + .feature + .revision_create_max_operations + .unwrap_or(defaults.max_operations), + max_metadata_entries: self + .0 + .feature + .revision_create_max_metadata_entries + .unwrap_or(defaults.max_metadata_entries), + max_metadata_bytes: self + .0 + .feature + .revision_create_max_metadata_bytes + .unwrap_or(defaults.max_metadata_bytes), + max_path_bytes: self + .0 + .feature + .revision_create_max_path_bytes + .unwrap_or(defaults.max_path_bytes), + } + }, ); let revision_diff_config = crate::grpc::thinclient::v1::revision_diff::RevisionDiffConfig { source_cap: self.0.feature.revision_diff_source_cap.unwrap_or( @@ -561,6 +610,7 @@ impl GrpcServerBuilder { self.0.mutable_store.clone(), rpc_timeout, revision_diff_config, + self.0.presign_config, ); let repository_svc = LoreRepositoryService::new( self.0.environment.clone(), diff --git a/lore-server/src/grpc/storage/v1/mod.rs b/lore-server/src/grpc/storage/v1/mod.rs index c2c79dbfd..39aff153d 100644 --- a/lore-server/src/grpc/storage/v1/mod.rs +++ b/lore-server/src/grpc/storage/v1/mod.rs @@ -7,10 +7,12 @@ pub mod get_resolved; pub mod mutable_compare_and_swap; pub mod mutable_load; pub mod mutable_store; +pub mod presign_download; pub mod put; pub mod put_resolved; pub mod query; pub mod service; +pub mod upload_content; pub mod verify; #[cfg(test)] diff --git a/lore-server/src/grpc/storage/v1/presign_download.rs b/lore-server/src/grpc/storage/v1/presign_download.rs new file mode 100644 index 000000000..fec1f3c41 --- /dev/null +++ b/lore-server/src/grpc/storage/v1/presign_download.rs @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::sync::Arc; +use std::time::Duration; + +use lore_base::runtime::LORE_CONTEXT; +use lore_base::types::Address; +use lore_proto::lore::storage::v1 as storage_v1; +use lore_storage::StoreError; +use lore_storage::StoreMatch; +use tonic::Request; +use tonic::Response; +use tonic::Status; +use tracing::Instrument; + +use crate::grpc::extract_correlation_id; +use crate::grpc::get_repository; +use crate::grpc::get_user_id; +use crate::grpc::log_server_error; +use crate::util::setup_execution; + +const DEFAULT_PRESIGN_EXPIRES_IN_SECONDS: u64 = 300; +const MAX_PRESIGN_EXPIRES_IN_SECONDS: u64 = 3600; + +fn map_store_error(err: StoreError) -> Status { + match err { + StoreError::AddressNotFound(_) => Status::not_found("fragment not found"), + StoreError::SlowDown(_) => Status::resource_exhausted("server overloaded, slow down"), + StoreError::Oversized(_) => Status::out_of_range(err.to_string()), + StoreError::NotAuthorized(_) => Status::permission_denied(err.to_string()), + StoreError::NotAuthenticated(_) => Status::unauthenticated(err.to_string()), + StoreError::Maintenance(_) | StoreError::Disconnected(_) => { + Status::unavailable(err.to_string()) + } + StoreError::NotSupported(_) => Status::unimplemented(err.to_string()), + StoreError::NoRemote(_) + | StoreError::NotFound(_) + | StoreError::PayloadNotFound(_) + | StoreError::Internal(_) => Status::internal(err.to_string()), + } +} + +#[tracing::instrument(name = "StorageServiceV1::PresignDownload", skip_all)] +pub async fn handler( + request: Request, + immutable_store: Arc, +) -> Result, Status> { + let repository = get_repository(request.metadata())?; + let user_id = get_user_id(request.extensions()); + let correlation_id = extract_correlation_id(&request).unwrap_or_default(); + + let execution = setup_execution(module_path!(), correlation_id, user_id); + + LORE_CONTEXT + .scope( + execution, + async move { + let req = request.into_inner(); + + if req.addresses.len() > crate::protocol::storage::query::MAX_FRAGMENTS { + return Err(Status::invalid_argument(format!( + "too many addresses: {} exceeds limit {}", + req.addresses.len(), + crate::protocol::storage::query::MAX_FRAGMENTS, + ))); + } + + let expires_in_seconds = if req.expires_in_seconds == 0 { + DEFAULT_PRESIGN_EXPIRES_IN_SECONDS + } else { + req.expires_in_seconds.min(MAX_PRESIGN_EXPIRES_IN_SECONDS) + }; + + let addresses: Vec
= req.addresses.iter().map(Address::from).collect(); + let downloads = immutable_store + .presign_downloads( + repository, + &addresses, + StoreMatch::MatchFull, + Duration::from_secs(expires_in_seconds), + ) + .await + .map_err(map_store_error) + .inspect_err(log_server_error)?; + + let downloads = downloads + .into_iter() + .map(|download| storage_v1::PresignedDownload { + address: Some(download.address.into()), + fragment: Some(download.fragment.into()), + url: download.url, + expires_at_epoch_seconds: download.expires_at_epoch_seconds, + }) + .collect(); + + Ok(Response::new(storage_v1::PresignDownloadResponse { + downloads, + })) + } + .in_current_span(), + ) + .await +} diff --git a/lore-server/src/grpc/storage/v1/service.rs b/lore-server/src/grpc/storage/v1/service.rs index 46a8f19ee..14a181faf 100644 --- a/lore-server/src/grpc/storage/v1/service.rs +++ b/lore-server/src/grpc/storage/v1/service.rs @@ -17,11 +17,13 @@ use super::get_resolved::GetResolvedResponseStream; use super::mutable_compare_and_swap; use super::mutable_load; use super::mutable_store; +use super::presign_download; use super::put; use super::put::PutResponseStream; use super::put_resolved; use super::put_resolved::PutResolvedResponseStream; use super::query; +use super::upload_content; use super::verify; use crate::grpc::storage_service::LoreStorageService; @@ -84,6 +86,18 @@ impl StorageServiceV1 for LoreStorageService { put::handler(request, self.immutable_store().clone(), self).await } + async fn upload_content( + &self, + request: Request>, + ) -> Result, Status> { + upload_content::handler( + request, + self.immutable_store().clone(), + self.upload_content_max_bytes(), + ) + .await + } + async fn query( &self, request: Request, @@ -91,6 +105,13 @@ impl StorageServiceV1 for LoreStorageService { query::handler(request, self.immutable_store().clone()).await } + async fn presign_download( + &self, + request: Request, + ) -> Result, Status> { + presign_download::handler(request, self.immutable_store().clone()).await + } + type CopyStream = CopyResponseStream; async fn copy( diff --git a/lore-server/src/grpc/storage/v1/upload_content.rs b/lore-server/src/grpc/storage/v1/upload_content.rs new file mode 100644 index 000000000..473836fa7 --- /dev/null +++ b/lore-server/src/grpc/storage/v1/upload_content.rs @@ -0,0 +1,267 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::io; +use std::sync::Arc; + +use futures::StreamExt; +use lore_base::runtime::LORE_CONTEXT; +use lore_base::types::Context; +use lore_proto::lore::storage::v1 as storage_v1; +use lore_proto::lore::storage::v1::upload_content_request::Part; +use lore_storage::ContentStreamError; +use lore_storage::options::WriteOptions; +use tokio_util::io::StreamReader; +use tonic::Request; +use tonic::Response; +use tonic::Status; +use tonic::Streaming; + +use crate::grpc::extract_correlation_id; +use crate::grpc::get_repository; +use crate::grpc::get_user_id; +use crate::util::setup_execution; + +fn uuid_v7(bytes: &[u8], field: &str) -> Result { + let id = uuid::Uuid::from_slice(bytes) + .map_err(|error| Status::invalid_argument(format!("invalid {field}: {error}")))?; + if id.get_version_num() != 7 { + return Err(Status::invalid_argument(format!( + "{field} must be a UUIDv7" + ))); + } + Ok(id) +} + +fn storage_status(error: ContentStreamError, max_bytes: Option) -> Status { + let ContentStreamError::Storage(error) = error else { + return Status::invalid_argument(error.to_string()); + }; + if error.is_oversized() { + match max_bytes { + Some(max_bytes) => Status::resource_exhausted(format!( + "upload exceeds configured limit of {max_bytes} bytes" + )), + None => Status::resource_exhausted(error.to_string()), + } + } else if error.is_slow_down() { + Status::resource_exhausted(error.to_string()) + } else { + Status::internal(error.to_string()) + } +} + +/// Ingest one raw file stream. The first message is a required header; every +/// remaining message must carry bytes. The stream is adapted directly to +/// `AsyncRead`, preserving tonic/HTTP2 backpressure all the way into FastCDC. +#[tracing::instrument(name = "StorageServiceV1::UploadContent", skip_all)] +pub async fn handler( + request: Request>, + immutable_store: Arc, + max_bytes: Option, +) -> Result, Status> { + let repository = get_repository(request.metadata())?; + let user_id = get_user_id(request.extensions()); + let correlation_id = extract_correlation_id(&request).unwrap_or_default(); + let stream = request.into_inner(); + let execution = setup_execution(module_path!(), correlation_id, user_id); + let response = LORE_CONTEXT + .scope( + execution, + ingest(stream, immutable_store, repository, max_bytes), + ) + .await?; + Ok(Response::new(response)) +} + +pub(crate) async fn ingest( + mut stream: S, + immutable_store: Arc, + repository: lore_base::types::Partition, + max_bytes: Option, +) -> Result +where + S: futures::Stream> + Unpin, +{ + let first = stream + .next() + .await + .transpose()? + .ok_or_else(|| Status::invalid_argument("upload stream is missing its header"))?; + let header = match first.part { + Some(Part::Header(header)) => header, + _ => { + return Err(Status::invalid_argument( + "the first upload message must be the header", + )); + } + }; + let _request_id = uuid_v7(header.request_id.as_ref(), "request_id")?; + let context = if header.file_id.is_empty() { + Context::from(uuid::Uuid::now_v7()) + } else { + Context::from(uuid_v7(header.file_id.as_ref(), "file_id")?) + }; + if let (Some(expected_size), Some(max_bytes)) = (header.expected_size, max_bytes) + && expected_size > max_bytes + { + return Err(Status::resource_exhausted(format!( + "declared upload size {expected_size} exceeds configured limit of {max_bytes} bytes" + ))); + } + + let bytes = stream.map(|message| match message { + Ok(message) => match message.part { + Some(Part::Chunk(chunk)) if !chunk.is_empty() => Ok(chunk), + Some(Part::Chunk(_)) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "upload chunks must not be empty", + )), + Some(Part::Header(_)) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "upload header may appear only once and must be first", + )), + None => Err(io::Error::new( + io::ErrorKind::InvalidData, + "upload message is missing its part", + )), + }, + Err(status) => Err(io::Error::other(status.to_string())), + }); + let reader = StreamReader::new(bytes); + let result = lore_storage::write_content_stream( + immutable_store, + repository, + context, + reader, + header.expected_size, + max_bytes, + WriteOptions::default().with_local_cache_priority(), + None, + None, + ) + .await + .map_err(|error| storage_status(error, max_bytes))?; + + Ok(storage_v1::UploadContentResponse { + address: Some(result.0.into()), + size: result.2, + }) +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use futures::stream; + use lore_base::types::Address; + use lore_base::types::Context; + use lore_base::types::Partition; + use lore_proto::lore::storage::v1::UploadContentHeader; + use lore_proto::lore::storage::v1::UploadContentRequest; + use lore_proto::lore::storage::v1::upload_content_request::Part; + use lore_storage::options::ReadOptions; + + use super::*; + use crate::store::test_store_create; + + fn header(request_id: uuid::Uuid, file_id: uuid::Uuid, size: u64) -> UploadContentRequest { + UploadContentRequest { + part: Some(Part::Header(UploadContentHeader { + file_id: Bytes::copy_from_slice(file_id.as_bytes()), + expected_size: Some(size), + request_id: Bytes::copy_from_slice(request_id.as_bytes()), + })), + } + } + + fn chunk(bytes: Bytes) -> UploadContentRequest { + UploadContentRequest { + part: Some(Part::Chunk(bytes)), + } + } + + #[tokio::test] + async fn ingest_streams_large_content_and_is_address_idempotent() { + let repository = Partition::from(uuid::Uuid::now_v7()); + let file_id = uuid::Uuid::now_v7(); + let request_id = uuid::Uuid::now_v7(); + let payload = Bytes::from(vec![0x5a; lore_storage::FRAGMENT_SIZE_THRESHOLD * 3 + 19]); + let (immutable, _mutable, execution) = test_store_create().await.expect("stores"); + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let messages = vec![ + Ok(header(request_id, file_id, payload.len() as u64)), + Ok(chunk(payload.slice(..100_000))), + Ok(chunk(payload.slice(100_000..))), + ]; + let first = ingest(stream::iter(messages), immutable.clone(), repository, None) + .await + .expect("first upload"); + assert_eq!(first.size, payload.len() as u64); + + let retry = ingest( + stream::iter(vec![ + Ok(header(request_id, file_id, payload.len() as u64)), + Ok(chunk(payload.clone())), + ]), + immutable.clone(), + repository, + None, + ) + .await + .expect("idempotent upload retry"); + assert_eq!(retry.address, first.address); + + let address = Address::from(first.address.expect("address")); + assert_eq!(address.context, Context::from(file_id)); + let (_, restored) = lore_storage::read( + immutable, + repository, + address, + None, + ReadOptions::default(), + None, + ) + .await + .expect("read uploaded content"); + assert_eq!(restored, payload); + })) + .await; + } + + #[tokio::test] + async fn ingest_enforces_actual_size_and_header_framing() { + let repository = Partition::from(uuid::Uuid::now_v7()); + let file_id = uuid::Uuid::now_v7(); + let request_id = uuid::Uuid::now_v7(); + let (immutable, _mutable, execution) = test_store_create().await.expect("stores"); + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let oversized = ingest( + stream::iter(vec![ + Ok(header(request_id, file_id, 4)), + Ok(chunk(Bytes::from_static(b"12345"))), + ]), + immutable.clone(), + repository, + Some(4), + ) + .await + .expect_err("actual size over limit"); + assert_eq!(oversized.code(), tonic::Code::ResourceExhausted); + + let repeated_header = ingest( + stream::iter(vec![ + Ok(header(request_id, file_id, 1)), + Ok(header(request_id, file_id, 1)), + ]), + immutable, + repository, + None, + ) + .await + .expect_err("header may appear only once"); + assert_eq!(repeated_header.code(), tonic::Code::InvalidArgument); + })) + .await; + } +} diff --git a/lore-server/src/grpc/storage_service.rs b/lore-server/src/grpc/storage_service.rs index d6652a9fd..bc7a17347 100644 --- a/lore-server/src/grpc/storage_service.rs +++ b/lore-server/src/grpc/storage_service.rs @@ -73,6 +73,7 @@ pub struct LoreStorageService { immutable_store: Arc, local_store: Arc, mutable_store: Arc, + upload_content_max_bytes: Option, } impl LoreStorageService { @@ -85,9 +86,19 @@ impl LoreStorageService { immutable_store, local_store, mutable_store, + upload_content_max_bytes: None, } } + pub fn with_upload_content_max_bytes(mut self, max_bytes: Option) -> Self { + self.upload_content_max_bytes = max_bytes; + self + } + + pub fn upload_content_max_bytes(&self) -> Option { + self.upload_content_max_bytes + } + pub fn local_immutable_store(&self) -> &Arc { &self.local_store } diff --git a/lore-server/src/grpc/thinclient/v1/content_diff.rs b/lore-server/src/grpc/thinclient/v1/content_diff.rs new file mode 100644 index 000000000..f75df48a9 --- /dev/null +++ b/lore-server/src/grpc/thinclient/v1/content_diff.rs @@ -0,0 +1,658 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::mem::size_of; +use std::pin::Pin; +use std::sync::Arc; + +use bytes::Bytes; +use lore_base::lore_spawn; +use lore_base::runtime::LORE_CONTEXT; +use lore_base::types::Address; +use lore_base::types::Context; +use lore_base::types::Hash; +use lore_proto::lore::thin_client::v1 as thin_client_v1; +use lore_proto::lore::thin_client::v1::ContentDiffRequest; +use lore_proto::lore::thin_client::v1::ContentDiffResponse; +use lore_proto::lore::thin_client::v1::content_diff_response::Payload; +use lore_revision::file::diff::DEFAULT_CONTEXT_LINES; +use lore_revision::immutable; +use lore_revision::immutable::read_options_from_repository; +use lore_revision::infer::infer_is_diffable_by_slice; +use lore_revision::merge::merge3_text; +use lore_revision::repository::RepositoryContext; +use lore_revision::util::encoding::decode_text_for_display; +use lore_revision::util::encoding::is_utf16_bom; +use lore_telemetry::tracing::fields::REPOSITORY_ID; +use tokio::sync::mpsc; +use tokio_stream::Stream; +use tokio_stream::wrappers::ReceiverStream; +use tonic::Request; +use tonic::Response; +use tonic::Status; +use tracing::Instrument; +use tracing::debug; +use tracing::warn; + +use crate::grpc::extract_correlation_id; +use crate::grpc::get_repository; +use crate::grpc::get_user_id; +use crate::grpc::warn_error_to_status; +use crate::util::setup_execution; + +type ContentDiffStream = + Pin> + Send + 'static>>; + +const CONTENT_DIFF_CHUNK_SIZE: usize = 64 * 1024; + +/// `lore.thin_client.v1.ThinClientService.ContentDiff` handler. +/// +/// Server-streams a `ContentDiffHeader` first, then zero or more text +/// chunks whose concatenation is the unified diff. The request is +/// address-only: empty bytes mean an absent side, 32 bytes mean a CAS hash +/// in the current repository context, and 48 bytes mean a full +/// `Address { hash, context }`. +/// +/// Content is loaded up-front so storage failures surface as gRPC +/// `Status` before the stream opens. The stream itself only emits the +/// already-rendered header and diff chunks. +#[tracing::instrument(name = "ContentDiff::v1::handle", skip_all)] +pub async fn handler( + request: Request, + immutable_store: Arc, + mutable_store: Arc, +) -> Result, Status> { + let repository_id = get_repository(request.metadata())?; + let user_id = get_user_id(request.extensions()); + let correlation_id = extract_correlation_id(&request).unwrap_or_default(); + let req = request.into_inner(); + + let execution = setup_execution(module_path!(), correlation_id, user_id); + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store, + mutable_store, + repository_id, + )); + + LORE_CONTEXT + .scope(execution, async move { + let result = build_content_diff(repository, req).await?; + let (tx, rx) = mpsc::channel(8); + + lore_spawn!( + async move { + stream_content_diff(result, tx).await; + } + .in_current_span() + ); + + let stream: ContentDiffStream = Box::pin(ReceiverStream::from(rx)); + Ok(Response::new(stream)) + }) + .await +} + +async fn build_content_diff( + repository: Arc, + req: ContentDiffRequest, +) -> Result { + let from = read_content_side(repository.clone(), &req.address_from, "from").await?; + let to = read_content_side(repository.clone(), &req.address_to, "to").await?; + let base = match req.address_base.as_ref().filter(|bytes| !bytes.is_empty()) { + Some(bytes) => Some(read_content_side(repository, bytes, "base").await?), + None => None, + }; + + let any_binary = + from.is_binary() || to.is_binary() || base.as_ref().is_some_and(|b| b.is_binary()); + if any_binary { + return Ok(ContentDiffResult { + header: thin_client_v1::ContentDiffHeader { + binary: true, + ..Default::default() + }, + diff: None, + }); + } + + let options = DiffRenderOptions { + context_lines: req.context_lines.unwrap_or(DEFAULT_CONTEXT_LINES), + ignore_whitespace_eol: req.ignore_whitespace_eol, + ignore_whitespace_inline: req.ignore_whitespace_inline, + }; + + let (diff, has_conflicts, conflict_count) = match base { + Some(base) => build_three_way_diff(base.text(), from.text(), to.text(), options), + None => ( + build_unified_patch(from.text(), to.text(), "from", "to", options), + false, + 0, + ), + }; + let (lines_added, lines_deleted) = diff.as_deref().map_or((0, 0), count_patch_stats); + let truncated = diff + .as_ref() + .is_some_and(|diff| req.max_diff_size.is_some_and(|max| diff.len() as u64 > max)); + let diff = if truncated { None } else { diff }; + + Ok(ContentDiffResult { + header: thin_client_v1::ContentDiffHeader { + lines_added, + lines_deleted, + binary: false, + truncated, + has_conflicts, + conflict_count, + }, + diff, + }) +} + +async fn read_content_side( + repository: Arc, + bytes: &Bytes, + label: &'static str, +) -> Result { + let Some(address) = parse_content_address(bytes, label)? else { + return Ok(DiffContent::empty()); + }; + + if address.is_zero() { + return Ok(DiffContent::empty()); + } + + let content = immutable::read( + repository.clone(), + address, + None, + read_options_from_repository(&repository) + .with_decompress() + .with_verify() + .no_remote(), + ) + .await + .map_err(|err| { + if err.is_address_not_found() || err.is_payload_not_found() || err.is_not_found() { + Status::not_found(format!("{label} content not found")) + } else { + warn!( + {REPOSITORY_ID} = %repository.id, + side = label, + ?address, + ?err, + "Failed to read content for thin-client diff", + ); + warn_error_to_status(&err, |e| Status::internal(e.to_string())) + } + })?; + + Ok(make_diff_content(&content)) +} + +fn parse_content_address(bytes: &Bytes, label: &'static str) -> Result, Status> { + match bytes.len() { + 0 => Ok(None), + len if len == size_of::() => Ok(Some(Address::zero_context_hash(Hash::from(bytes)))), + len if len == size_of::
() => { + let hash = Hash::from(&bytes[..size_of::()]); + let context = Context::from(&bytes[size_of::()..]); + Ok(Some(Address { hash, context })) + } + len => Err(Status::invalid_argument(format!( + "{label} address must be empty, {} hash bytes, or {} address bytes; got {len}", + size_of::(), + size_of::
(), + ))), + } +} + +fn build_three_way_diff( + base: &str, + from: &str, + to: &str, + options: DiffRenderOptions, +) -> (Option, bool, u32) { + match merge3_text(base, from, to, Some("base"), Some("from"), Some("to")) { + Ok(merged) => ( + build_unified_patch(from, &merged, "from", "merged", options), + false, + 0, + ), + Err(conflicted) => { + let conflict_count = count_conflicts(&conflicted); + ( + build_unified_patch(from, &conflicted, "from", "merged", options), + true, + conflict_count, + ) + } + } +} + +async fn stream_content_diff( + result: ContentDiffResult, + tx: mpsc::Sender>, +) { + if tx + .send(Ok(ContentDiffResponse { + payload: Some(Payload::Header(result.header)), + })) + .await + .is_err() + { + debug!("ContentDiff receiver dropped before header"); + return; + } + + let Some(diff) = result.diff else { + debug!("ContentDiff complete: header only"); + return; + }; + + let mut emitted = 0; + for chunk in utf8_chunks(&diff, CONTENT_DIFF_CHUNK_SIZE) { + if tx + .send(Ok(ContentDiffResponse { + payload: Some(Payload::Chunk(thin_client_v1::ContentDiffChunkResponse { + diff: chunk.to_string(), + })), + })) + .await + .is_err() + { + debug!(emitted, "ContentDiff receiver dropped mid-stream"); + return; + } + emitted += 1; + } + + debug!(emitted, "ContentDiff complete"); +} + +struct ContentDiffResult { + header: thin_client_v1::ContentDiffHeader, + diff: Option, +} + +#[derive(Clone, Copy)] +struct DiffRenderOptions { + context_lines: u32, + ignore_whitespace_eol: bool, + ignore_whitespace_inline: bool, +} + +fn build_unified_patch( + old: &str, + new: &str, + from_label: &str, + to_label: &str, + options: DiffRenderOptions, +) -> Option { + let patch = if options.ignore_whitespace_eol || options.ignore_whitespace_inline { + format_patch_preserving_originals( + old, + new, + options.context_lines, + options.ignore_whitespace_eol, + options.ignore_whitespace_inline, + )? + } else { + // diffy's `Display`/`to_string()` defaults to `suppress_blank_empty: true`, + // which drops the leading space on blank context lines (bare `\n`). Standard + // unified-diff parsers require every hunk-body line to start with a sentinel + // (' ', '+', '-', '\'), so format explicitly with suppression disabled. + let patch = diffy::DiffOptions::new() + .set_context_len(options.context_lines as usize) + .create_patch(old, new); + let s = diffy::PatchFormatter::new() + .suppress_blank_empty(false) + .fmt_patch(&patch) + .to_string(); + if s.ends_with("+++ modified\n") { + return None; + } + s + }; + let patch = patch.replace("--- original", &format!("--- {from_label}")); + let patch = patch.replace("+++ modified", &format!("+++ {to_label}")); + Some(patch) +} + +/// Normalise `old` and `new` per-line for comparison, run diffy, then re-emit +/// the unified diff with original (un-normalised) line content. Returns +/// `None` when no hunks remain after normalisation (i.e. the files are equal +/// under the selected whitespace rules). +/// +/// The line count of each normalised side equals the line count of the +/// original side, so diffy's 1-based hunk line numbers index back into the +/// original line arrays correctly. +fn format_patch_preserving_originals( + old: &str, + new: &str, + context_lines: u32, + ignore_eol: bool, + ignore_inline: bool, +) -> Option { + let old_lines: Vec<&str> = old.split_inclusive('\n').collect(); + let new_lines: Vec<&str> = new.split_inclusive('\n').collect(); + + let old_norm: String = old_lines + .iter() + .map(|l| normalise_line(l, ignore_eol, ignore_inline)) + .collect(); + let new_norm: String = new_lines + .iter() + .map(|l| normalise_line(l, ignore_eol, ignore_inline)) + .collect(); + + let patch = diffy::DiffOptions::new() + .set_context_len(context_lines as usize) + .create_patch(&old_norm, &new_norm); + + if patch.hunks().is_empty() { + return None; + } + + let mut out = String::new(); + out.push_str("--- original\n"); + out.push_str("+++ modified\n"); + + for hunk in patch.hunks() { + out.push_str(&format!( + "@@ -{} +{} @@\n", + hunk.old_range(), + hunk.new_range() + )); + let mut old_idx = hunk.old_range().start(); + let mut new_idx = hunk.new_range().start(); + + for line in hunk.lines() { + match line { + diffy::Line::Context(_) => { + let orig = old_lines + .get(old_idx.saturating_sub(1)) + .copied() + .unwrap_or(""); + write_patch_line(&mut out, ' ', orig); + old_idx += 1; + new_idx += 1; + } + diffy::Line::Delete(_) => { + let orig = old_lines + .get(old_idx.saturating_sub(1)) + .copied() + .unwrap_or(""); + write_patch_line(&mut out, '-', orig); + old_idx += 1; + } + diffy::Line::Insert(_) => { + let orig = new_lines + .get(new_idx.saturating_sub(1)) + .copied() + .unwrap_or(""); + write_patch_line(&mut out, '+', orig); + new_idx += 1; + } + } + } + } + + Some(out) +} + +/// Per-line normalisation. Keeps the trailing `\n` (if present) so the line +/// count is preserved between original and normalised content. `\r` is +/// treated as whitespace so the EOL/inline rules apply uniformly to LF and +/// CRLF inputs. +fn normalise_line(line: &str, ignore_eol: bool, ignore_inline: bool) -> String { + let (content, terminator) = match line.strip_suffix('\n') { + Some(rest) => (rest, "\n"), + None => (line, ""), + }; + + let mut work = if ignore_inline { + collapse_inline_whitespace(content) + } else { + content.to_string() + }; + + if ignore_eol { + let trimmed_len = work.trim_end_matches([' ', '\t', '\r']).len(); + work.truncate(trimmed_len); + } + + work.push_str(terminator); + work +} + +/// Collapses runs of ASCII space/tab/CR to a single space. Does not invent +/// whitespace where there was none, and does not touch newline characters +/// (callers strip the terminator before invoking). Folding `\r` in keeps +/// LF and CRLF line endings on equal footing for inline comparison. +fn collapse_inline_whitespace(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_ws = false; + for c in s.chars() { + if c == ' ' || c == '\t' || c == '\r' { + if !in_ws { + out.push(' '); + in_ws = true; + } + } else { + out.push(c); + in_ws = false; + } + } + out +} + +/// Writes one unified-diff line. Every hunk-body line begins with its sentinel +/// (' ', '+', '-') — including blank context lines, which are emitted as `" \n"` +/// so standard unified-diff parsers count them as rows. Lines without a trailing +/// `\n` get a `\ No newline at end of file` marker. +fn write_patch_line(out: &mut String, sign: char, line: &str) { + out.push(sign); + out.push_str(line); + if !line.ends_with('\n') { + out.push('\n'); + out.push_str("\\ No newline at end of file\n"); + } +} + +/// One side of a diff: either decoded display text, or a marker that the raw +/// bytes were detected as binary (non-text) content. Binary content carries no +/// text — it is never rendered through the diff/merge pipeline. +enum DiffContent { + Text(String), + Binary, +} + +impl DiffContent { + /// An absent side (file missing on this revision / `/dev/null`). Treated as + /// empty text, never binary. + fn empty() -> Self { + DiffContent::Text(String::new()) + } + + fn is_binary(&self) -> bool { + matches!(self, DiffContent::Binary) + } + + /// The decoded text for a text side; `""` for binary. Callers short-circuit + /// on `is_binary()` before reaching this, so the binary case is never read + /// in practice. + fn text(&self) -> &str { + match self { + DiffContent::Text(s) => s, + DiffContent::Binary => "", + } + } +} + +/// Build display content from raw bytes. +/// +/// An empty buffer (an absent side) is text, not binary. UTF-16 BOM input is +/// exempt from the binary check: `decode_text_for_display` renders it as +/// readable text, and the diff path intentionally shows UTF-16 as text — unlike +/// the merge path, where `infer_is_diffable_by_slice` treats UTF-16 as binary +/// to preserve bytes. Everything else (null bytes, non-text MIME, Unreal +/// packages, invalid UTF-8) is classified as binary, and its bytes are never +/// decoded. +fn make_diff_content(bytes: &[u8]) -> DiffContent { + if !bytes.is_empty() && !is_utf16_bom(bytes) && !infer_is_diffable_by_slice(bytes) { + DiffContent::Binary + } else { + DiffContent::Text(decode_text_for_display(bytes)) + } +} + +fn count_patch_stats(patch: &str) -> (u64, u64) { + let mut lines_added = 0; + let mut lines_deleted = 0; + + for line in patch.lines() { + if line.starts_with("+++") || line.starts_with("---") { + continue; + } + if line.starts_with('+') { + lines_added += 1; + } else if line.starts_with('-') { + lines_deleted += 1; + } + } + + (lines_added, lines_deleted) +} + +fn count_conflicts(text: &str) -> u32 { + text.lines() + .filter(|line| line.starts_with("<<<<<<< ")) + .count() as u32 +} + +fn utf8_chunks(text: &str, max_bytes: usize) -> Vec<&str> { + let max_bytes = max_bytes.max(1); + let mut chunks = Vec::new(); + let mut start = 0; + + while start < text.len() { + let mut end = (start + max_bytes).min(text.len()); + while end > start && !text.is_char_boundary(end) { + end -= 1; + } + if end == start { + end = start + + text[start..] + .chars() + .next() + .map(char::len_utf8) + .unwrap_or_default(); + } + chunks.push(&text[start..end]); + start = end; + } + + chunks +} + +#[cfg(test)] +mod tests { + use super::*; + + fn options() -> DiffRenderOptions { + DiffRenderOptions { + context_lines: DEFAULT_CONTEXT_LINES, + ignore_whitespace_eol: false, + ignore_whitespace_inline: false, + } + } + + #[test] + fn parse_content_address_accepts_empty() { + assert!( + parse_content_address(&Bytes::new(), "from") + .expect("parse") + .is_none() + ); + } + + #[test] + fn parse_content_address_accepts_hash_only() { + let bytes = Bytes::from(vec![0x11; size_of::()]); + let address = parse_content_address(&bytes, "from") + .expect("parse") + .expect("address"); + + assert_eq!(address.hash, Hash::from(&bytes)); + assert_eq!(address.context, Context::default()); + } + + #[test] + fn parse_content_address_accepts_full_address() { + let mut bytes = vec![0x11; size_of::()]; + bytes.extend([0x22; size_of::()]); + let bytes = Bytes::from(bytes); + let address = parse_content_address(&bytes, "to") + .expect("parse") + .expect("address"); + + assert_eq!(address.hash, Hash::from(&bytes[..size_of::()])); + assert_eq!(address.context, Context::from(&bytes[size_of::()..])); + } + + #[test] + fn parse_content_address_rejects_bad_length() { + let err = + parse_content_address(&Bytes::from_static(b"short"), "base").expect_err("bad length"); + assert_eq!(err.code(), tonic::Code::InvalidArgument); + } + + #[test] + fn two_way_diff_reports_stats() { + let patch = build_unified_patch("old\n", "new\n", "from", "to", options()).expect("patch"); + let (added, deleted) = count_patch_stats(&patch); + + assert!(patch.contains("--- from\n")); + assert!(patch.contains("+++ to\n")); + assert_eq!((added, deleted), (1, 1)); + } + + #[test] + fn unchanged_diff_has_no_patch() { + assert!(build_unified_patch("same\n", "same\n", "from", "to", options()).is_none()); + } + + #[test] + fn ignored_whitespace_can_remove_patch() { + let mut options = options(); + options.ignore_whitespace_eol = true; + assert!(build_unified_patch("same \n", "same\n", "from", "to", options).is_none()); + } + + #[test] + fn three_way_conflict_sets_conflict_count() { + let (patch, has_conflicts, conflict_count) = + build_three_way_diff("base\n", "from\n", "to\n", options()); + + assert!(patch.expect("patch").contains("<<<<<<< from")); + assert!(has_conflicts); + assert_eq!(conflict_count, 1); + } + + #[test] + fn binary_content_is_detected() { + assert!(make_diff_content(&[0x00, 0x01, 0x02, 0xFF, 0xFE, 0x00]).is_binary()); + } + + #[test] + fn utf8_chunks_split_on_character_boundaries() { + let text = "aa🙂bb"; + let chunks = utf8_chunks(text, 3); + + assert_eq!(chunks.concat(), text); + assert!( + chunks + .iter() + .all(|chunk| std::str::from_utf8(chunk.as_bytes()).is_ok()) + ); + } +} diff --git a/lore-server/src/grpc/thinclient/v1/helpers.rs b/lore-server/src/grpc/thinclient/v1/helpers.rs index 58b482dab..804ea3643 100644 --- a/lore-server/src/grpc/thinclient/v1/helpers.rs +++ b/lore-server/src/grpc/thinclient/v1/helpers.rs @@ -9,6 +9,7 @@ use lore_base::types::Hash; use lore_proto::lore::model::v1 as model_v1; use lore_proto::lore::thin_client::v1 as thin_client_v1; use lore_proto::lore::thin_client::v1::revision_diff_request; +use lore_proto::lore::thin_client::v1::revision_file_download_request; use lore_proto::lore::thin_client::v1::revision_info_request; use lore_proto::lore::thin_client::v1::revision_tree_request; use lore_revision::branch; @@ -59,6 +60,15 @@ impl From for RevisionSpec { } } +impl From for RevisionSpec { + fn from(query: revision_file_download_request::Query) -> Self { + match query { + revision_file_download_request::Query::Signature(sig) => Self::Signature(sig), + revision_file_download_request::Query::Identifier(id) => Self::Identifier(id), + } + } +} + impl From for RevisionSpec { fn from(query: revision_diff_request::QueryFrom) -> Self { match query { @@ -241,7 +251,7 @@ fn file_action_to_v1_action(action: FileAction) -> thin_client_v1::Action { /// Convert an internal `NodeChange` into a v1 `DiffChange`. The /// `to.flags` drive `node_type` for non-delete actions; for deletes /// `from.flags` is the surviving record of what the path used to be. -/// `content_from` / `content_to` carry the from / to side's CAS hash, +/// `content_from` / `content_to` carry the from / to side's full `Address`, /// or empty bytes for ADD (no from) and DELETE (no to). /// /// `link_repository_index` is passed through verbatim; the handler @@ -263,12 +273,12 @@ pub(super) async fn node_change_to_diff_change( let content_from = if action == thin_client_v1::Action::Add { Bytes::new() } else { - change.from.address.hash.into() + change.from.address.into() }; let content_to = if action == thin_client_v1::Action::Delete { Bytes::new() } else { - change.to.address.hash.into() + change.to.address.into() }; thin_client_v1::DiffChange { path: change.path.to_string(), @@ -309,12 +319,14 @@ pub(super) fn link_pin_change_to_diff_change( /// take separate indices: they can land in different partitions. pub(super) async fn diff_conflict_from_pair( pair: &(NodeChange, NodeChange), + conflict_id: lore_base::types::Hash, link_repository_index_from: u32, link_repository_index_to: u32, ) -> thin_client_v1::DiffConflict { thin_client_v1::DiffConflict { change_from: Some(node_change_to_diff_change(&pair.0, link_repository_index_from).await), change_to: Some(node_change_to_diff_change(&pair.1, link_repository_index_to).await), + conflict_id: conflict_id.into(), } } @@ -409,6 +421,15 @@ mod tests { assert_eq!(mapped.link_repository_index, 0); assert_eq!(mapped.path, "dir/file.txt"); assert_eq!(mapped.action, thin_client_v1::Action::Add as i32); + assert!( + mapped.content_from.is_empty(), + "ADD has no from-side content" + ); + assert_eq!( + Address::from(&mapped.content_to), + change.to.address, + "to-side content carries the full address" + ); let mapped = node_change_to_diff_change(&change, 7).await; assert_eq!(mapped.link_repository_index, 7); @@ -434,6 +455,15 @@ mod tests { let mapped = node_change_to_diff_change(&change, 0).await; assert_eq!(mapped.node_type, thin_client_v1::NodeType::Link as i32); + assert_eq!( + Address::from(&mapped.content_from), + change.from.address, + "delete keeps the full from-side address" + ); + assert!( + mapped.content_to.is_empty(), + "DELETE has no to-side content" + ); } #[tokio::test] @@ -441,7 +471,7 @@ mod tests { let from = make_change(lore_revision::change::FileAction::Keep); let to = make_change(lore_revision::change::FileAction::Keep); - let mapped = diff_conflict_from_pair(&(from, to), 0, 3).await; + let mapped = diff_conflict_from_pair(&(from, to), Hash::from_u64(7), 0, 3).await; assert_eq!( mapped.change_from.as_ref().unwrap().link_repository_index, 0, @@ -452,6 +482,7 @@ mod tests { 3, "to-half carries its own index, distinct from from-half", ); + assert_eq!(Hash::from(&mapped.conflict_id), Hash::from_u64(7)); } fn make_pin_change() -> LinkPinChange { diff --git a/lore-server/src/grpc/thinclient/v1/mod.rs b/lore-server/src/grpc/thinclient/v1/mod.rs index 0044161cd..b94d4ce8c 100644 --- a/lore-server/src/grpc/thinclient/v1/mod.rs +++ b/lore-server/src/grpc/thinclient/v1/mod.rs @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: 2026 Epic Games, Inc. // SPDX-License-Identifier: MIT +pub mod content_diff; mod helpers; pub mod revision_diff; +pub mod revision_file_download; pub mod revision_info; pub mod revision_tree; pub mod service; diff --git a/lore-server/src/grpc/thinclient/v1/revision_diff.rs b/lore-server/src/grpc/thinclient/v1/revision_diff.rs index 939cb315a..6626778ae 100644 --- a/lore-server/src/grpc/thinclient/v1/revision_diff.rs +++ b/lore-server/src/grpc/thinclient/v1/revision_diff.rs @@ -72,11 +72,11 @@ pub const DEFAULT_REVISION_DIFF_SOURCE_CAP: usize = 100_000; #[derive(Clone, Copy, Debug)] pub struct RevisionDiffConfig { /// Source-side change-count cap. The handler passes this to - /// `branch::diff3_with_source_cap` so the producer aborts with + /// `branch::diff3_with_options` so the producer aborts with /// `BranchError::Oversized` before target's walk runs. pub source_cap: usize, /// Permit count for the parallel history-walk semaphore inside - /// `revision::diff3_with_source_cap`. `None` falls back to + /// `revision::diff3_with_options`. `None` falls back to /// `lore_revision::revision::DEFAULT_HISTORY_WALK_CONCURRENCY`. pub history_walk_concurrency: Option, } @@ -97,13 +97,15 @@ impl Default for RevisionDiffConfig { /// then `DiffChange` items, then — in 3-way mode only — `DiffConflict` /// items. /// -/// Mode selection is server-side from revision metadata: -/// * **2-way** when both revisions live on the same branch, or when -/// one is the branch point of the other's branch. -/// * **3-way** otherwise. The common ancestor is found via the -/// branches' stacks (then `find_branch_point` as a fallback). When -/// no common ancestor exists, the call fails with -/// `FAILED_PRECONDITION`. +/// Callers can force pairwise (2-way) or merge (3-way) semantics. AUTO +/// preserves the historical metadata-driven selection: +/// * **2-way** when both revisions live on the same branch, or when one is +/// the branch point of the other's branch. +/// * **3-way** otherwise. +/// +/// Three-way mode finds the common ancestor via the branches' stacks (then +/// `find_branch_point` as a fallback). When no common ancestor exists, the +/// call fails with `FAILED_PRECONDITION`. /// /// Identical revisions short-circuit to an OK header-only stream with /// `_base` unset. @@ -131,6 +133,12 @@ pub async fn handler( )); }; let autoresolve = req.autoresolve; + let mode = thin_client_v1::RevisionDiffMode::try_from(req.mode).map_err(|_| { + Status::invalid_argument(format!( + "RevisionDiffRequest.mode has unknown value {}", + req.mode + )) + })?; let execution = setup_execution(module_path!(), correlation_id, user_id); let repository = Arc::new( @@ -156,6 +164,7 @@ pub async fn handler( to_sig, to_id, autoresolve, + mode, config, tx, ) @@ -178,11 +187,18 @@ async fn stream_diff( to_sig: Hash, to_id: model_v1::RevisionIdentifier, autoresolve: bool, + mode: thin_client_v1::RevisionDiffMode, config: RevisionDiffConfig, tx: mpsc::Sender>, ) { // Identical short-circuit: emit a header-only OK stream. if from_sig == to_sig { + let effective_mode = match mode { + thin_client_v1::RevisionDiffMode::Merge => thin_client_v1::RevisionDiffMode::Merge, + thin_client_v1::RevisionDiffMode::Auto | thin_client_v1::RevisionDiffMode::Pairwise => { + thin_client_v1::RevisionDiffMode::Pairwise + } + }; let header = thin_client_v1::RevisionDiffHeader { identifier_from: Some(from_id), signature_from: from_sig.into(), @@ -190,6 +206,7 @@ async fn stream_diff( signature_to: to_sig.into(), identifier_base: None, signature_base: None, + mode: effective_mode as i32, }; let _ = send_header(&tx, header).await; return; @@ -198,41 +215,14 @@ async fn stream_diff( let from_branch = BranchId::from(&from_id.branch_id); let to_branch = BranchId::from(&to_id.branch_id); - // 2-way mode kicks in when the two revisions share a branch OR when - // one is the branch point of the other's branch — in both cases - // there is no divergence to merge. - let two_way = if from_branch == to_branch { - debug!( - {REPOSITORY_ID} = %repository.id, - {BRANCH_ID} = %from_branch, - "RevisionDiff: same branch → 2-way", - ); - true - } else if from_branch.is_zero() || to_branch.is_zero() { - debug!( - {REPOSITORY_ID} = %repository.id, - from_branch = %from_branch, - to_branch = %to_branch, - "RevisionDiff: a branch is zeroed → 2-way", - ); - true - } else { - match is_branch_point_of_other(&repository, from_sig, to_branch, to_sig, from_branch).await - { - Ok(true) => { - debug!( - {REPOSITORY_ID} = %repository.id, - "RevisionDiff: branch-point-of-other → 2-way", - ); - true - } - Ok(false) => false, + let two_way = + match select_two_way(&repository, mode, from_sig, from_branch, to_sig, to_branch).await { + Ok(two_way) => two_way, Err(status) => { let _ = tx.send(Err(status)).await; return; } - } - }; + }; if two_way { if let Err(status) = run_two_way(&repository, from_sig, from_id, to_sig, to_id, &tx).await { @@ -256,6 +246,62 @@ async fn stream_diff( } } +async fn select_two_way( + repository: &Arc, + mode: thin_client_v1::RevisionDiffMode, + from_sig: Hash, + from_branch: BranchId, + to_sig: Hash, + to_branch: BranchId, +) -> Result { + match mode { + thin_client_v1::RevisionDiffMode::Pairwise => { + debug!( + {REPOSITORY_ID} = %repository.id, + "RevisionDiff: explicit pairwise mode → 2-way", + ); + return Ok(true); + } + thin_client_v1::RevisionDiffMode::Merge => { + debug!( + {REPOSITORY_ID} = %repository.id, + "RevisionDiff: explicit merge mode → 3-way", + ); + return Ok(false); + } + thin_client_v1::RevisionDiffMode::Auto => {} + } + + // AUTO chooses 2-way when the revisions share a branch OR when one is the + // branch point of the other's branch — in both cases the historical + // behavior treats the snapshots as not diverged. + if from_branch == to_branch { + debug!( + {REPOSITORY_ID} = %repository.id, + {BRANCH_ID} = %from_branch, + "RevisionDiff: AUTO same branch → 2-way", + ); + Ok(true) + } else if from_branch.is_zero() || to_branch.is_zero() { + debug!( + {REPOSITORY_ID} = %repository.id, + from_branch = %from_branch, + to_branch = %to_branch, + "RevisionDiff: AUTO zero branch → 2-way", + ); + Ok(true) + } else if is_branch_point_of_other(repository, from_sig, to_branch, to_sig, from_branch).await? + { + debug!( + {REPOSITORY_ID} = %repository.id, + "RevisionDiff: AUTO branch-point-of-other → 2-way", + ); + Ok(true) + } else { + Ok(false) + } +} + /// Returns `Ok(true)` when either `from_sig` appears as a branch point /// in `to_branch`'s stack, or `to_sig` appears in `from_branch`'s /// stack. Used to fold the "branch point of other's branch" case into @@ -336,6 +382,7 @@ async fn run_two_way( signature_to: to_sig.into(), identifier_base: None, signature_base: None, + mode: thin_client_v1::RevisionDiffMode::Pairwise as i32, }; send_header(tx, header).await?; @@ -481,6 +528,7 @@ async fn run_three_way( signature_to: to_sig.into(), identifier_base: Some(base_id), signature_base: Some(base.into()), + mode: thin_client_v1::RevisionDiffMode::Merge as i32, }; send_header(tx, header).await?; @@ -491,19 +539,20 @@ async fn run_three_way( let (producer_tx, mut producer_rx) = mpsc::channel::>(256); let repo_clone = repository.clone(); let producer = lore_spawn!(async move { - Box::pin(branch::diff3_with_source_cap( + Box::pin(branch::diff3_with_options( repo_clone, from_branch, from_sig, to_branch, to_sig, - None, - false, - autoresolve, - Some(config.source_cap), - config.history_walk_concurrency, - // The display diff reports every changed path individually. - None, + branch::Diff3Options { + auto_resolve: autoresolve, + source_cap: Some(config.source_cap), + history_walk_concurrency: config.history_walk_concurrency, + // The display diff reports every changed path individually. + graft_view: None, + ..Default::default() + }, producer_tx, )) .await @@ -556,7 +605,12 @@ async fn run_three_way( unreachable!("resolve_or_announce returns Sent only via Ok") } }; - Payload::Conflict(diff_conflict_from_pair(&pair, index_from, index_to).await) + let conflict_id = lore_revision::merge_resolution::conflict_id( + base, from_sig, to_sig, &pair.0, &pair.1, + ); + Payload::Conflict( + diff_conflict_from_pair(&pair, conflict_id, index_from, index_to).await, + ) } }; match send_payload(tx, payload).await { @@ -767,11 +821,28 @@ mod test { from: QueryFrom, to: QueryTo, autoresolve: bool, + ) -> Request { + make_request_with_mode( + repository, + from, + to, + autoresolve, + thin_client_v1::RevisionDiffMode::Auto, + ) + } + + fn make_request_with_mode( + repository: RepositoryId, + from: QueryFrom, + to: QueryTo, + autoresolve: bool, + mode: thin_client_v1::RevisionDiffMode, ) -> Request { let mut request = Request::new(RevisionDiffRequest { query_from: Some(from), query_to: Some(to), autoresolve, + mode: mode as i32, }); request.metadata_mut().insert_bin( REPOSITORY_ID_KEY, @@ -887,6 +958,7 @@ mod test { query_from: None, query_to: Some(QueryTo::SignatureTo(Hash::default().into())), autoresolve: false, + mode: thin_client_v1::RevisionDiffMode::Auto as i32, }); request.metadata_mut().insert_bin( REPOSITORY_ID_KEY, @@ -908,6 +980,36 @@ mod test { .await; } + #[tokio::test] + async fn unknown_mode_returns_invalid_argument() { + let repository = random::(); + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("test stores"); + Box::pin(LORE_CONTEXT.scope(execution, async move { + let mut request = make_request( + repository, + QueryFrom::SignatureFrom(Hash::default().into()), + QueryTo::SignatureTo(Hash::default().into()), + false, + ); + request.get_mut().mode = i32::MAX; + + let err = match handler( + request, + immutable_store, + mutable_store, + RevisionDiffConfig::default(), + ) + .await + { + Ok(_) => panic!("unknown mode must fail"), + Err(err) => err, + }; + assert_eq!(err.code(), tonic::Code::InvalidArgument); + })) + .await; + } + #[tokio::test] async fn identical_revisions_return_header_only() { let repository = random::(); @@ -958,6 +1060,10 @@ mod test { assert_eq!(Hash::from(header.signature_to.as_ref()), rev); assert!(header.identifier_base.is_none()); assert!(header.signature_base.is_none()); + assert_eq!( + header.mode, + thin_client_v1::RevisionDiffMode::Pairwise as i32 + ); })) .await; } @@ -1018,6 +1124,10 @@ mod test { // 2-way: no base. assert!(header.identifier_base.is_none()); assert!(header.signature_base.is_none()); + assert_eq!( + header.mode, + thin_client_v1::RevisionDiffMode::Pairwise as i32 + ); let changes: Vec<&thin_client_v1::DiffChange> = items[1..] .iter() @@ -1210,6 +1320,7 @@ mod test { assert_eq!(BranchId::from(&base_id.branch_id), main); let base_sig = header.signature_base.as_ref().expect("signature_base set"); assert_eq!(Hash::from(base_sig.as_ref()), main_rev); + assert_eq!(header.mode, thin_client_v1::RevisionDiffMode::Merge as i32); // At least one conflict reported. assert!( items[1..] @@ -1221,6 +1332,192 @@ mod test { .await; } + #[tokio::test] + async fn explicit_pairwise_compares_diverged_branches_directly() { + let repository = random::(); + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("test stores"); + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let repository_context = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository, + )); + let main = create_branch(&repository_context, "main", vec![]).await; + let base = push_revision( + &repository_context, + main, + Hash::default(), + 1, + &[("changed.txt", b"base".as_slice())], + ) + .await; + let branch_a = create_branch( + &repository_context, + "branch_a", + vec![BranchPoint { + branch: main, + revision: base, + }], + ) + .await; + let revision_a = push_revision( + &repository_context, + branch_a, + base, + 1, + &[("changed.txt", b"left".as_slice())], + ) + .await; + let branch_b = create_branch( + &repository_context, + "branch_b", + vec![BranchPoint { + branch: main, + revision: base, + }], + ) + .await; + let revision_b = push_revision( + &repository_context, + branch_b, + base, + 1, + &[("changed.txt", b"right".as_slice())], + ) + .await; + + let response = handler( + make_request_with_mode( + repository, + QueryFrom::SignatureFrom(revision_a.into()), + QueryTo::SignatureTo(revision_b.into()), + false, + thin_client_v1::RevisionDiffMode::Pairwise, + ), + immutable_store, + mutable_store, + RevisionDiffConfig::default(), + ) + .await + .expect("handler ok"); + let items: Vec<_> = collect(response) + .await + .into_iter() + .map(|item| item.expect("stream item")) + .collect(); + + let header = match &items[0].payload { + Some(Payload::Header(header)) => header, + other => panic!("expected header, got {other:?}"), + }; + assert!(header.identifier_base.is_none()); + assert!(header.signature_base.is_none()); + assert_eq!( + header.mode, + thin_client_v1::RevisionDiffMode::Pairwise as i32 + ); + assert!( + items[1..] + .iter() + .all(|item| !matches!(item.payload, Some(Payload::Conflict(_)))) + ); + assert!(items[1..].iter().any(|item| { + matches!( + &item.payload, + Some(Payload::Change(change)) + if change.path == "changed.txt" + && change.content_from != change.content_to + ) + })); + })) + .await; + } + + #[tokio::test] + async fn explicit_merge_resolves_same_branch_divergence() { + let repository = random::(); + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("test stores"); + + Box::pin(LORE_CONTEXT.scope(execution, async move { + let repository_context = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository, + )); + let main = create_branch(&repository_context, "main", vec![]).await; + let base = push_revision( + &repository_context, + main, + Hash::default(), + 1, + &[("changed.txt", b"base".as_slice())], + ) + .await; + let detached = push_revision( + &repository_context, + main, + base, + 2, + &[("changed.txt", b"detached".as_slice())], + ) + .await; + let current = push_revision( + &repository_context, + main, + base, + 2, + &[("changed.txt", b"current".as_slice())], + ) + .await; + + let response = handler( + make_request_with_mode( + repository, + QueryFrom::SignatureFrom(detached.into()), + QueryTo::SignatureTo(current.into()), + false, + thin_client_v1::RevisionDiffMode::Merge, + ), + immutable_store, + mutable_store, + RevisionDiffConfig::default(), + ) + .await + .expect("handler ok"); + let items: Vec<_> = collect(response) + .await + .into_iter() + .map(|item| item.expect("stream item")) + .collect(); + + let header = match &items[0].payload { + Some(Payload::Header(header)) => header, + other => panic!("expected header, got {other:?}"), + }; + assert_eq!( + Hash::from( + header + .signature_base + .as_ref() + .expect("signature_base") + .as_ref() + ), + base + ); + assert_eq!(header.mode, thin_client_v1::RevisionDiffMode::Merge as i32); + assert!( + items[1..] + .iter() + .any(|item| matches!(item.payload, Some(Payload::Conflict(_)))), + "same-branch siblings should be evaluated as a merge conflict", + ); + })) + .await; + } + #[tokio::test] async fn unknown_signature_returns_not_found() { let repository = random::(); @@ -1409,6 +1706,7 @@ mod test { query_from: Some(QueryFrom::SignatureFrom(Hash::default().into())), query_to: None, autoresolve: false, + mode: thin_client_v1::RevisionDiffMode::Auto as i32, }); request.metadata_mut().insert_bin( REPOSITORY_ID_KEY, diff --git a/lore-server/src/grpc/thinclient/v1/revision_file_download.rs b/lore-server/src/grpc/thinclient/v1/revision_file_download.rs new file mode 100644 index 000000000..8cd1d0463 --- /dev/null +++ b/lore-server/src/grpc/thinclient/v1/revision_file_download.rs @@ -0,0 +1,598 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::sync::Arc; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use http::HeaderValue; +use lore_base::runtime::LORE_CONTEXT; +use lore_base::types::Hash; +use lore_proto::lore::model::v1 as model_v1; +use lore_proto::lore::thin_client::v1 as thin_client_v1; +use lore_proto::lore::thin_client::v1::RevisionFileDownloadRequest; +use lore_proto::lore::thin_client::v1::RevisionFileDownloadResponse; +use lore_revision::lore::RepositoryId; +use lore_revision::repository::RepositoryContext; +use lore_revision::state::State; +use lore_revision::util::path::RelativePath; +use lore_storage::StoreMatch; +use lore_telemetry::tracing::fields::ADDRESS; +use lore_telemetry::tracing::fields::REPOSITORY_ID; +use lore_telemetry::tracing::fields::REVISION; +use tonic::Request; +use tonic::Response; +use tonic::Status; +use tracing::warn; + +use super::helpers::resolve_to_identifier; +use crate::grpc::FilterSlowDownExt; +use crate::grpc::extract_correlation_id; +use crate::grpc::get_authorization; +use crate::grpc::get_repository; +use crate::grpc::get_user_id; +use crate::grpc::link_read_authorizer; +use crate::grpc::warn_error_to_status; +use crate::http::presign_token::CURRENT_TOKEN_VERSION; +use crate::http::presign_token::PresignTokenPayload; +use crate::http::presign_token::sign; +use crate::http::server::PresignConfig; +use crate::util::setup_execution; + +const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream"; + +/// `lore.thin_client.v1.ThinClientService.RevisionFileDownload` handler. +/// +/// The gRPC request performs all mutable-name resolution and authorization. +/// Its result is pinned to a concrete revision and immutable content address; +/// redemption only streams that logical address through Lore's HTTP server. +#[tracing::instrument(name = "RevisionFileDownload::v1::handle", skip_all)] +pub async fn handler( + request: Request, + immutable_store: Arc, + mutable_store: Arc, + presign_config: Option, +) -> Result, Status> { + let repository_id = get_repository(request.metadata())?; + let user_id = get_user_id(request.extensions()); + let authorization = get_authorization(request.extensions()).ok(); + let correlation_id = extract_correlation_id(&request).unwrap_or_default(); + let request = request.into_inner(); + let presign_config = presign_config.ok_or_else(|| { + Status::failed_precondition( + "logical download URLs are disabled because the Lore HTTP presign feature is not configured", + ) + })?; + + let Some(query) = request.query else { + return Err(Status::invalid_argument( + "RevisionFileDownloadRequest.query must be set (identifier or signature)", + )); + }; + let path = RelativePath::new_from_initial_path(&request.path) + .map_err(|error| Status::invalid_argument(format!("invalid path: {error}")))?; + if path.is_empty() { + return Err(Status::invalid_argument("path must name a file")); + } + let content_type = if request.content_type.is_empty() { + DEFAULT_CONTENT_TYPE.to_string() + } else { + request.content_type + }; + HeaderValue::from_str(&content_type) + .map_err(|_| Status::invalid_argument("content_type is not a valid HTTP header value"))?; + + let can_read = link_read_authorizer(authorization); + let execution = setup_execution(module_path!(), correlation_id, user_id); + LORE_CONTEXT + .scope(execution, async move { + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store, + repository_id, + )); + let (signature, identifier) = resolve_to_identifier(&repository, query.into()).await?; + if signature.is_zero() { + return Err(Status::invalid_argument( + "cannot download a file from a zeroed revision", + )); + } + + let state = load_state(&repository, signature).await?; + let node_link = state + .find_node_link(repository.clone(), path.as_str()) + .await + .map_err(|error| map_path_error(repository_id, signature, &path, error))?; + if !node_link.is_valid() || !can_read(node_link.repository) { + return Err(Status::not_found(format!( + "file path {} was not found", + path.as_str() + ))); + } + + let resolved_repository = if node_link.repository == repository.id { + repository.clone() + } else { + Arc::new(repository.to_link_context(node_link.repository).await) + }; + let resolved_state = if node_link.repository == repository.id + && node_link.revision == state.revision() + { + state + } else { + load_state(&resolved_repository, node_link.revision).await? + }; + let node = resolved_state + .node(resolved_repository.clone(), node_link.node) + .await + .map_err(|error| { + warn!( + {REPOSITORY_ID} = %node_link.repository, + {REVISION} = %node_link.revision, + path = path.as_str(), + ?error, + "Failed to load resolved download node", + ); + warn_error_to_status(&error, |e| Status::internal(e.to_string())) + })?; + if !node.is_file() { + return Err(Status::failed_precondition(format!( + "path {} is not a file", + path.as_str() + ))); + } + + validate_content_address( + immutable_store, + node_link.repository, + node.address, + node.size, + ) + .await?; + + let ttl = if request.ttl_seconds == 0 { + presign_config.default_ttl_seconds + } else { + request.ttl_seconds + } + .clamp( + presign_config.min_ttl_seconds, + presign_config.max_ttl_seconds, + ); + let expires_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| Status::internal(format!("system clock error: {error}")))? + .as_secs() + .saturating_add(ttl); + let file_name = path.name().to_string(); + let content_disposition = content_disposition(&file_name, request.inline); + HeaderValue::from_str(&content_disposition).map_err(|_| { + Status::internal("generated Content-Disposition is not a valid HTTP header value") + })?; + + let repository_resolved = node_link.repository; + let address = node.address; + let payload = PresignTokenPayload { + version: CURRENT_TOKEN_VERSION, + key_id: presign_config.key_id, + repository: repository_resolved.to_string(), + address: address.to_string(), + expires_at, + content_type: Some(content_type), + content_encoding: None, + content_disposition: Some(content_disposition), + content_length: Some(node.size), + }; + let token = sign(&payload, &presign_config.hmac_key); + let url_suffix = format!("/v1/presigned/{repository_resolved}/{address}?token={token}"); + + Ok(Response::new(RevisionFileDownloadResponse { + revision: Some(thin_client_v1::RevisionTreeHeader { + identifier: Some(identifier), + signature: signature.into(), + }), + repository_resolved: bytes::Bytes::copy_from_slice(repository_resolved.data()), + address: Some(model_v1::Address { + hash: address.hash.into(), + context: address.context.into(), + }), + size: node.size, + url_suffix, + expires_at_epoch_seconds: expires_at, + file_name, + mode: node.mode as u32, + })) + }) + .await +} + +async fn load_state( + repository: &Arc, + signature: Hash, +) -> Result, Status> { + State::deserialize(repository.clone(), signature) + .await + .filter_slow_down()? + .map_err(|error| { + if error.is_not_found() || error.is_revision_not_found() { + Status::not_found(format!("Revision {signature} not found")) + } else { + warn!( + {REPOSITORY_ID} = %repository.id, + {REVISION} = %signature, + ?error, + "Failed to deserialize revision for file download", + ); + warn_error_to_status(&error, |e| Status::internal(e.to_string())) + } + }) +} + +fn map_path_error( + repository: RepositoryId, + revision: Hash, + path: &RelativePath, + error: lore_revision::state::StateError, +) -> Status { + if error.is_not_found() || error.is_node_not_found() || error.is_link_not_found() { + Status::not_found(format!("file path {} was not found", path.as_str())) + } else { + warn!( + {REPOSITORY_ID} = %repository, + {REVISION} = %revision, + path = path.as_str(), + ?error, + "Failed to resolve file download path", + ); + warn_error_to_status(&error, |e| Status::internal(e.to_string())) + } +} + +async fn validate_content_address( + immutable_store: Arc, + repository: RepositoryId, + address: lore_storage::Address, + expected_size: u64, +) -> Result<(), Status> { + if address.hash.is_zero() { + return if expected_size == 0 { + Ok(()) + } else { + Err(Status::data_loss( + "non-empty file has a zero content address", + )) + }; + } + + let query = immutable_store + .get_metadata(repository, address) + .await + .map_err(|error| { + warn!( + {REPOSITORY_ID} = %repository, + {ADDRESS} = %address, + ?error, + "Failed to query resolved file content", + ); + Status::internal(format!("failed to query file content: {error}")) + })?; + if query.match_made != StoreMatch::MatchFull { + return Err(Status::not_found("file content was not found")); + } + if query.fragment.size_content != expected_size { + return Err(Status::data_loss(format!( + "file node size {expected_size} does not match stored content size {}", + query.fragment.size_content + ))); + } + Ok(()) +} + +fn content_disposition(file_name: &str, inline: bool) -> String { + let kind = if inline { "inline" } else { "attachment" }; + let fallback: String = file_name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_') { + c + } else { + '_' + } + }) + .collect(); + let fallback = if fallback.is_empty() { + "download" + } else { + &fallback + }; + format!( + "{kind}; filename=\"{fallback}\"; filename*=UTF-8''{}", + encode_rfc5987(file_name) + ) +} + +fn encode_rfc5987(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() + || matches!( + byte, + b'!' | b'#' | b'$' | b'&' | b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~' + ) + { + encoded.push(byte as char); + } else { + use std::fmt::Write as _; + let _ = write!(encoded, "%{byte:02X}"); + } + } + encoded +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use lore_base::types::Context; + use lore_proto::lore::thin_client::v1::revision_file_download_request::Query; + use lore_revision::branch; + use lore_revision::branch::DEFAULT_HISTORY_STEP_SIZE; + use lore_revision::lore::BranchId; + use lore_revision::metadata::Metadata; + use lore_revision::node::Node; + use lore_revision::node::NodeFlags; + use lore_revision::node::ROOT_NODE; + use lore_revision::state; + use lore_storage::WriteOptions; + use lore_storage::hash_string; + use lore_transport::grpc::REPOSITORY_ID_KEY; + use tonic::Code; + + use super::*; + use crate::grpc::get_write_token; + use crate::grpc::handlers::branch_push; + use crate::http::presign_token::verify; + use crate::store::test_store_create; + + fn test_presign_config() -> PresignConfig { + PresignConfig { + hmac_key: ring::hmac::Key::new(ring::hmac::HMAC_SHA256, &[7u8; 32]), + key_id: "download_test_key".to_string(), + min_ttl_seconds: 10, + default_ttl_seconds: 300, + max_ttl_seconds: 600, + content_type_allowlist: crate::http::security_headers::ContentTypeAllowlist::default(), + } + } + + fn request( + repository: RepositoryId, + query: Query, + path: &str, + ) -> Request { + let mut request = Request::new(RevisionFileDownloadRequest { + query: Some(query), + path: path.to_string(), + ttl_seconds: 60, + content_type: "text/plain; charset=utf-8".to_string(), + inline: false, + }); + request.metadata_mut().insert_bin( + REPOSITORY_ID_KEY, + tonic::metadata::BinaryMetadataValue::from_bytes(repository.data()), + ); + request + } + + async fn push_file( + repository: &Arc, + immutable_store: Arc, + name: &str, + content: Bytes, + ) -> (BranchId, Hash, lore_storage::Address) { + let write_token = get_write_token(); + let branch_id = BranchId::from(uuid::Uuid::now_v7()); + branch::create( + repository.clone(), + &write_token, + branch_id, + "download-test", + branch::default_category(), + "creator", + 1, + vec![], + false, + false, + ) + .await + .expect("create branch"); + + let address = lore_storage::write_content( + immutable_store, + repository.id, + Context::from(uuid::Uuid::now_v7()), + content.clone(), + WriteOptions::default(), + None, + None, + None, + ) + .await + .expect("write content") + .address; + let mut metadata = Metadata::new(); + metadata.set_branch(branch_id).expect("set branch"); + let metadata_hash = metadata + .serialize(repository.clone()) + .await + .expect("serialize metadata"); + let state = state::State::new(); + state.set_revision_number(1); + state.set_metadata_hash(metadata_hash); + state + .node_add( + repository.clone(), + ROOT_NODE, + Node { + flags: NodeFlags::File.bits(), + mode: 1, + name_hash: hash_string(name), + size: content.len() as u64, + address, + ..Default::default() + }, + name, + ) + .await + .expect("add file"); + let serialized = state + .serialize(repository.clone(), &write_token) + .await + .expect("serialize state"); + let signature = branch_push::push( + repository.clone(), + branch_id, + serialized, + true, + true, + false, + DEFAULT_HISTORY_STEP_SIZE, + crate::grpc::server::RevisionListAcceleration::default(), + ) + .await + .expect("push revision") + .revision; + (branch_id, signature, address) + } + + #[test] + fn content_disposition_has_ascii_fallback_and_utf8_filename() { + assert_eq!( + content_disposition("résumé 2026.txt", false), + "attachment; filename=\"r_sum__2026.txt\"; filename*=UTF-8''r%C3%A9sum%C3%A9%202026.txt" + ); + assert_eq!( + content_disposition("photo.png", true), + "inline; filename=\"photo.png\"; filename*=UTF-8''photo.png" + ); + } + + #[tokio::test] + async fn resolves_branch_tip_and_signs_pinned_logical_download() { + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("test stores"); + LORE_CONTEXT + .scope(execution, async move { + let repository_id = RepositoryId::from(uuid::Uuid::now_v7()); + let repository = Arc::new(RepositoryContext::new_server_context( + immutable_store.clone(), + mutable_store.clone(), + repository_id, + )); + let content = Bytes::from_static(b"logical file bytes"); + let (branch_id, signature, address) = push_file( + &repository, + immutable_store.clone(), + "résumé.txt", + content.clone(), + ) + .await; + let config = test_presign_config(); + let response = handler( + request( + repository_id, + Query::Identifier(model_v1::RevisionIdentifier { + branch_id: branch_id.into(), + number: 0, + }), + "résumé.txt", + ), + immutable_store.clone(), + mutable_store.clone(), + Some(config.clone()), + ) + .await + .expect("download response") + .into_inner(); + + let revision = response.revision.expect("resolved revision"); + assert_eq!(Hash::from(revision.signature.as_ref()), signature); + assert_eq!(revision.identifier.unwrap().number, 1); + assert_eq!(response.size, content.len() as u64); + assert_eq!(response.file_name, "résumé.txt"); + assert_eq!(response.mode, 1); + assert_eq!( + lore_storage::Address::from(response.address.as_ref().unwrap()), + address + ); + assert_eq!(response.repository_resolved.as_ref(), repository_id.data()); + + let token = response + .url_suffix + .split_once("?token=") + .expect("signed token in URL") + .1; + let payload = verify( + token, + &config.hmac_key, + &config.key_id, + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + ) + .expect("valid signed token"); + assert_eq!(payload.repository, repository_id.to_string()); + assert_eq!(payload.address, address.to_string()); + assert_eq!(payload.content_length, Some(content.len() as u64)); + assert_eq!( + payload.content_type.as_deref(), + Some("text/plain; charset=utf-8") + ); + assert!( + payload + .content_disposition + .as_deref() + .unwrap() + .starts_with("attachment; filename=\"r_sum_.txt\"") + ); + + let error = handler( + request( + repository_id, + Query::Signature(signature.into()), + "missing.txt", + ), + immutable_store, + mutable_store, + Some(config), + ) + .await + .expect_err("missing file must not be signed"); + assert_eq!(error.code(), Code::NotFound); + }) + .await; + } + + #[tokio::test] + async fn fails_when_logical_download_urls_are_not_configured() { + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("test stores"); + LORE_CONTEXT + .scope(execution, async move { + let repository_id = RepositoryId::from(uuid::Uuid::now_v7()); + let error = handler( + request( + repository_id, + Query::Signature(Hash::hash_buffer(b"revision").into()), + "file.txt", + ), + immutable_store, + mutable_store, + None, + ) + .await + .expect_err("presign must be configured"); + assert_eq!(error.code(), Code::FailedPrecondition); + }) + .await; + } +} diff --git a/lore-server/src/grpc/thinclient/v1/revision_tree.rs b/lore-server/src/grpc/thinclient/v1/revision_tree.rs index 0b4561038..8643ff3db 100644 --- a/lore-server/src/grpc/thinclient/v1/revision_tree.rs +++ b/lore-server/src/grpc/thinclient/v1/revision_tree.rs @@ -161,6 +161,10 @@ async fn stream_tree( hash: address.hash.into(), context: address.context.into(), }), + last_changed_revision_signature: tree_path + .last_changed_revision + .map(Into::into) + .unwrap_or_default(), size: tree_path.size, mode: tree_path.mode, tracking: tree_path.tracking, @@ -571,6 +575,9 @@ mod test { .iter() .all(|n| n.node_type == thin_client_v1::NodeType::File as i32) ); + assert!(nodes.iter().all(|node| { + Hash::from(node.last_changed_revision_signature.as_ref()) == signature + })); })) .await; } @@ -1088,6 +1095,15 @@ mod test { let child = &nodes[1]; assert_eq!(child.path, "linked/inner.txt"); assert_eq!(child.node_type, thin_client_v1::NodeType::File as i32); + assert_eq!( + Hash::from(link.last_changed_revision_signature.as_ref()), + signature, + ); + assert_eq!( + Hash::from(child.last_changed_revision_signature.as_ref()), + signature, + "linked descendants inherit the revision that changed the link pin", + ); })) .await; } diff --git a/lore-server/src/grpc/thinclient/v1/service.rs b/lore-server/src/grpc/thinclient/v1/service.rs index f4f868385..35439f94a 100644 --- a/lore-server/src/grpc/thinclient/v1/service.rs +++ b/lore-server/src/grpc/thinclient/v1/service.rs @@ -8,6 +8,8 @@ use lore_proto::lore::thin_client::v1::ContentDiffRequest; use lore_proto::lore::thin_client::v1::ContentDiffResponse; use lore_proto::lore::thin_client::v1::RevisionDiffRequest; use lore_proto::lore::thin_client::v1::RevisionDiffResponse; +use lore_proto::lore::thin_client::v1::RevisionFileDownloadRequest; +use lore_proto::lore::thin_client::v1::RevisionFileDownloadResponse; use lore_proto::lore::thin_client::v1::RevisionInfoRequest; use lore_proto::lore::thin_client::v1::RevisionInfoResponse; use lore_proto::lore::thin_client::v1::RevisionTreeRequest; @@ -19,10 +21,13 @@ use tonic::Response; use tonic::Status; use tonic::codegen::tokio_stream::Stream; +use super::content_diff; use super::revision_diff; +use super::revision_file_download; use super::revision_info; use super::revision_tree; use crate::grpc::timeout_grpc; +use crate::http::server::PresignConfig; type ContentDiffStream = Pin> + Send + 'static>>; @@ -53,6 +58,7 @@ pub struct LoreThinClientV1Service { mutable_store: Arc, rpc_timeout: Duration, revision_diff_config: revision_diff::RevisionDiffConfig, + presign_config: Option, #[allow(dead_code)] instrument_provider: ThinClientServiceInstrumentProvider, } @@ -63,12 +69,14 @@ impl LoreThinClientV1Service { mutable_store: Arc, rpc_timeout: Duration, revision_diff_config: revision_diff::RevisionDiffConfig, + presign_config: Option, ) -> Self { Self { immutable_store, mutable_store, rpc_timeout, revision_diff_config, + presign_config, instrument_provider: ThinClientServiceInstrumentProvider, } } @@ -88,11 +96,14 @@ impl ThinClientService for LoreThinClientV1Service { async fn content_diff( &self, - _request: Request, + request: Request, ) -> Result, Status> { - Err(Status::unimplemented( - "lore.thin_client.v1.ThinClientService.ContentDiff not yet implemented", - )) + content_diff::handler( + request, + self.immutable_store.clone(), + self.mutable_store.clone(), + ) + .await } async fn revision_info( @@ -138,6 +149,22 @@ impl ThinClientService for LoreThinClientV1Service { ) .await } + + async fn revision_file_download( + &self, + request: Request, + ) -> Result, Status> { + timeout_grpc( + self.rpc_timeout, + revision_file_download::handler( + request, + self.immutable_store.clone(), + self.mutable_store.clone(), + self.presign_config.clone(), + ), + ) + .await + } } #[cfg(test)] diff --git a/lore-server/src/hooks/lore_index_events.rs b/lore-server/src/hooks/lore_index_events.rs new file mode 100644 index 000000000..522476a96 --- /dev/null +++ b/lore-server/src/hooks/lore_index_events.rs @@ -0,0 +1,551 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT + +use std::collections::BTreeMap; +use std::fs; +use std::sync::mpsc; +use std::sync::mpsc::Receiver; +use std::sync::mpsc::Sender; +use std::thread; + +use async_trait::async_trait; +use parking_lot::Mutex; +use postgres::Client; +use postgres::NoTls; +use uuid::Uuid; + +use crate::hooks::Hook; +use crate::hooks::HookContext; +use crate::hooks::HookError; +use crate::hooks::HookFactory; +use crate::hooks::HookPoint; +use crate::hooks::HookRegistrationContext; +use crate::hooks::HookRegistry; +use crate::hooks::HookResponse; + +const HOOK_NAME: &str = "lore_index_events"; +const SOURCE: &str = "lore_hook"; +const DEFAULT_DATABASE_URL_ENV: &str = "DATABASE_URL"; +const INDEX_HOOK_POINTS: &[HookPoint] = &[ + HookPoint::BranchPush, + HookPoint::BranchCreate, + HookPoint::BranchDelete, + HookPoint::RepositoryCreate, +]; +const INSERT_EVENT_SQL: &str = r#" +insert into lore_index_events ( + id, + source, + event_key, + event_type, + lore_repository_id, + lore_branch_id, + lore_revision_signature, + lore_revision_number, + user_id, + correlation_id, + payload +) values ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11::text::jsonb +) +on conflict (event_key) do nothing +"#; + +struct LoreIndexEventsHook { + tx: Mutex>, +} + +#[async_trait] +impl Hook for LoreIndexEventsHook { + fn name(&self) -> &'static str { + HOOK_NAME + } + + fn hook_points(&self) -> &'static [HookPoint] { + INDEX_HOOK_POINTS + } + + fn pre_handler(&self, _ctx: &HookContext) -> Result<(), HookError> { + Ok(()) + } + + fn response_handler(&self, _ctx: &HookContext) -> Result { + Ok(HookResponse::empty()) + } + + async fn post_handler(&self, ctx: &HookContext) -> Result<(), HookError> { + self.enqueue("post", ctx); + Ok(()) + } +} + +impl LoreIndexEventsHook { + fn enqueue(&self, phase: &'static str, ctx: &HookContext) { + let event = EventRecord::from_context(phase, ctx); + if let Err(err) = self.tx.lock().send(event) { + eprintln!("[loreserver-hook:{HOOK_NAME}] failed to enqueue event: {err}"); + } + } +} + +struct LoreIndexEventsHookFactory; + +impl HookFactory for LoreIndexEventsHookFactory { + fn name(&self) -> &'static str { + HOOK_NAME + } + + fn create(&self, config: &toml::Value) -> Result, HookError> { + let database_url = database_url_from_config(config)?; + let tx = spawn_event_worker(database_url)?; + Ok(Box::new(LoreIndexEventsHook { tx: Mutex::new(tx) })) + } +} + +#[derive(Debug, Clone)] +struct EventRecord { + id: Uuid, + event_key: String, + event_type: String, + lore_repository_id: Vec, + lore_branch_id: Option>, + lore_revision_signature: Option>, + lore_revision_number: Option, + user_id: Option, + correlation_id: Option, + payload: String, +} + +impl EventRecord { + fn from_context(phase: &'static str, ctx: &HookContext) -> Self { + let lore_repository_id = ctx.repository().as_ref().to_vec(); + let lore_branch_id = ctx.branch().map(|branch| branch.as_ref().to_vec()); + let lore_revision_signature = ctx.revision().map(|revision| revision.as_ref().to_vec()); + let lore_revision_number = ctx + .revision_number() + .map(|revision_number| i64::try_from(revision_number).unwrap_or(i64::MAX)); + let raw_user_id = non_empty(ctx.user()); + let user_id = raw_user_id.as_deref().and_then(uuid_value); + + let raw_correlation_id = non_empty(Some(ctx.correlation_id())); + let correlation_id = raw_correlation_id.as_deref().and_then(uuid_value); + let hook_point = canonical_hook_point(ctx.hook_point()); + let event_type = format!("{hook_point}.{phase}"); + let metadata: BTreeMap = ctx + .metadata() + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let event_key = event_key( + &event_type, + &lore_repository_id, + lore_branch_id.as_deref(), + lore_revision_signature.as_deref(), + lore_revision_number, + raw_user_id.as_deref(), + raw_correlation_id.as_deref(), + &metadata, + ); + let payload = payload_json( + phase, + hook_point, + &lore_repository_id, + lore_branch_id.as_deref(), + lore_revision_signature.as_deref(), + lore_revision_number, + user_id.as_ref(), + raw_user_id.as_deref(), + correlation_id.as_ref(), + raw_correlation_id.as_deref(), + metadata, + ); + + Self { + id: Uuid::now_v7(), + event_key, + event_type, + lore_repository_id, + lore_branch_id, + lore_revision_signature, + lore_revision_number, + user_id, + correlation_id, + payload, + } + } +} + +fn canonical_hook_point(hook_point: HookPoint) -> &'static str { + match hook_point { + HookPoint::BranchPush => "branch_push", + HookPoint::BranchCreate => "branch_create", + HookPoint::BranchDelete => "branch_delete", + HookPoint::RepositoryCreate => "repository_create", + HookPoint::Obliterate => "obliterate", + } +} + +fn uuid_value(value: &str) -> Option { + Uuid::try_parse(value).ok() +} + +fn non_empty(value: Option<&str>) -> Option { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) +} + +fn event_key( + event_type: &str, + lore_repository_id: &[u8], + lore_branch_id: Option<&[u8]>, + lore_revision_signature: Option<&[u8]>, + lore_revision_number: Option, + user_id: Option<&str>, + correlation_id: Option<&str>, + metadata: &BTreeMap, +) -> String { + let mut hasher = blake3::Hasher::new(); + update_str(&mut hasher, "source", SOURCE); + update_str(&mut hasher, "event_type", event_type); + update_bytes(&mut hasher, "lore_repository_id", lore_repository_id); + update_optional_bytes(&mut hasher, "lore_branch_id", lore_branch_id); + update_optional_bytes( + &mut hasher, + "lore_revision_signature", + lore_revision_signature, + ); + update_optional_i64(&mut hasher, "lore_revision_number", lore_revision_number); + update_optional_str(&mut hasher, "user_id", user_id); + update_optional_str(&mut hasher, "correlation_id", correlation_id); + for (key, value) in metadata { + update_str(&mut hasher, "metadata_key", key); + update_str(&mut hasher, "metadata_value", value); + } + hasher.finalize().to_hex().to_string() +} + +fn update_str(hasher: &mut blake3::Hasher, field: &str, value: &str) { + update_bytes(hasher, field, value.as_bytes()); +} + +fn update_optional_str(hasher: &mut blake3::Hasher, field: &str, value: Option<&str>) { + match value { + Some(value) => update_str(hasher, field, value), + None => update_bytes(hasher, field, b""), + } +} + +fn update_optional_i64(hasher: &mut blake3::Hasher, field: &str, value: Option) { + match value { + Some(value) => update_bytes(hasher, field, &value.to_le_bytes()), + None => update_bytes(hasher, field, b""), + } +} + +fn update_optional_bytes(hasher: &mut blake3::Hasher, field: &str, value: Option<&[u8]>) { + match value { + Some(value) => update_bytes(hasher, field, value), + None => update_bytes(hasher, field, b""), + } +} + +fn update_bytes(hasher: &mut blake3::Hasher, field: &str, value: &[u8]) { + hasher.update(&(field.len() as u64).to_le_bytes()); + hasher.update(field.as_bytes()); + hasher.update(&(value.len() as u64).to_le_bytes()); + hasher.update(value); +} + +#[allow(clippy::too_many_arguments)] +fn payload_json( + phase: &str, + hook_point: &str, + lore_repository_id: &[u8], + lore_branch_id: Option<&[u8]>, + lore_revision_signature: Option<&[u8]>, + lore_revision_number: Option, + user_id: Option<&Uuid>, + raw_user_id: Option<&str>, + correlation_id: Option<&Uuid>, + raw_correlation_id: Option<&str>, + metadata: BTreeMap, +) -> String { + let user_id = user_id.map(Uuid::to_string); + let correlation_id = correlation_id.map(Uuid::to_string); + let payload = serde_json::json!({ + "hook_name": HOOK_NAME, + "phase": phase, + "hook_point": hook_point, + "lore_repository_id": hex::encode(lore_repository_id), + "lore_branch_id": lore_branch_id.map(hex::encode), + "lore_revision_signature": lore_revision_signature.map(hex::encode), + "lore_revision_number": lore_revision_number, + "user_id": user_id, + "raw_user_id": raw_user_id, + "correlation_id": correlation_id, + "raw_correlation_id": raw_correlation_id, + "metadata": metadata, + }); + + serde_json::to_string(&payload).unwrap_or_else(|err| { + eprintln!("[loreserver-hook:{HOOK_NAME}] failed to serialize payload: {err}"); + "{}".to_string() + }) +} + +fn spawn_event_worker(database_url: String) -> Result, HookError> { + let (tx, rx) = mpsc::channel(); + thread::Builder::new() + .name("lore-index-events".to_string()) + .spawn(move || run_event_worker(database_url, rx)) + .map_err(|err| { + HookError::init_error(HOOK_NAME, format!("failed to spawn worker: {err}")) + })?; + Ok(tx) +} + +fn run_event_worker(database_url: String, rx: Receiver) { + let mut client = None; + for event in rx { + insert_with_reconnect(&database_url, &mut client, &event); + } +} + +fn insert_with_reconnect(database_url: &str, client: &mut Option, event: &EventRecord) { + for attempt in 0..2 { + if client.is_none() { + match Client::connect(database_url, NoTls) { + Ok(connected) => { + *client = Some(connected); + } + Err(err) => { + eprintln!("[loreserver-hook:{HOOK_NAME}] failed to connect to Postgres: {err}"); + return; + } + } + } + + let Some(connected) = client.as_mut() else { + return; + }; + + match insert_event(connected, event) { + Ok(_) => return, + Err(err) => { + eprintln!( + "[loreserver-hook:{HOOK_NAME}] failed to insert event {}: {err:?}", + event.event_key + ); + if !err.is_closed() { + return; + } + *client = None; + if attempt == 1 { + return; + } + } + } + } +} + +fn insert_event(client: &mut Client, event: &EventRecord) -> Result { + client.execute( + INSERT_EVENT_SQL, + &[ + &event.id, + &SOURCE, + &event.event_key, + &event.event_type, + &event.lore_repository_id, + &event.lore_branch_id, + &event.lore_revision_signature, + &event.lore_revision_number, + &event.user_id, + &event.correlation_id, + &event.payload, + ], + ) +} + +fn database_url_from_config(config: &toml::Value) -> Result { + if let Some(database_url) = config_str(config, "database_url")? { + return Ok(database_url.to_string()); + } + + let env_name = config_str(config, "database_url_env")?.unwrap_or(DEFAULT_DATABASE_URL_ENV); + if let Ok(database_url) = std::env::var(env_name) + && !database_url.trim().is_empty() + { + return Ok(database_url); + } + + if let Some(path) = config_str(config, "database_url_file")? + && let Some(database_url) = database_url_from_file(path, env_name)? + { + return Ok(database_url); + } + + Err(HookError::config_error( + HOOK_NAME, + format!( + "missing database URL; set database_url, {env_name}, or database_url_file containing {env_name}", + ), + )) +} + +fn config_str<'a>(config: &'a toml::Value, key: &str) -> Result, HookError> { + match config.get(key) { + Some(value) => value + .as_str() + .map(Some) + .ok_or_else(|| HookError::config_error(HOOK_NAME, format!("'{key}' must be a string"))), + None => Ok(None), + } +} + +fn database_url_from_file(path: &str, env_name: &str) -> Result, HookError> { + let content = fs::read_to_string(path).map_err(|err| { + HookError::config_error( + HOOK_NAME, + format!("failed to read database_url_file '{path}': {err}"), + ) + })?; + + for line in content.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + continue; + }; + if key.trim() == env_name { + let value = unquote_env_value(value.trim()); + if !value.trim().is_empty() { + return Ok(Some(value)); + } + } + } + + Ok(None) +} + +fn unquote_env_value(value: &str) -> String { + if value.len() >= 2 { + let first = value.as_bytes()[0]; + let last = value.as_bytes()[value.len() - 1]; + if (first == b'"' && last == b'"') || (first == b'\'' && last == b'\'') { + return value[1..value.len() - 1].to_string(); + } + } + value.to_string() +} + +pub fn register(registry: &mut HookRegistry, _ctx: &HookRegistrationContext) { + registry.register_hook(Box::new(LoreIndexEventsHookFactory)); +} + +#[cfg(test)] +mod tests { + use lore_revision::lore::RepositoryId; + + use super::*; + + fn config_with_database_url() -> toml::Value { + toml::toml! { + database_url = "postgres://postgres@localhost/test" + } + .into() + } + + fn test_context() -> HookContext { + HookContext::builder() + .correlation_id("corr-123") + .hook_point(HookPoint::BranchPush) + .repository(RepositoryId::default()) + .user("local-user") + .metadata("branch_name", "main") + .build() + } + + #[test] + fn factory_creates_hook_for_indexable_hook_points() { + let hook = LoreIndexEventsHookFactory + .create(&config_with_database_url()) + .unwrap(); + + assert_eq!(hook.name(), HOOK_NAME); + assert_eq!(hook.hook_points(), INDEX_HOOK_POINTS); + } + + #[test] + fn event_key_is_deterministic_but_id_is_unique() { + let ctx = test_context(); + + let first = EventRecord::from_context("post", &ctx); + let second = EventRecord::from_context("post", &ctx); + + assert_eq!(first.event_type, "branch_push.post"); + assert_eq!(first.event_key, second.event_key); + assert_ne!(first.id, second.id); + assert!(first.user_id.is_none()); + let payload: serde_json::Value = serde_json::from_str(&first.payload).unwrap(); + assert_eq!(payload["raw_user_id"], "local-user"); + assert_eq!(first.id.get_version_num(), 7); + } + + #[test] + fn response_handler_does_not_modify_client_response() { + let (tx, rx) = mpsc::channel(); + let hook = LoreIndexEventsHook { tx: Mutex::new(tx) }; + let ctx = test_context(); + + let response = hook.response_handler(&ctx).unwrap(); + + assert!(response.message.is_none()); + assert!(rx.try_recv().is_err()); + } + + #[test] + #[ignore = "requires DATABASE_URL and a local lore_index_events table"] + fn inserts_event_into_postgres() { + let database_url = std::env::var(DEFAULT_DATABASE_URL_ENV).unwrap(); + let mut client = Client::connect(&database_url, NoTls).unwrap(); + let mut event = EventRecord::from_context("post", &test_context()); + let unknown_user_id = Uuid::now_v7(); + event.user_id = Some(unknown_user_id); + event.event_key = format!("test:{SOURCE}:{}", Uuid::now_v7()); + + insert_event(&mut client, &event).unwrap(); + let rows = client + .query( + "select source, event_type, payload->>'phase', user_id from lore_index_events where event_key = $1", + &[&event.event_key], + ) + .unwrap(); + client + .execute( + "delete from lore_index_events where event_key = $1", + &[&event.event_key], + ) + .unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].get::<_, String>(0), SOURCE); + assert_eq!(rows[0].get::<_, String>(1), "branch_push.post"); + assert_eq!(rows[0].get::<_, String>(2), "post"); + assert_eq!(rows[0].get::<_, Uuid>(3), unknown_user_id); + } +} diff --git a/lore-server/src/hooks/mod.rs b/lore-server/src/hooks/mod.rs index cf445074f..590995750 100644 --- a/lore-server/src/hooks/mod.rs +++ b/lore-server/src/hooks/mod.rs @@ -242,6 +242,9 @@ pub mod dispatch; pub mod registry; pub mod traits; +// Auto-discovered hook modules +pub mod lore_index_events; + // Re-export commonly used items at the module level pub use context::HookContext; pub use context::HookContextBuilder; @@ -268,8 +271,7 @@ pub use traits::StatusCode; /// * `registry` - The hook registry to register hooks with /// * `ctx` - Registration context providing runtime dependencies for hook factories pub fn register_all_hooks(registry: &mut HookRegistry, ctx: &HookRegistrationContext) { - // No hooks discovered - registry will be empty - let _ = (registry, ctx); // Suppress unused variable warnings + lore_index_events::register(registry, ctx); } #[cfg(test)] diff --git a/lore-server/src/http/presign_token.rs b/lore-server/src/http/presign_token.rs index b9e71b04a..36ea46d92 100644 --- a/lore-server/src/http/presign_token.rs +++ b/lore-server/src/http/presign_token.rs @@ -21,6 +21,10 @@ pub struct PresignTokenPayload { pub content_encoding: Option, #[serde(skip_serializing_if = "Option::is_none")] pub content_disposition: Option, + /// Logical byte length after Lore defragmentation/decompression. Optional + /// so URLs issued before this field existed remain redeemable. + #[serde(skip_serializing_if = "Option::is_none")] + pub content_length: Option, } #[derive(Debug, Error, PartialEq)] @@ -110,6 +114,7 @@ mod tests { content_type: None, content_encoding: None, content_disposition: None, + content_length: None, } } @@ -184,10 +189,12 @@ mod tests { payload.content_type = Some("image/png".to_string()); payload.content_encoding = Some("gzip".to_string()); payload.content_disposition = Some("inline".to_string()); + payload.content_length = Some(42); let token = sign(&payload, &key); let result = verify(&token, &key, "test_key_id", 0 /* now_unix */).unwrap(); assert_eq!(result.content_type.as_deref(), Some("image/png")); assert_eq!(result.content_encoding.as_deref(), Some("gzip")); assert_eq!(result.content_disposition.as_deref(), Some("inline")); + assert_eq!(result.content_length, Some(42)); } } diff --git a/lore-server/src/http/presigned/repository/redeem.rs b/lore-server/src/http/presigned/repository/redeem.rs index b5b6201b3..f9c299d5f 100644 --- a/lore-server/src/http/presigned/repository/redeem.rs +++ b/lore-server/src/http/presigned/repository/redeem.rs @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: 2026 Epic Games, Inc. // SPDX-License-Identifier: MIT +use std::ops::Range; use std::sync::Arc; use std::time::SystemTime; use std::time::UNIX_EPOCH; @@ -11,10 +12,16 @@ use axum::extract::State; use axum::http::HeaderMap; use axum::http::HeaderValue; use axum::http::StatusCode; +use axum::http::header::ACCEPT_RANGES; use axum::http::header::CONTENT_DISPOSITION; use axum::http::header::CONTENT_ENCODING; +use axum::http::header::CONTENT_LENGTH; +use axum::http::header::CONTENT_RANGE; use axum::http::header::CONTENT_TYPE; +use axum::http::header::ETAG; +use axum::http::header::IF_RANGE; use axum::http::header::InvalidHeaderValue; +use axum::http::header::RANGE; use axum::response::IntoResponse; use hex::FromHexError; use lore_base::runtime::LORE_CONTEXT; @@ -24,11 +31,13 @@ use lore_revision::immutable::ImmutableError; use lore_revision::immutable::read_options_from_repository; use lore_revision::lore::RepositoryId; use lore_revision::repository::RepositoryContext; +use lore_storage::StoreError; +use lore_storage::StoreMatch; use lore_transport::grpc::CORRELATION_ID_HEADER; -use reqwest::header::CONTENT_LENGTH; use serde::Deserialize; use thiserror::Error; use tokio::sync::mpsc::channel; +use tokio_stream::StreamExt; use tokio_stream::wrappers::ReceiverStream; use crate::http::log_http_error; @@ -56,6 +65,14 @@ pub enum RedeemError { ReadStream(ImmutableError), #[error("Failed to generate response headers: {0}")] HeaderGeneration(InvalidHeaderValue), + #[error("Requested byte range is not satisfiable for a {0}-byte representation")] + RangeNotSatisfiable(u64), + #[error("Failed to query content metadata: {0}")] + Query(StoreError), + #[error("Address not found")] + NotFound, + #[error("Signed content length {signed} does not match stored content length {stored}")] + ContentLengthMismatch { signed: u64, stored: u64 }, } impl IntoResponse for RedeemError { @@ -71,11 +88,18 @@ impl IntoResponse for RedeemError { RedeemError::ReadStream(e) if e.is_address_not_found() || e.is_payload_not_found() => { (StatusCode::NOT_FOUND, "address not found".to_string()) } + RedeemError::NotFound => (StatusCode::NOT_FOUND, "address not found".to_string()), + RedeemError::RangeNotSatisfiable(_) => { + (StatusCode::RANGE_NOT_SATISFIABLE, self.to_string()) + } RedeemError::NotConfigured => ( StatusCode::NOT_FOUND, "presigned URL feature is not enabled".to_string(), ), - RedeemError::ReadStream(_) | RedeemError::HeaderGeneration(_) => ( + RedeemError::ReadStream(_) + | RedeemError::HeaderGeneration(_) + | RedeemError::Query(_) + | RedeemError::ContentLengthMismatch { .. } => ( StatusCode::INTERNAL_SERVER_ERROR, "Something went wrong. See server log for more info.".to_string(), ), @@ -85,6 +109,12 @@ impl IntoResponse for RedeemError { let mut headers = HeaderMap::new(); headers.insert("content-type", "text/plain".parse().unwrap()); + if let RedeemError::RangeNotSatisfiable(content_length) = &self { + headers.insert(ACCEPT_RANGES, HeaderValue::from_static("bytes")); + if let Ok(value) = HeaderValue::from_str(&format!("bytes */{content_length}")) { + headers.insert(CONTENT_RANGE, value); + } + } (status, headers, msg).into_response() } } @@ -153,13 +183,62 @@ pub async fn handler( )); let options = read_options_from_repository(&repository); - - let (tx, rx) = channel(CHUNKED_RESPONSE_BUFFER_SIZE); - - let content_length = - immutable::read_stream(repository, parsed_address, None, options, tx) - .await - .map_err(RedeemError::ReadStream)?; + let signed_content_length = payload.content_length; + let content_length = match signed_content_length { + Some(content_length) => content_length, + None if parsed_address.hash.is_zero() => 0, + None => { + let query = repository + .immutable_store() + .get_metadata(parsed_repository, parsed_address) + .await + .map_err(RedeemError::Query)?; + if query.match_made != StoreMatch::MatchFull { + return Err(RedeemError::NotFound); + } + query.fragment.size_content + } + }; + let etag = format!("\"{parsed_address}\""); + let range_header = match headers.get(IF_RANGE) { + Some(if_range) if if_range.as_bytes() != etag.as_bytes() => None, + _ => headers.get(RANGE), + }; + let requested_range = parse_range(range_header, content_length)?; + let response_range = requested_range.clone().unwrap_or(0..content_length); + + let body = if parsed_address.hash.is_zero() { + if content_length != 0 { + return Err(RedeemError::ContentLengthMismatch { + signed: content_length, + stored: 0, + }); + } + Body::empty() + } else { + let (tx, rx) = channel(CHUNKED_RESPONSE_BUFFER_SIZE); + let (stored_length, normalized_range) = immutable::read_stream_range( + repository, + parsed_address, + requested_range.clone(), + options, + tx, + ) + .await + .map_err(RedeemError::ReadStream)?; + if stored_length != content_length { + return Err(RedeemError::ContentLengthMismatch { + signed: content_length, + stored: stored_length, + }); + } + if normalized_range != response_range { + return Err(RedeemError::RangeNotSatisfiable(content_length)); + } + let stream = + ReceiverStream::new(rx).map(|item| item.map_err(RedeemError::ReadStream)); + Body::from_stream(stream) + }; let mut response_headers = HeaderMap::new(); response_headers.insert(CONTENT_TYPE, content_type); @@ -178,9 +257,26 @@ pub async fn handler( response_headers.insert( CONTENT_LENGTH, - HeaderValue::from_str(&format!("{content_length}")) + HeaderValue::from_str(&(response_range.end - response_range.start).to_string()) .map_err(RedeemError::HeaderGeneration)?, ); + response_headers.insert(ACCEPT_RANGES, HeaderValue::from_static("bytes")); + response_headers.insert( + ETAG, + HeaderValue::from_str(&etag).map_err(RedeemError::HeaderGeneration)?, + ); + if requested_range.is_some() { + response_headers.insert( + CONTENT_RANGE, + HeaderValue::from_str(&format!( + "bytes {}-{}/{}", + response_range.start, + response_range.end - 1, + content_length + )) + .map_err(RedeemError::HeaderGeneration)?, + ); + } let remaining_ttl = payload.expires_at.saturating_sub(now); response_headers.insert( @@ -191,22 +287,83 @@ pub async fn handler( apply_security_headers(&mut response_headers); - let stream = ReceiverStream::new(rx); - Ok((StatusCode::OK, response_headers, Body::from_stream(stream))) + let status = if requested_range.is_some() { + StatusCode::PARTIAL_CONTENT + } else { + StatusCode::OK + }; + Ok((status, response_headers, body)) }) .await } +fn parse_range( + header: Option<&HeaderValue>, + content_length: u64, +) -> Result>, RedeemError> { + let Some(header) = header else { + return Ok(None); + }; + let value = header + .to_str() + .map_err(|_| RedeemError::RangeNotSatisfiable(content_length))?; + let value = value + .strip_prefix("bytes=") + .ok_or(RedeemError::RangeNotSatisfiable(content_length))?; + if value.contains(',') || content_length == 0 { + return Err(RedeemError::RangeNotSatisfiable(content_length)); + } + let (start, end) = value + .split_once('-') + .ok_or(RedeemError::RangeNotSatisfiable(content_length))?; + + let range = if start.is_empty() { + let suffix = end + .parse::() + .ok() + .filter(|suffix| *suffix != 0) + .ok_or(RedeemError::RangeNotSatisfiable(content_length))?; + content_length.saturating_sub(suffix)..content_length + } else { + let start = start + .parse::() + .map_err(|_| RedeemError::RangeNotSatisfiable(content_length))?; + if start >= content_length { + return Err(RedeemError::RangeNotSatisfiable(content_length)); + } + let end_exclusive = if end.is_empty() { + content_length + } else { + end.parse::() + .map_err(|_| RedeemError::RangeNotSatisfiable(content_length))? + .saturating_add(1) + .min(content_length) + }; + if end_exclusive <= start { + return Err(RedeemError::RangeNotSatisfiable(content_length)); + } + start..end_exclusive + }; + Ok(Some(range)) +} + #[cfg(test)] mod tests { use std::sync::Arc; use std::time::SystemTime; use std::time::UNIX_EPOCH; + use axum::http::HeaderValue; use axum::http::StatusCode; + use axum::http::header::ACCEPT_RANGES; + use axum::http::header::CONTENT_LENGTH; + use axum::http::header::CONTENT_RANGE; use axum::http::header::CONTENT_TYPE; + use axum::http::header::IF_RANGE; + use axum::http::header::RANGE; use axum_test::TestServer; use lore_base::runtime::LORE_CONTEXT; + use lore_base::types::Address; use lore_revision::fragment; use lore_revision::lore::RepositoryId; use rand::random; @@ -249,6 +406,7 @@ mod tests { content_type: content_type.map(String::from), content_encoding: None, content_disposition: None, + content_length: None, }; sign(&payload, &config.hmac_key) } @@ -358,6 +516,7 @@ mod tests { content_type: None, content_encoding: None, content_disposition: None, + content_length: None, }; let token = sign(&payload, &config.hmac_key); @@ -432,6 +591,153 @@ mod tests { .await; } + #[tokio::test] + async fn returns_206_for_a_logical_byte_range() { + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("Failed to create stores"); + LORE_CONTEXT + .scope(execution, async move { + let repository = random::(); + let (fragment_data, address, payload) = fragment::generate_random(); + assert!(payload.len() > 8, "test payload must cover requested range"); + immutable_store + .clone() + .put( + repository, + address, + fragment_data, + Some(payload.clone()), + false, + ) + .await + .expect("Failed to put data in immutable store"); + + let config = presign_config(); + let repo_hex = repository.to_string(); + let address_str = address.to_string(); + let token = valid_token(&repo_hex, &address_str, &config); + let test_health = ServerHealth::new_without_availability(immutable_store.clone()); + let state = ServerState { + immutable_store, + mutable_store, + jwt_verifier: None, + max_file_size: 100, + presign_config: Some(config), + }; + let app = create_router(state, test_health, &LoreHttpServerSettings::default()); + let server = TestServer::new(app).unwrap(); + + let response = server + .get(&format!("/v1/presigned/{repo_hex}/{address_str}")) + .add_query_param("token", token) + .add_header(RANGE, HeaderValue::from_static("bytes=3-7")) + .add_header( + IF_RANGE, + HeaderValue::from_str(&format!("\"{address_str}\"")).unwrap(), + ) + .await; + + assert_eq!(response.status_code(), StatusCode::PARTIAL_CONTENT); + assert_eq!(response.headers()[CONTENT_RANGE], "bytes 3-7/32"); + assert_eq!(response.headers()[CONTENT_LENGTH], "5"); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + assert_eq!(response.as_bytes(), &payload[3..8]); + }) + .await; + } + + #[tokio::test] + async fn returns_416_for_an_out_of_bounds_range() { + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("Failed to create stores"); + LORE_CONTEXT + .scope(execution, async move { + let repository = random::(); + let (fragment_data, address, payload) = fragment::generate_random(); + immutable_store + .clone() + .put(repository, address, fragment_data, Some(payload), false) + .await + .expect("Failed to put data in immutable store"); + let config = presign_config(); + let repo_hex = repository.to_string(); + let address_str = address.to_string(); + let token = valid_token(&repo_hex, &address_str, &config); + let test_health = ServerHealth::new_without_availability(immutable_store.clone()); + let state = ServerState { + immutable_store, + mutable_store, + jwt_verifier: None, + max_file_size: 100, + presign_config: Some(config), + }; + let app = create_router(state, test_health, &LoreHttpServerSettings::default()); + let server = TestServer::new(app).unwrap(); + + let response = server + .get(&format!("/v1/presigned/{repo_hex}/{address_str}")) + .add_query_param("token", token) + .add_header(RANGE, HeaderValue::from_static("bytes=100-200")) + .await; + + assert_eq!(response.status_code(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!(response.headers()[CONTENT_RANGE], "bytes */32"); + }) + .await; + } + + #[tokio::test] + async fn returns_an_empty_body_for_a_signed_zero_address() { + let (immutable_store, mutable_store, execution) = + test_store_create().await.expect("Failed to create stores"); + LORE_CONTEXT + .scope(execution, async move { + let repository = random::(); + let address = Address::default().to_string(); + let config = presign_config(); + let repo_hex = repository.to_string(); + let expires_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + + 3600; + let token = sign( + &PresignTokenPayload { + version: CURRENT_TOKEN_VERSION, + key_id: config.key_id.clone(), + repository: repo_hex.clone(), + address: address.clone(), + expires_at, + content_type: Some("application/octet-stream".to_string()), + content_encoding: None, + content_disposition: Some("attachment; filename=\"empty.txt\"".to_string()), + content_length: Some(0), + }, + &config.hmac_key, + ); + let test_health = ServerHealth::new_without_availability(immutable_store.clone()); + let state = ServerState { + immutable_store, + mutable_store, + jwt_verifier: None, + max_file_size: 100, + presign_config: Some(config), + }; + let app = create_router(state, test_health, &LoreHttpServerSettings::default()); + let server = TestServer::new(app).unwrap(); + + let response = server + .get(&format!("/v1/presigned/{repo_hex}/{address}")) + .add_query_param("token", token) + .await; + + assert_eq!(response.status_code(), StatusCode::OK); + assert_eq!(response.headers()[CONTENT_LENGTH], "0"); + assert!(response.as_bytes().is_empty()); + }) + .await; + } + #[tokio::test] async fn returns_400_when_no_token_query_param() { let (immutable_store, mutable_store, execution) = diff --git a/lore-server/src/http/repositories/repository/contents/content/get_repository_content.rs b/lore-server/src/http/repositories/repository/contents/content/get_repository_content.rs index cdbf66a38..958d975dd 100644 --- a/lore-server/src/http/repositories/repository/contents/content/get_repository_content.rs +++ b/lore-server/src/http/repositories/repository/contents/content/get_repository_content.rs @@ -28,6 +28,7 @@ use reqwest::header::CONTENT_LENGTH; use serde::Deserialize; use thiserror::Error; use tokio::sync::mpsc::channel; +use tokio_stream::StreamExt; use tokio_stream::wrappers::ReceiverStream; use tracing::debug; @@ -156,7 +157,8 @@ pub async fn handler( .await .map_err(GetContentError::ReadStream)?; - let stream = ReceiverStream::new(rx); + let stream = + ReceiverStream::new(rx).map(|item| item.map_err(GetContentError::ReadStream)); let headers = create_stream_response_headers(query, content_length) .map_err(GetContentError::HeaderGeneration)?; diff --git a/lore-server/src/http/repositories/repository/contents/content/presign_repository_content.rs b/lore-server/src/http/repositories/repository/contents/content/presign_repository_content.rs index f377a9742..b1e90319f 100644 --- a/lore-server/src/http/repositories/repository/contents/content/presign_repository_content.rs +++ b/lore-server/src/http/repositories/repository/contents/content/presign_repository_content.rs @@ -18,7 +18,6 @@ use lore_base::runtime::LORE_CONTEXT; use lore_base::types::Address; use lore_revision::lore::RepositoryId; use lore_storage::StoreMatch; -use lore_storage::immutable_store::query_one; use lore_transport::grpc::CORRELATION_ID_HEADER; use serde::Deserialize; use serde::Serialize; @@ -176,8 +175,11 @@ pub async fn handler( LORE_CONTEXT .scope(execution, async move { - // Verify the address exists before issuing a URL for it. - let match_result = query_one(&immutable_store, repository, parsed_address) + // Verify the exact repository/context association and retain the + // logical length so redemption can serve byte ranges without a + // separate metadata round trip. + let match_result = immutable_store + .get_metadata(repository, parsed_address) .await .map_err(|e| { warn!(%e, "Presign resolve check failed"); @@ -212,6 +214,7 @@ pub async fn handler( content_type: body.content_type, content_encoding: body.content_encoding, content_disposition: body.content_disposition, + content_length: Some(match_result.fragment.size_content), }; let token_str = sign(&payload, &presign_config.hmac_key); diff --git a/lore-server/src/http/server.rs b/lore-server/src/http/server.rs index fa92d8446..9e4b3d01c 100644 --- a/lore-server/src/http/server.rs +++ b/lore-server/src/http/server.rs @@ -193,7 +193,7 @@ fn describe_allowed_types(types: &[String]) -> String { } } -fn build_presign_config(settings: &PresignSettings) -> Result> { +pub(crate) fn build_presign_config(settings: &PresignSettings) -> Result> { let Some(key_hex) = settings.hmac_key.as_deref() else { return Ok(None); }; diff --git a/lore-server/src/plugins/aws.rs b/lore-server/src/plugins/aws.rs index 625bea839..40b8bd480 100644 --- a/lore-server/src/plugins/aws.rs +++ b/lore-server/src/plugins/aws.rs @@ -119,6 +119,11 @@ pub struct AwsImmutableStorePluginConfig { /// non-AWS hostnames like `MinIO` in Docker). #[serde(default)] pub s3_force_path_style: bool, + + /// Allow clients to download immutable fragment payloads directly from S3 + /// using short-lived presigned URLs. Disable to force server-proxied reads. + #[serde(default = "default_direct_downloads_enabled")] + pub direct_downloads_enabled: bool, } /// Configuration for the AWS mutable store plugin. @@ -195,6 +200,10 @@ fn default_timeout() -> u64 { 5000 } +fn default_direct_downloads_enabled() -> bool { + true +} + // ============================================================================= // Plugin Factory Implementations // ============================================================================= @@ -338,7 +347,8 @@ impl ImmutableStorePluginFactory for AwsImmutableStorePluginFactory { s3_settings, dynamodb_settings, plugin_config.force_write, - ); + ) + .with_direct_downloads_enabled(plugin_config.direct_downloads_enabled); let store = AwsImmutableStore::new(s3_client, dynamodb_client, &store_settings); @@ -671,6 +681,7 @@ mod tests { dynamodb_slow_operation_threshold_millis = 500 timeout_millis = 3000 force_write = true + direct_downloads_enabled = false "#; let config: toml::Value = toml::from_str(config_str).unwrap(); @@ -696,6 +707,7 @@ mod tests { assert_eq!(plugin_config.dynamodb_slow_operation_threshold_millis, 500); assert_eq!(plugin_config.timeout_millis, 3000); assert!(plugin_config.force_write); + assert!(!plugin_config.direct_downloads_enabled); } /// A configuration written before this change points at the table holding fragment metadata, @@ -764,6 +776,7 @@ mod tests { ); assert_eq!(plugin_config.timeout_millis, 5000); assert!(!plugin_config.force_write); + assert!(plugin_config.direct_downloads_enabled); } #[tokio::test] diff --git a/lore-server/src/protocol/storage.rs b/lore-server/src/protocol/storage.rs index 533391fd9..bdad63834 100644 --- a/lore-server/src/protocol/storage.rs +++ b/lore-server/src/protocol/storage.rs @@ -15,6 +15,7 @@ pub mod mutable_cas; pub mod mutable_load; pub mod mutable_store_handler; pub mod ping; +pub mod presign_download; pub mod put; pub mod put_resolved; pub mod query; @@ -31,6 +32,7 @@ pub mod requests { pub use crate::protocol::storage::mutable_load::MutableLoad; pub use crate::protocol::storage::mutable_store_handler::MutableStoreOp; pub use crate::protocol::storage::ping::Ping; + pub use crate::protocol::storage::presign_download::PresignDownload; pub use crate::protocol::storage::put::Put; pub use crate::protocol::storage::put_resolved::PutResolved; pub use crate::protocol::storage::query::Query; @@ -48,6 +50,7 @@ pub mod responses { pub use crate::protocol::storage::mutable_load::MutableLoadResponse; pub use crate::protocol::storage::mutable_store_handler::MutableStoreResponse; pub use crate::protocol::storage::ping::PingResponse; + pub use crate::protocol::storage::presign_download::PresignDownloadResponse; pub use crate::protocol::storage::put::PutResponse; pub use crate::protocol::storage::put_resolved::PutResolvedResponse; pub use crate::protocol::storage::query::QueryResponse; diff --git a/lore-server/src/protocol/storage/messages.rs b/lore-server/src/protocol/storage/messages.rs index 4c90a83ab..62c8d03be 100644 --- a/lore-server/src/protocol/storage/messages.rs +++ b/lore-server/src/protocol/storage/messages.rs @@ -97,6 +97,7 @@ impl From for MessageHandleError { match value { StoreError::SlowDown(_) => MessageHandleError::SlowDown, StoreError::Oversized(_) => MessageHandleError::Oversized, + StoreError::NotSupported(_) => MessageHandleError::NotImplemented, _ => MessageHandleError::StoreFailure, } } @@ -146,6 +147,7 @@ pub enum LoreResponse { Put(responses::PutResponse), Query(responses::QueryResponse), Ping(responses::PingResponse), + PresignDownload(responses::PresignDownloadResponse), Correlate(responses::CorrelateResponse), Verify(responses::VerifyResponse), MutableLoad(responses::MutableLoadResponse), diff --git a/lore-server/src/protocol/storage/presign_download.rs b/lore-server/src/protocol/storage/presign_download.rs new file mode 100644 index 000000000..d441ecbdb --- /dev/null +++ b/lore-server/src/protocol/storage/presign_download.rs @@ -0,0 +1,261 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::BufMut; +use bytes::Bytes; +use bytes::BytesMut; +use lore_base::types::Address; +use lore_base::types::DirectDownload; +use lore_base::types::Fragment; +use lore_base::types::TypedBytes; +use lore_revision::lore::RepositoryId; +use lore_storage::ImmutableStore; +use lore_storage::StoreError; +use lore_storage::StoreMatch; +use tracing::debug; +use zerocopy::IntoBytes; + +use crate::protocol::attribute_map::AttributeMap; +use crate::protocol::storage::messages::LoreResponse; +use crate::protocol::storage::messages::Message; +use crate::protocol::storage::messages::MessageHandleError; +use crate::protocol::storage::messages::MessageParseError; +use crate::protocol::storage::messages::Response; +use crate::protocol::storage::query::MAX_FRAGMENTS; + +const EXPIRES_IN_BYTES: usize = size_of::(); +const MAX_FRAGMENTS_LENGTH: usize = size_of::
() * MAX_FRAGMENTS; +const DEFAULT_PRESIGN_EXPIRES_IN_SECONDS: u64 = 300; +const MAX_PRESIGN_EXPIRES_IN_SECONDS: u64 = 3600; + +#[derive(Clone, Debug, PartialEq)] +pub struct PresignDownload { + pub expires_in: Duration, + pub address: Bytes, +} + +impl PresignDownload { + pub fn parse(bytes: Bytes) -> Result + where + Self: Sized, + { + if bytes.len() < EXPIRES_IN_BYTES { + return Err(MessageParseError::InvalidFieldLength); + } + + let expires_in_seconds = u64::from_le_bytes( + bytes[..EXPIRES_IN_BYTES] + .try_into() + .map_err(|_| MessageParseError::InvalidFieldLength)?, + ); + let expires_in_seconds = if expires_in_seconds == 0 { + DEFAULT_PRESIGN_EXPIRES_IN_SECONDS + } else { + expires_in_seconds.min(MAX_PRESIGN_EXPIRES_IN_SECONDS) + }; + let address = bytes.slice(EXPIRES_IN_BYTES..); + let length = address.len(); + if !length.is_multiple_of(size_of::
()) { + return Err(MessageParseError::InvalidQueryLength); + } + if length > MAX_FRAGMENTS_LENGTH { + return Err(MessageParseError::TooManyFragments( + MAX_FRAGMENTS, + length / size_of::
(), + )); + } + + Ok(Self { + expires_in: Duration::from_secs(expires_in_seconds), + address, + }) + } +} + +fn map_store_error(err: StoreError) -> MessageHandleError { + match err { + StoreError::AddressNotFound(_) => MessageHandleError::FragmentNotFound, + StoreError::SlowDown(_) => MessageHandleError::SlowDown, + StoreError::Oversized(_) => MessageHandleError::Oversized, + StoreError::NotSupported(_) => MessageHandleError::NotImplemented, + _ => MessageHandleError::StoreFailure, + } +} + +pub async fn handle_presign_download( + address: &Bytes, + expires_in: Duration, + repository: RepositoryId, + immutable_store: Arc, +) -> Result { + let addresses = address.as_type_slice::
(); + debug!( + "Handling PresignDownload request for {} fragments in repository: {repository}", + addresses.len() + ); + + let downloads = immutable_store + .presign_downloads(repository, addresses, StoreMatch::MatchFull, expires_in) + .await + .map_err(map_store_error)?; + + Ok(LoreResponse::PresignDownload(PresignDownloadResponse { + downloads, + })) +} + +#[async_trait] +impl Message for PresignDownload { + #[tracing::instrument(name = "PresignDownload::handle", skip_all)] + async fn handle( + &self, + context: Arc, + immutable_store: Arc, + ) -> Result { + let repository = *context + .get_or::(MessageHandleError::NotConnected)?; + handle_presign_download(&self.address, self.expires_in, repository, immutable_store).await + } +} + +#[derive(Debug, PartialEq)] +pub struct PresignDownloadResponse { + pub downloads: Vec, +} + +impl Response for PresignDownloadResponse { + fn data(&self) -> Vec { + let url_bytes = self + .downloads + .iter() + .map(|download| download.url.len()) + .sum::(); + let record_overhead = + size_of::
() + size_of::() + size_of::() + size_of::(); + let mut buffer = BytesMut::with_capacity( + size_of::() + self.downloads.len() * record_overhead + url_bytes, + ); + + buffer.put_u32_le(self.downloads.len() as u32); + for download in &self.downloads { + buffer.extend_from_slice(download.address.as_bytes()); + buffer.extend_from_slice(download.fragment.as_bytes()); + buffer.put_u64_le(download.expires_at_epoch_seconds); + buffer.put_u32_le(download.url.len() as u32); + buffer.extend_from_slice(download.url.as_bytes()); + } + + vec![buffer.freeze()] + } +} + +#[cfg(test)] +mod tests { + use bytes::Buf; + use lore_base::types::Context; + use lore_base::types::FragmentFlags; + use lore_base::types::Hash; + use rand::random; + + use super::*; + + fn address(value: u64) -> Address { + Address { + hash: Hash::from_u64(value), + context: Context::from([value as u8; 16]), + } + } + + fn request_bytes(expires_in_seconds: u64, addresses: &[Address]) -> Bytes { + let mut bytes = + BytesMut::with_capacity(size_of::() + std::mem::size_of_val(addresses)); + bytes.put_u64_le(expires_in_seconds); + for address in addresses { + bytes.extend_from_slice(address.as_bytes()); + } + bytes.freeze() + } + + #[test] + fn test_parse_valid() { + let addresses = [address(1), address(2)]; + let parsed = PresignDownload::parse(request_bytes(60, &addresses)).expect("parse failed"); + + assert_eq!(parsed.expires_in, Duration::from_secs(60)); + assert_eq!(parsed.address.as_type_slice::
(), &addresses); + } + + #[test] + fn test_parse_uses_default_expiry_for_zero() { + let parsed = PresignDownload::parse(request_bytes(0, &[address(1)])).expect("parse failed"); + + assert_eq!( + parsed.expires_in, + Duration::from_secs(DEFAULT_PRESIGN_EXPIRES_IN_SECONDS) + ); + } + + #[test] + fn test_parse_clamps_expiry() { + let parsed = + PresignDownload::parse(request_bytes(u64::MAX, &[address(1)])).expect("parse failed"); + + assert_eq!( + parsed.expires_in, + Duration::from_secs(MAX_PRESIGN_EXPIRES_IN_SECONDS) + ); + } + + #[test] + fn test_parse_rejects_invalid_address_length() { + let mut bytes = request_bytes(60, &[address(1)]).to_vec(); + bytes.push(1); + + assert_eq!( + PresignDownload::parse(Bytes::from(bytes)), + Err(MessageParseError::InvalidQueryLength) + ); + } + + #[test] + fn test_response_data() { + let address = Address { + hash: random::(), + context: random::(), + }; + let fragment = Fragment { + flags: FragmentFlags::PayloadStoredDurable.bits(), + size_payload: 42, + size_content: 42, + }; + let response = PresignDownloadResponse { + downloads: vec![DirectDownload { + address, + fragment, + url: "http://127.0.0.1/object".to_string(), + expires_at_epoch_seconds: 123, + }], + }; + + let mut bytes = response.data().pop().expect("missing response bytes"); + assert_eq!(bytes.get_u32_le(), 1); + assert_eq!( + Address::from(&bytes.split_to(size_of::
())), + address + ); + assert_eq!( + Fragment::from(&bytes.split_to(size_of::())), + fragment + ); + assert_eq!(bytes.get_u64_le(), 123); + let url_len = bytes.get_u32_le() as usize; + assert_eq!( + String::from_utf8(bytes.split_to(url_len).to_vec()).unwrap(), + "http://127.0.0.1/object" + ); + assert!(bytes.is_empty()); + } +} diff --git a/lore-server/src/quic/storage_service.rs b/lore-server/src/quic/storage_service.rs index 492c55319..e4e9c11ab 100644 --- a/lore-server/src/quic/storage_service.rs +++ b/lore-server/src/quic/storage_service.rs @@ -100,6 +100,18 @@ pub(crate) fn build_storage_protocol_request_span( { CORRELATION_ID } = correlation_id, { USER_ID } = user_id, ), + Ok(Command::PresignDownload) => info_span!( + parent: None, + "StoragePresignDownloadTask", + { SAMPLING_TIER_LOW } = true, + { TRANSPORT } = %Transport::Quic, + { PROTOCOL } = %protocol, + { QUIC_OPCODE } = opcode_label, + { CONNECTION_ID } = connection_id, + { REPOSITORY_ID } = repository_id, + { CORRELATION_ID } = correlation_id, + { USER_ID } = user_id, + ), Ok(Command::Put) => info_span!( parent: None, "StoragePutTask", @@ -252,6 +264,7 @@ pub enum ParsedStorageRequest { /// Stores a fragment and publishes a mutable key naming it, saving the caller the round trip /// a separate `MutableStore` would cost. v4-only: it needs both stores. PutResolved(requests::PutResolved), + PresignDownload(requests::PresignDownload), Put(requests::Put), Query(requests::Query), Correlate(requests::Correlate), @@ -271,6 +284,7 @@ fn quic_error(message_error: &MessageHandleError) -> QuicServiceError { } MessageHandleError::SlowDown => QuicServiceError::SlowDown, MessageHandleError::Oversized => QuicServiceError::Oversized, + MessageHandleError::NotImplemented => QuicServiceError::NotSupported, _ => QuicServiceError::Failed, } } @@ -307,6 +321,9 @@ pub fn parse_message_for_opcode( Command::GetMetadata => Ok(ParsedStorageRequest::GetMetadata( crate::protocol::storage::get::GetMetadata::parse(bytes)?, )), + Command::PresignDownload => Ok(ParsedStorageRequest::PresignDownload( + requests::PresignDownload::parse(bytes)?, + )), Command::Put => Ok(ParsedStorageRequest::Put(requests::Put::parse(bytes)?)), Command::Query => Ok(ParsedStorageRequest::Query(requests::Query::parse(bytes)?)), Command::Verify => Ok(ParsedStorageRequest::Verify(requests::Verify::parse( diff --git a/lore-server/src/quic/storage_service_v4.rs b/lore-server/src/quic/storage_service_v4.rs index 67f554f2a..ae2d09167 100644 --- a/lore-server/src/quic/storage_service_v4.rs +++ b/lore-server/src/quic/storage_service_v4.rs @@ -30,6 +30,7 @@ use crate::protocol::storage::messages::Response; use crate::protocol::storage::mutable_cas::handle_mutable_cas; use crate::protocol::storage::mutable_load::handle_mutable_load; use crate::protocol::storage::mutable_store_handler::handle_mutable_store; +use crate::protocol::storage::presign_download::handle_presign_download; use crate::protocol::storage::put::handle_put; use crate::protocol::storage::query::handle_query; use crate::protocol::storage::session::SessionError; @@ -80,6 +81,7 @@ fn quic_error_v4(error: &MessageHandleError) -> QuicServiceError { QuicServiceError::SlowDown } MessageHandleError::Oversized => QuicServiceError::Oversized, + MessageHandleError::NotImplemented => QuicServiceError::NotSupported, _ => QuicServiceError::Failed, } } @@ -265,6 +267,17 @@ impl QuicService for StorageServiceV4 { ) .await } + crate::quic::storage_service::ParsedStorageRequest::PresignDownload( + presign, + ) => { + handle_presign_download( + &presign.address, + presign.expires_in, + repository, + self.immutable_store.clone(), + ) + .await + } crate::quic::storage_service::ParsedStorageRequest::Put(put) => { handle_put( &put, diff --git a/lore-server/src/server.rs b/lore-server/src/server.rs index a24129310..b3f250640 100644 --- a/lore-server/src/server.rs +++ b/lore-server/src/server.rs @@ -76,6 +76,7 @@ use crate::http::LoreHttpServer; use crate::http::security_headers::ContentTypePolicy; use crate::http::server::LoreHttpServerSettings; use crate::http::server::PresignSettings; +use crate::http::server::build_presign_config; use crate::plugins; use crate::plugins::PluginRegistry; use crate::plugins::traits::NotificationPluginContext; @@ -468,6 +469,19 @@ async fn launch_grpc_server( let mut environment = settings.environment.clone().unwrap_or_default(); let feature = settings.feature.clone().unwrap_or_default(); + let presign_config = match settings.server.http.as_ref().filter(|http| http.enabled) { + Some(http) => build_presign_config(&PresignSettings { + hmac_key: http.presigned_url_hmac_key.clone(), + min_ttl_seconds: http.presigned_url_min_ttl_seconds, + default_ttl_seconds: http.presigned_url_default_ttl_seconds, + max_ttl_seconds: http.presigned_url_max_ttl_seconds, + content_type_policy: ContentTypePolicy { + extra: http.presigned_url_extra_content_types.clone(), + denied: http.presigned_url_denied_content_types.clone(), + }, + })?, + None => None, + }; // Enforce store limits if let Some(limit) = immutable_store.max_query_batch() { @@ -502,6 +516,7 @@ async fn launch_grpc_server( service_settings, user_agent_filter, forwarded_requests, + presign_config, ) .with_jwt_verifier(jwt_verifier)? .serve(addr, async move { diff --git a/lore-storage/src/immutable_store.rs b/lore-storage/src/immutable_store.rs index 383169989..fa671357c 100644 --- a/lore-storage/src/immutable_store.rs +++ b/lore-storage/src/immutable_store.rs @@ -11,6 +11,7 @@ use lore_error_set::prelude::*; use crate::Address; use crate::Context; +use crate::DirectDownload; use crate::Fragment; use crate::FragmentFlags; use crate::FragmentReference; @@ -438,6 +439,22 @@ pub trait ImmutableStore: Any + Send + Sync { address: Address, ) -> Result; + /// Return presigned direct-download URLs for immutable payloads. + /// + /// Implementations must preserve `get` authorization semantics: a returned URL must only be + /// issued when `match_required` is satisfied for the given `(partition, address)`. + async fn presign_downloads( + self: Arc, + _partition: Partition, + _addresses: &[Address], + _match_required: StoreMatch, + _expires_in: Duration, + ) -> Result, StoreError> { + Err(StoreError::from(NotSupported { + operation: "immutable direct download".to_string(), + })) + } + /// Put the immutable data for the given address within the partition. /// If the payload buffer is not given and the store has no previous instance of the data, /// the function will return an error and the caller should try again after obtaining the payload. diff --git a/lore-storage/src/lib.rs b/lore-storage/src/lib.rs index 61eae91a8..aca5d40a7 100644 --- a/lore-storage/src/lib.rs +++ b/lore-storage/src/lib.rs @@ -26,6 +26,7 @@ pub mod options; pub mod packstore; pub mod read; pub mod store_types; +pub mod stream_write; #[cfg(test)] pub(crate) mod test_util; pub(crate) mod typed_bytes; @@ -99,6 +100,7 @@ pub use local::mutable_store::LocalMutableStoreError; pub use local::mutable_store::MutableStoreSettings; use lore_base::lore_info; use lore_base::lore_warn; +pub use lore_base::types::DirectDownload; // Re-export maintenance functions pub use maintenance::compactor; pub use maintenance::evictor; @@ -131,6 +133,8 @@ pub use store_types::StoreGetData; pub use store_types::StoreMatch; pub use store_types::StoreMatchResult; pub use store_types::StoreObliterateStats; +pub use stream_write::ContentStreamError; +pub use stream_write::write_content_stream; pub use typed_bytes::TypedBytes; pub use typed_bytes::TypedBytesMut; pub use types::Address; diff --git a/lore-storage/src/read.rs b/lore-storage/src/read.rs index 171f5d16f..5bfddac3c 100644 --- a/lore-storage/src/read.rs +++ b/lore-storage/src/read.rs @@ -616,6 +616,17 @@ pub async fn read_stream( ) .await?; + if let Some(max) = options.max_content_size + && fragment.size_content > max + { + return Err(StorageError::from(crate::errors::Oversized { + context: format!( + "fragment size_content {} exceeds caller-supplied max {max}", + fragment.size_content + ), + })); + } + let range = resolve_content_range(range, fragment.size_content); let streamed = range.start as u64..range.end as u64; if range.is_empty() { diff --git a/lore-storage/src/stream_write.rs b/lore-storage/src/stream_write.rs new file mode 100644 index 000000000..12ce11498 --- /dev/null +++ b/lore-storage/src/stream_write.rs @@ -0,0 +1,684 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +//! Bounded-memory ingestion of raw content streams into Lore's immutable +//! fragment graph. + +use std::mem::size_of; +use std::sync::Arc; + +use bytes::Bytes; +use bytes::BytesMut; +use lore_transport::StorageSession; +use tokio::io::AsyncRead; +use tokio_stream::StreamExt; +use zerocopy::FromZeros; +use zerocopy::IntoBytes; + +use crate::compress::FRAGMENT_SIZE_THRESHOLD; +use crate::concurrency::FRAGMENT_SIZE_EXPECTED; +use crate::concurrency::FRAGMENT_SIZE_MINIMUM; +use crate::error::StorageError; +use crate::fragment_flags::FragmentFlags; +use crate::hash; +use crate::immutable_store::ImmutableStore; +use crate::options::WriteOptions; +use crate::types::Address; +use crate::types::Context; +use crate::types::Fragment; +use crate::types::FragmentReference; +use crate::types::Hash; +use crate::types::Partition; +use crate::write::store_fragment; +use crate::write_tracker::WriteTracker; + +const MAX_FRAGMENT_TREE_DEPTH: usize = 8; + +struct StoredFragment { + address: Address, + fragment: Fragment, +} + +/// Errors specific to raw stream framing/integrity, plus failures from the +/// underlying immutable fragment store. +#[derive(Debug)] +pub enum ContentStreamError { + InvalidArguments(String), + Storage(StorageError), +} + +impl std::fmt::Display for ContentStreamError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidArguments(reason) => formatter.write_str(reason), + Self::Storage(error) => std::fmt::Display::fmt(error, formatter), + } + } +} + +impl std::error::Error for ContentStreamError {} + +impl From for ContentStreamError { + fn from(error: StorageError) -> Self { + Self::Storage(error) + } +} + +/// Stream raw content into Lore's immutable store using the same FastCDC and +/// fragment graph encoding as [`crate::write_content`]. Memory is bounded by +/// one CDC chunk plus at most one fragment-reference page at each tree level. +/// +/// `expected_size` is an integrity assertion, not a memory allocation hint. +/// A mismatch is reported after EOF; fragments written before the mismatch are +/// harmless unreachable content-addressed objects. +#[allow(clippy::too_many_arguments)] +pub async fn write_content_stream( + store: Arc, + partition: Partition, + context: Context, + reader: R, + expected_size: Option, + max_size: Option, + flags: WriteOptions, + remote_session: Option>, + tracker: Option>, +) -> Result<(Address, Fragment, u64), ContentStreamError> +where + R: AsyncRead + Unpin, +{ + write_content_stream_impl( + store, + partition, + context, + reader, + expected_size, + max_size, + flags, + remote_session, + tracker, + FRAGMENT_SIZE_THRESHOLD / size_of::(), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn write_content_stream_impl( + store: Arc, + partition: Partition, + context: Context, + reader: R, + expected_size: Option, + max_size: Option, + flags: WriteOptions, + remote_session: Option>, + tracker: Option>, + references_per_page: usize, +) -> Result<(Address, Fragment, u64), ContentStreamError> +where + R: AsyncRead + Unpin, +{ + if references_per_page < 2 { + return Err(StorageError::internal( + "fragment reference pages must hold at least two entries", + ) + .into()); + } + + let mut chunker = fastcdc::v2020::AsyncStreamCDC::with_level( + reader, + FRAGMENT_SIZE_MINIMUM as u32, + FRAGMENT_SIZE_EXPECTED as u32, + FRAGMENT_SIZE_THRESHOLD as u32, + fastcdc::v2020::Normalization::Level1, + ); + let chunks = chunker.as_stream(); + tokio::pin!(chunks); + + let mut first: Option<(StoredFragment, u64)> = None; + let mut builder = FragmentListBuilder::new( + store.clone(), + partition, + context, + flags, + remote_session.clone(), + tracker.clone(), + references_per_page, + ); + let mut actual_size = 0u64; + + while let Some(chunk) = chunks.next().await { + let chunk = chunk.map_err(|error| match error { + fastcdc::v2020::Error::IoError(error) + if error.kind() == std::io::ErrorKind::InvalidData => + { + ContentStreamError::InvalidArguments(error.to_string()) + } + error => ContentStreamError::from(StorageError::internal_with_context( + error, + "stream chunking failed", + )), + })?; + let offset = chunk.offset; + let chunk_size = u64::try_from(chunk.length) + .map_err(|error| StorageError::internal_with_context(error, "chunk size overflow"))?; + actual_size = offset + .checked_add(chunk_size) + .ok_or_else(|| StorageError::internal("stream size exceeds u64"))?; + if let Some(max_size) = max_size + && actual_size > max_size + { + return Err(StorageError::from(crate::errors::Oversized { + context: format!( + "streamed content size exceeds configured upload limit {max_size}" + ), + }) + .into()); + } + + let stored = store_leaf( + store.clone(), + partition, + context, + Bytes::from(chunk.data), + flags, + remote_session.clone(), + tracker.clone(), + ) + .await?; + + if let Some((first_stored, first_offset)) = first.take() { + builder + .push(FragmentReference { + hash: first_stored.address.hash, + offset_content: first_offset, + }) + .await?; + builder + .push(FragmentReference { + hash: stored.address.hash, + offset_content: offset, + }) + .await?; + } else if builder.is_empty() { + first = Some((stored, offset)); + } else { + builder + .push(FragmentReference { + hash: stored.address.hash, + offset_content: offset, + }) + .await?; + } + } + + if let Some(expected_size) = expected_size + && expected_size != actual_size + { + return Err(ContentStreamError::InvalidArguments(format!( + "expected content size {expected_size} does not match streamed size {actual_size}" + ))); + } + + if let Some((stored, _)) = first { + return Ok((stored.address, stored.fragment, actual_size)); + } + if builder.is_empty() { + return Ok(( + Address { + context, + hash: Hash::new_zeroed(), + }, + Fragment::new_zeroed(), + 0, + )); + } + + let stored = builder.finish(actual_size).await?; + Ok((stored.address, stored.fragment, actual_size)) +} + +#[allow(clippy::too_many_arguments)] +async fn store_leaf( + store: Arc, + partition: Partition, + context: Context, + buffer: Bytes, + flags: WriteOptions, + remote_session: Option>, + tracker: Option>, +) -> Result { + let address = Address { + context, + hash: hash::hash_slice(buffer.as_ref()), + }; + let fragment = Fragment { + flags: flags.into(), + size_payload: buffer.len() as u32, + size_content: buffer.len() as u64, + }; + let permit = crate::concurrency::acquire_fragment_memory_permit(buffer.len()).await; + let stored = store_fragment( + store, + partition, + address, + fragment, + buffer, + flags.local_cache_priority, + remote_session, + tracker, + permit, + ) + .await?; + Ok(StoredFragment { + address: stored.address, + fragment, + }) +} + +struct FragmentListBuilder { + store: Arc, + partition: Partition, + context: Context, + flags: WriteOptions, + remote_session: Option>, + tracker: Option>, + references_per_page: usize, + levels: Vec>, +} + +impl FragmentListBuilder { + #[allow(clippy::too_many_arguments)] + fn new( + store: Arc, + partition: Partition, + context: Context, + flags: WriteOptions, + remote_session: Option>, + tracker: Option>, + references_per_page: usize, + ) -> Self { + Self { + store, + partition, + context, + flags, + remote_session, + tracker, + references_per_page, + levels: vec![Vec::with_capacity(references_per_page + 1)], + } + } + + fn is_empty(&self) -> bool { + self.levels.iter().all(Vec::is_empty) + } + + async fn push(&mut self, reference: FragmentReference) -> Result<(), StorageError> { + let mut level = 0usize; + let mut reference = reference; + loop { + if level >= MAX_FRAGMENT_TREE_DEPTH { + return Err(StorageError::internal(format!( + "fragment tree depth exceeds {MAX_FRAGMENT_TREE_DEPTH}" + ))); + } + if self.levels.len() <= level { + self.levels + .push(Vec::with_capacity(self.references_per_page + 1)); + } + let entries = &mut self.levels[level]; + entries.push(reference); + if entries.len() <= self.references_per_page { + return Ok(()); + } + + let end_offset = entries + .last() + .expect("overflow page has a lookahead entry") + .offset_content; + let page = entries + .drain(..self.references_per_page) + .collect::>(); + let start_offset = page[0].offset_content; + let stored = self + .store_page(&page, start_offset, end_offset, false) + .await?; + reference = FragmentReference { + hash: stored.address.hash, + offset_content: start_offset, + }; + level += 1; + } + } + + async fn finish(mut self, content_size: u64) -> Result { + let mut level = 0usize; + loop { + let has_higher = self + .levels + .iter() + .skip(level + 1) + .any(|entries| !entries.is_empty()); + let page = std::mem::take( + self.levels + .get_mut(level) + .ok_or_else(|| StorageError::internal("missing fragment tree level"))?, + ); + if page.is_empty() { + return Err(StorageError::internal("empty fragment reference page")); + } + let start_offset = page[0].offset_content; + let stored = self + .store_page(&page, start_offset, content_size, !has_higher) + .await?; + if !has_higher { + return Ok(stored); + } + + self.push_at_level( + level + 1, + FragmentReference { + hash: stored.address.hash, + offset_content: start_offset, + }, + ) + .await?; + level += 1; + } + } + + async fn push_at_level( + &mut self, + mut level: usize, + mut reference: FragmentReference, + ) -> Result<(), StorageError> { + loop { + if level >= MAX_FRAGMENT_TREE_DEPTH { + return Err(StorageError::internal(format!( + "fragment tree depth exceeds {MAX_FRAGMENT_TREE_DEPTH}" + ))); + } + if self.levels.len() <= level { + self.levels + .push(Vec::with_capacity(self.references_per_page + 1)); + } + let entries = &mut self.levels[level]; + entries.push(reference); + if entries.len() <= self.references_per_page { + return Ok(()); + } + let end_offset = entries + .last() + .expect("overflow page has a lookahead entry") + .offset_content; + let page = entries + .drain(..self.references_per_page) + .collect::>(); + let start_offset = page[0].offset_content; + let stored = self + .store_page(&page, start_offset, end_offset, false) + .await?; + reference = FragmentReference { + hash: stored.address.hash, + offset_content: start_offset, + }; + level += 1; + } + } + + async fn store_page( + &self, + references: &[FragmentReference], + start_offset: u64, + end_offset: u64, + root: bool, + ) -> Result { + let size_content = end_offset + .checked_sub(start_offset) + .ok_or_else(|| StorageError::internal("fragment page offsets are not monotonic"))?; + let payload_size = references + .len() + .checked_mul(size_of::()) + .ok_or_else(|| StorageError::internal("fragment page payload size overflow"))?; + if payload_size > FRAGMENT_SIZE_THRESHOLD { + return Err(StorageError::internal( + "fragment reference page exceeds fragment threshold", + )); + } + let mut payload = BytesMut::with_capacity(payload_size); + for reference in references { + payload.extend_from_slice(reference.as_bytes()); + } + let payload = payload.freeze(); + let address = Address { + context: self.context, + hash: hash::hash_slice(payload.as_ref()), + }; + let fragment = Fragment { + flags: self.flags.as_u32() | FragmentFlags::PayloadFragmented.bits(), + size_payload: payload.len() as u32, + size_content, + }; + let permit = crate::concurrency::acquire_fragment_memory_permit(payload.len()).await; + let stored = store_fragment( + self.store.clone(), + self.partition, + address, + fragment, + payload, + root || self.flags.local_cache_priority, + self.remote_session.clone(), + self.tracker.clone(), + permit, + ) + .await?; + Ok(StoredFragment { + address: stored.address, + fragment, + }) + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + use std::path::PathBuf; + + use super::*; + use crate::local::immutable_store::ImmutableStoreSettings; + use crate::local::immutable_store::LocalImmutableStore; + use crate::options::ReadOptions; + use crate::test_util::TempDir; + + async fn make_test_store(label: &str) -> (TempDir, Arc) { + let dir = TempDir::new(label); + let store = LocalImmutableStore::new( + Some(PathBuf::from(dir.as_ref())), + ImmutableStoreSettings::default(), + ) + .await + .expect("create test store"); + (dir, store) + } + + fn content(size: usize) -> Vec { + let mut state = 0x9E37_79B9u32; + (0..size) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state as u8 + }) + .collect() + } + + #[tokio::test] + async fn stream_writer_matches_buffer_writer_fragment_graph() { + let (_dir, store) = make_test_store("lore-stream-parity-").await; + let partition = Partition::from([0x31; 16]); + let context = Context::from([0x41; 16]); + let payload = content(FRAGMENT_SIZE_THRESHOLD * 4 + 17); + + let (stream_address, stream_fragment, size) = write_content_stream( + store.clone(), + partition, + context, + Cursor::new(payload.clone()), + Some(payload.len() as u64), + None, + WriteOptions::default(), + None, + None, + ) + .await + .expect("stream write"); + let buffer_result = crate::write_content( + store.clone(), + partition, + context, + Bytes::from(payload), + WriteOptions::default(), + None, + None, + None, + ) + .await + .expect("buffer write"); + let buffer_address = buffer_result.address; + let buffer_fragment = store + .clone() + .get_metadata(partition, buffer_address) + .await + .expect("buffer metadata") + .fragment; + + assert_eq!(size, buffer_fragment.size_content); + assert_eq!(stream_address, buffer_address); + assert_eq!(stream_fragment.size_content, buffer_fragment.size_content); + assert_eq!(stream_fragment.size_payload, buffer_fragment.size_payload); + } + + #[tokio::test] + async fn stream_writer_handles_fragment_threshold_boundaries() { + let (_dir, store) = make_test_store("lore-stream-boundaries-").await; + let partition = Partition::from([0x35; 16]); + + for (index, size) in [ + FRAGMENT_SIZE_THRESHOLD - 1, + FRAGMENT_SIZE_THRESHOLD, + FRAGMENT_SIZE_THRESHOLD + 1, + ] + .into_iter() + .enumerate() + { + let payload = content(size); + let context = Context::from([0x50 + index as u8; 16]); + let (address, fragment, actual) = write_content_stream( + store.clone(), + partition, + context, + Cursor::new(payload.clone()), + Some(size as u64), + None, + WriteOptions::default(), + None, + None, + ) + .await + .expect("boundary stream write"); + assert_eq!(actual, size as u64); + assert_eq!(fragment.size_content, size as u64); + let (_, restored) = crate::read( + store.clone(), + partition, + address, + None, + ReadOptions::default(), + None, + ) + .await + .expect("boundary content read"); + assert_eq!(restored.as_ref(), payload.as_slice()); + } + } + + #[tokio::test] + async fn stream_writer_reassembles_incremental_multilevel_lists() { + let (_dir, store) = make_test_store("lore-stream-multilevel-").await; + let partition = Partition::from([0x32; 16]); + let context = Context::from([0x42; 16]); + let payload = content(FRAGMENT_SIZE_THRESHOLD * 6 + 29); + + let (address, fragment, size) = write_content_stream_impl( + store.clone(), + partition, + context, + Cursor::new(payload.clone()), + None, + None, + WriteOptions::default(), + None, + None, + 2, + ) + .await + .expect("stream write with tiny list pages"); + assert_ne!(fragment.flags & FragmentFlags::PayloadFragmented.bits(), 0); + assert_eq!(size, payload.len() as u64); + + let (_, restored) = crate::read( + store, + partition, + address, + None, + ReadOptions::default(), + None, + ) + .await + .expect("reassemble streamed fragment tree"); + assert_eq!(restored.as_ref(), payload.as_slice()); + } + + #[tokio::test] + async fn stream_writer_rejects_expected_size_mismatch() { + let (_dir, store) = make_test_store("lore-stream-size-").await; + let payload = content(4096); + let error = write_content_stream( + store, + Partition::from([0x33; 16]), + Context::from([0x43; 16]), + Cursor::new(payload.clone()), + Some(payload.len() as u64 + 1), + None, + WriteOptions::default(), + None, + None, + ) + .await + .expect_err("size mismatch must fail"); + assert!(matches!(error, ContentStreamError::InvalidArguments(_))); + } + + #[tokio::test] + async fn stream_writer_supports_empty_content_without_storing_a_fragment() { + let (_dir, store) = make_test_store("lore-stream-empty-").await; + let context = Context::from([0x44; 16]); + let (address, fragment, size) = write_content_stream( + store, + Partition::from([0x34; 16]), + context, + Cursor::new(Vec::::new()), + Some(0), + None, + WriteOptions::default(), + None, + None, + ) + .await + .expect("empty stream"); + assert_eq!(address.context, context); + assert!(address.hash.is_zero()); + assert_eq!(fragment, Fragment::new_zeroed()); + assert_eq!(size, 0); + } +} diff --git a/lore-transport/Cargo.toml b/lore-transport/Cargo.toml index 04494e951..09dce56e9 100644 --- a/lore-transport/Cargo.toml +++ b/lore-transport/Cargo.toml @@ -25,6 +25,7 @@ parking_lot = { workspace = true } quinn = { workspace = true } quinn-proto = { workspace = true } rcgen = { workspace = true } +reqwest = { workspace = true } rustls = { workspace = true } rustls-native-certs = { workspace = true } serde = { workspace = true } diff --git a/lore-transport/examples/presign_download_bench.rs b/lore-transport/examples/presign_download_bench.rs new file mode 100644 index 000000000..ddbca6ac7 --- /dev/null +++ b/lore-transport/examples/presign_download_bench.rs @@ -0,0 +1,328 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::error::Error; +use std::sync::Arc; +use std::time::Instant; + +use bytes::Bytes; +use futures::StreamExt; +use futures::stream; +use lore_base::types::Address; +use lore_base::types::Context; +use lore_base::types::FRAGMENT_SIZE_THRESHOLD; +use lore_base::types::Hash; +use lore_proto::lore::model::v1 as model_v1; +use lore_proto::lore::storage::v1 as storage_v1; +use lore_proto::lore::storage::v1::storage_service_client::StorageServiceClient; +use lore_transport::grpc::REPOSITORY_ID_KEY; +use tokio::sync::Semaphore; +use tonic::Request; +use tonic::metadata::MetadataValue; + +const ENDPOINT: &str = "http://127.0.0.1:41337"; +const DIRECT_DOWNLOAD_CONCURRENCY: usize = 64; +const ROUNDS: usize = 5; +const PRESIGN_EXPIRES_SECONDS: u64 = 300; + +type BenchError = Box; + +#[derive(Clone, Copy)] +struct Scenario { + name: &'static str, + count: usize, + payload_size: usize, +} + +#[derive(Clone)] +struct BenchFragment { + address: Address, + payload: Bytes, +} + +struct RoundStats { + bytes: usize, + server_get_ms: f64, + presign_ms: f64, + direct_http_ms: f64, + direct_total_ms: f64, +} + +#[tokio::main] +async fn main() -> Result<(), BenchError> { + let endpoint = std::env::args() + .nth(1) + .unwrap_or_else(|| ENDPOINT.to_string()); + let repository = Context::from([ + 0x70, 0x72, 0x65, 0x73, 0x69, 0x67, 0x6e, 0x2d, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x2d, 0x30, + 0x31, + ]); + + let scenarios = [ + Scenario { + name: "many-small", + count: 256, + payload_size: 4 * 1024, + }, + Scenario { + name: "medium", + count: 64, + payload_size: 64 * 1024, + }, + Scenario { + name: "max-fragment", + count: 32, + payload_size: FRAGMENT_SIZE_THRESHOLD, + }, + ]; + + println!( + "endpoint={endpoint} repository={} rounds={ROUNDS} direct_http_concurrency={DIRECT_DOWNLOAD_CONCURRENCY}", + repository + ); + println!( + "| scenario | fragments | bytes | server_get_avg_ms | presign_avg_ms | direct_http_avg_ms | direct_total_avg_ms | direct_vs_server |" + ); + println!("|---|---:|---:|---:|---:|---:|---:|---:|"); + + for scenario in scenarios { + let fragments = make_fragments(repository, scenario); + seed_fragments(&endpoint, repository, &fragments).await?; + + let mut round_stats = Vec::new(); + let addresses = fragments + .iter() + .map(|fragment| fragment.address) + .collect::>(); + + // Warm both paths before measuring. + let _ = server_get(&endpoint, repository, &addresses).await?; + let _ = direct_presign_download(&endpoint, repository, &addresses).await?; + + for _ in 0..ROUNDS { + let server = server_get(&endpoint, repository, &addresses).await?; + let direct = direct_presign_download(&endpoint, repository, &addresses).await?; + round_stats.push(RoundStats { + bytes: server.0, + server_get_ms: server.1, + presign_ms: direct.1, + direct_http_ms: direct.2, + direct_total_ms: direct.3, + }); + } + + let avg = average(&round_stats); + let ratio = avg.direct_total_ms / avg.server_get_ms; + println!( + "| {} | {} | {} | {:.2} | {:.2} | {:.2} | {:.2} | {:.2}x |", + scenario.name, + scenario.count, + avg.bytes, + avg.server_get_ms, + avg.presign_ms, + avg.direct_http_ms, + avg.direct_total_ms, + ratio + ); + } + + Ok(()) +} + +fn average(stats: &[RoundStats]) -> RoundStats { + let count = stats.len() as f64; + RoundStats { + bytes: stats.first().map(|stats| stats.bytes).unwrap_or_default(), + server_get_ms: stats.iter().map(|stats| stats.server_get_ms).sum::() / count, + presign_ms: stats.iter().map(|stats| stats.presign_ms).sum::() / count, + direct_http_ms: stats.iter().map(|stats| stats.direct_http_ms).sum::() / count, + direct_total_ms: stats.iter().map(|stats| stats.direct_total_ms).sum::() / count, + } +} + +fn make_fragments(repository: Context, scenario: Scenario) -> Vec { + (0..scenario.count) + .map(|index| { + let payload = deterministic_payload(scenario.name, index, scenario.payload_size); + let hash = Hash::hash_buffer(&payload); + let mut context = repository; + context.data_mut()[0] ^= scenario.name.len() as u8; + context.data_mut()[1] ^= (index & 0xff) as u8; + context.data_mut()[2] ^= ((index >> 8) & 0xff) as u8; + BenchFragment { + address: Address { hash, context }, + payload: Bytes::from(payload), + } + }) + .collect() +} + +fn deterministic_payload(name: &str, index: usize, size: usize) -> Vec { + let mut state = 0xcbf29ce484222325u64 ^ index as u64; + for byte in name.as_bytes() { + state = state.wrapping_mul(0x100000001b3) ^ (*byte as u64); + } + + let mut payload = vec![0u8; size]; + for (offset, byte) in payload.iter_mut().enumerate() { + state ^= offset as u64; + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + *byte = (state >> 32) as u8; + } + payload +} + +async fn seed_fragments( + endpoint: &str, + repository: Context, + fragments: &[BenchFragment], +) -> Result<(), BenchError> { + let mut client = StorageServiceClient::connect(endpoint.to_string()).await?; + let requests = fragments + .iter() + .map(|fragment| storage_v1::PutRequest { + address: Some(to_proto_address(fragment.address)), + fragment: Some(model_v1::Fragment { + flags: 0, + size_payload: fragment.payload.len() as u32, + size_content: fragment.payload.len() as u64, + }), + payload: Some(fragment.payload.clone()), + }) + .collect::>(); + + let mut request = Request::new(stream::iter(requests)); + inject_repository(&mut request, repository); + let mut responses = client.put(request).await?.into_inner(); + let mut count = 0usize; + while let Some(response) = responses.next().await { + response?; + count += 1; + } + if count != fragments.len() { + return Err(format!( + "put response count mismatch: got {count}, expected {}", + fragments.len() + ) + .into()); + } + Ok(()) +} + +async fn server_get( + endpoint: &str, + repository: Context, + addresses: &[Address], +) -> Result<(usize, f64), BenchError> { + let mut client = StorageServiceClient::connect(endpoint.to_string()).await?; + let requests = addresses + .iter() + .copied() + .map(to_proto_address) + .collect::>(); + + let mut request = Request::new(stream::iter(requests)); + inject_repository(&mut request, repository); + + let started = Instant::now(); + let mut responses = client.get(request).await?.into_inner(); + let mut bytes = 0usize; + let mut count = 0usize; + while let Some(response) = responses.next().await { + let response = response?; + bytes += response.payload.len(); + count += 1; + } + if count != addresses.len() { + return Err(format!( + "get response count mismatch: got {count}, expected {}", + addresses.len() + ) + .into()); + } + + Ok((bytes, elapsed_ms(started))) +} + +async fn direct_presign_download( + endpoint: &str, + repository: Context, + addresses: &[Address], +) -> Result<(usize, f64, f64, f64), BenchError> { + let mut client = StorageServiceClient::connect(endpoint.to_string()).await?; + let mut request = Request::new(storage_v1::PresignDownloadRequest { + addresses: addresses.iter().copied().map(to_proto_address).collect(), + expires_in_seconds: PRESIGN_EXPIRES_SECONDS, + }); + inject_repository(&mut request, repository); + + let total_started = Instant::now(); + let presign_started = Instant::now(); + let response = client.presign_download(request).await?.into_inner(); + let presign_ms = elapsed_ms(presign_started); + if response.downloads.len() != addresses.len() { + return Err(format!( + "presign response count mismatch: got {}, expected {}", + response.downloads.len(), + addresses.len() + ) + .into()); + } + + let http = reqwest::Client::new(); + let limiter = Arc::new(Semaphore::new(DIRECT_DOWNLOAD_CONCURRENCY)); + let http_started = Instant::now(); + let mut tasks = Vec::with_capacity(response.downloads.len()); + for download in response.downloads { + let http = http.clone(); + let limiter = limiter.clone(); + tasks.push(tokio::spawn(async move { + let _permit = limiter.acquire_owned().await?; + let expected = download + .fragment + .ok_or("presign response missing fragment metadata")? + .size_payload as usize; + let bytes = http + .get(download.url) + .send() + .await? + .error_for_status()? + .bytes() + .await?; + if bytes.len() != expected { + return Err(format!( + "direct response size mismatch: got {}, expected {expected}", + bytes.len() + ) + .into()); + } + Ok::(bytes.len()) + })); + } + + let mut bytes = 0usize; + for task in tasks { + bytes += task.await??; + } + let http_ms = elapsed_ms(http_started); + let total_ms = elapsed_ms(total_started); + + Ok((bytes, presign_ms, http_ms, total_ms)) +} + +fn inject_repository(request: &mut Request, repository: Context) { + let value = MetadataValue::from_bytes(repository.data()); + request.metadata_mut().append_bin(REPOSITORY_ID_KEY, value); +} + +fn to_proto_address(address: Address) -> model_v1::Address { + model_v1::Address { + hash: Bytes::copy_from_slice(address.hash.data()), + context: Bytes::copy_from_slice(address.context.data()), + } +} + +fn elapsed_ms(started: Instant) -> f64 { + started.elapsed().as_secs_f64() * 1000.0 +} diff --git a/lore-transport/src/auth/exchange.rs b/lore-transport/src/auth/exchange.rs index 3729cd876..fa67f7c8b 100644 --- a/lore-transport/src/auth/exchange.rs +++ b/lore-transport/src/auth/exchange.rs @@ -65,6 +65,21 @@ fn cache() -> &'static AuthzCache { AUTHZ_CACHE.get_or_init(|| Mutex::new(HashMap::new())) } +/// Invalidates cached authorization tokens for one imported identity. +/// +/// Direct token login replaces the durable token-store entries, but live +/// transports may otherwise keep returning a still-unexpired token from this +/// process cache. The next exchange reloads the replacement token from the +/// store. +pub async fn invalidate_identity(auth_url: &str, identity: &str) { + cache() + .lock() + .await + .retain(|(cached_auth_url, cached_identity, _, _, _), _| { + cached_auth_url != auth_url || cached_identity != identity + }); +} + pub fn is_expired(expires: u64) -> bool { let expires = expires as u128; let current_time = SystemTime::now() diff --git a/lore-transport/src/auth/mod.rs b/lore-transport/src/auth/mod.rs index 292c8fd69..d599dbee1 100644 --- a/lore-transport/src/auth/mod.rs +++ b/lore-transport/src/auth/mod.rs @@ -45,7 +45,8 @@ pub mod authentication { REGISTER_BUILTIN_AUTHENTICATION.call_once(|| { let ucs_auth = Arc::new(ucs_auth::UcsAuthentication); let _ = add("ucs-auth", ucs_auth.clone()); - let _ = add("https", ucs_auth); // transition fallback + let _ = add("https", ucs_auth.clone()); // transition fallback + let _ = add("http", ucs_auth); // local development fallback }); let scheme = parse_scheme(auth_url)?; diff --git a/lore-transport/src/auth/ucs_auth.rs b/lore-transport/src/auth/ucs_auth.rs index 9412ab4b9..ad41cca8a 100644 --- a/lore-transport/src/auth/ucs_auth.rs +++ b/lore-transport/src/auth/ucs_auth.rs @@ -16,14 +16,15 @@ use crate::grpc::CorrelationInterceptor; use crate::traits::Authentication; use crate::types::*; -/// Strips the custom scheme from an auth URL and returns an HTTPS URL +/// Strips the custom scheme from an auth URL and returns a tonic endpoint /// suitable for gRPC connection. /// /// `ucs-auth://auth.example.com` -> `https://auth.example.com` /// `https://auth.example.com` -> `https://auth.example.com` (unchanged) +/// `http://127.0.0.1:8787` -> `http://127.0.0.1:8787` (local dev) fn grpc_endpoint(auth_url: &str) -> String { match auth_url.split_once("://") { - Some(("https", _)) => auth_url.to_string(), + Some(("https" | "http", _)) => auth_url.to_string(), Some((_, rest)) => format!("https://{rest}"), None => format!("https://{auth_url}"), } @@ -311,6 +312,14 @@ mod tests { ); } + #[test] + fn grpc_endpoint_http() { + assert_eq!( + grpc_endpoint("http://127.0.0.1:8787"), + "http://127.0.0.1:8787" + ); + } + #[test] fn grpc_endpoint_no_scheme() { assert_eq!( diff --git a/lore-transport/src/connection.rs b/lore-transport/src/connection.rs index aa162eade..9a962289f 100644 --- a/lore-transport/src/connection.rs +++ b/lore-transport/src/connection.rs @@ -44,6 +44,8 @@ static REGISTER_BUILTIN_PROTOCOLS: Once = Once::new(); pub fn find(scheme: &str) -> Result, ProtocolError> { REGISTER_BUILTIN_PROTOCOLS.call_once(|| { + let _ = add("spacesync", Arc::new(LoreProtocol::default())); + let _ = add("spacesyncs", Arc::new(LoreProtocol::default())); let _ = add("lore", Arc::new(LoreProtocol::default())); let _ = add("lores", Arc::new(LoreProtocol::default())); // Legacy protocol schemes for backwards compatibility diff --git a/lore-transport/src/direct_download.rs b/lore-transport/src/direct_download.rs new file mode 100644 index 000000000..6c54d9e4a --- /dev/null +++ b/lore-transport/src/direct_download.rs @@ -0,0 +1,320 @@ +// SPDX-FileCopyrightText: 2026 Epic Games, Inc. +// SPDX-License-Identifier: MIT +use std::collections::HashMap; +use std::future::Future; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use bytes::Bytes; +use lore_base::error::NotFound; +use lore_base::error::NotSupported; +use lore_base::error::SlowDown; +use lore_base::lore_debug; +use lore_base::lore_warn; +use lore_base::types::Address; +use lore_base::types::DirectDownload; +use lore_base::types::Fragment; +use reqwest::StatusCode; +use tokio::sync::Mutex; +use tokio::sync::Semaphore; +use tokio::sync::oneshot; + +use crate::error::ProtocolError; + +const DEFAULT_MAX_BATCH_SIZE: usize = 512; +const DEFAULT_COALESCE_DELAY: Duration = Duration::from_millis(2); +const DEFAULT_EXPIRES_IN: Duration = Duration::from_secs(300); +const DEFAULT_MAX_HTTP_DOWNLOADS: usize = 128; + +type PendingSender = oneshot::Sender>; + +struct PendingRequest { + address: Address, + sender: PendingSender, +} + +#[derive(Default)] +struct BatchState { + pending: Vec, + flushing: bool, +} + +pub(crate) struct DirectDownloadBatcher { + state: Mutex, + http: reqwest::Client, + max_batch_size: usize, + coalesce_delay: Duration, + expires_in: Duration, + http_downloads: Semaphore, + disabled: AtomicBool, +} + +impl DirectDownloadBatcher { + pub(crate) fn new() -> Self { + Self { + state: Mutex::new(BatchState::default()), + http: reqwest::Client::new(), + max_batch_size: DEFAULT_MAX_BATCH_SIZE, + coalesce_delay: DEFAULT_COALESCE_DELAY, + expires_in: DEFAULT_EXPIRES_IN, + http_downloads: Semaphore::new(DEFAULT_MAX_HTTP_DOWNLOADS), + disabled: AtomicBool::new(false), + } + } + + pub(crate) async fn presign( + &self, + address: Address, + fetch_batch: F, + ) -> Result + where + F: Fn(Vec
, Duration) -> Fut, + Fut: Future, ProtocolError>>, + { + if self.disabled.load(Ordering::Relaxed) { + return Err(ProtocolError::from(NotSupported { + operation: "direct download".to_string(), + })); + } + + let (sender, receiver) = oneshot::channel(); + let should_flush = { + let mut state = self.state.lock().await; + state.pending.push(PendingRequest { address, sender }); + if state.flushing { + false + } else { + state.flushing = true; + true + } + }; + + if should_flush { + self.flush(fetch_batch).await; + } + + receiver + .await + .map_err(|err| ProtocolError::internal_with_context(err, "direct download batch"))? + } + + async fn flush(&self, fetch_batch: F) + where + F: Fn(Vec
, Duration) -> Fut, + Fut: Future, ProtocolError>>, + { + loop { + tokio::time::sleep(self.coalesce_delay).await; + + let batch = { + let mut state = self.state.lock().await; + if state.pending.is_empty() { + state.flushing = false; + return; + } + let take = state.pending.len().min(self.max_batch_size); + state.pending.drain(..take).collect::>() + }; + + let addresses = batch.iter().map(|request| request.address).collect(); + let result = fetch_batch(addresses, self.expires_in).await; + + match result { + Ok(downloads) => { + let mut by_address: HashMap = downloads + .into_iter() + .map(|download| (download.address, download)) + .collect(); + + for request in batch { + let result = by_address + .remove(&request.address) + .ok_or_else(|| ProtocolError::from(NotFound)); + let _ = request.sender.send(result); + } + } + Err(err) => { + if matches!(err, ProtocolError::NotSupported(_)) { + self.disabled.store(true, Ordering::Relaxed); + } + for request in batch { + let _ = request.sender.send(Err(err.clone())); + } + } + } + } + } + + pub(crate) async fn download( + &self, + download: DirectDownload, + ) -> Result<(Fragment, Bytes), ProtocolError> { + let _permit = + self.http_downloads.acquire().await.map_err(|err| { + ProtocolError::internal_with_context(err, "direct download permit") + })?; + + if let Err(reason) = lore_base::types::validate_fragment_response(&download.fragment) { + return Err(ProtocolError::internal(format!( + "direct download: invalid fragment {:?}: {reason}", + download.fragment + ))); + } + + let response = + self.http.get(&download.url).send().await.map_err(|err| { + ProtocolError::internal_with_context(err, "direct download http get") + })?; + + let status = response.status(); + if !status.is_success() { + lore_warn!( + "Direct download failed for {} with HTTP status {}", + download.address, + status + ); + return Err(match status { + StatusCode::NOT_FOUND | StatusCode::FORBIDDEN => ProtocolError::from(NotFound), + StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE => { + ProtocolError::from(SlowDown) + } + _ => ProtocolError::internal(format!( + "direct download HTTP status {status} for {}", + download.address + )), + }); + } + + let payload = response.bytes().await.map_err(|err| { + ProtocolError::internal_with_context(err, "direct download response body") + })?; + + if payload.len() != download.fragment.size_payload as usize { + return Err(ProtocolError::internal(format!( + "direct download payload length mismatch for {}: got {}, expected {}", + download.address, + payload.len(), + download.fragment.size_payload + ))); + } + + lore_debug!( + "Direct downloaded {} bytes for {}", + payload.len(), + download.address + ); + Ok((download.fragment, payload)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + use lore_base::types::Context; + use lore_base::types::FragmentFlags; + use lore_base::types::Hash; + + use super::*; + + fn test_batcher(coalesce_delay: Duration) -> DirectDownloadBatcher { + DirectDownloadBatcher { + state: Mutex::new(BatchState::default()), + http: reqwest::Client::new(), + max_batch_size: DEFAULT_MAX_BATCH_SIZE, + coalesce_delay, + expires_in: DEFAULT_EXPIRES_IN, + http_downloads: Semaphore::new(DEFAULT_MAX_HTTP_DOWNLOADS), + disabled: AtomicBool::new(false), + } + } + + fn address(value: u64) -> Address { + Address { + hash: Hash::from_u64(value), + context: Context::from([value as u8; 16]), + } + } + + fn download(address: Address) -> DirectDownload { + DirectDownload { + address, + fragment: Fragment { + flags: FragmentFlags::PayloadStoredDurable.bits(), + size_payload: 1, + size_content: 1, + }, + url: format!("http://127.0.0.1/{address}"), + expires_at_epoch_seconds: 1, + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn presign_coalesces_concurrent_requests() { + let batcher = Arc::new(test_batcher(Duration::from_millis(20))); + let calls = Arc::new(Mutex::new(Vec::>::new())); + + let mut tasks = Vec::new(); + for address in (0..8).map(address) { + let batcher = batcher.clone(); + let calls = calls.clone(); + tasks.push(tokio::spawn(async move { + batcher + .presign(address, move |addresses, _expires_in| { + let calls = calls.clone(); + async move { + calls.lock().await.push(addresses.clone()); + Ok(addresses.into_iter().map(download).collect()) + } + }) + .await + })); + } + + let mut results = Vec::new(); + for task in tasks { + results.push(task.await.expect("task failed").expect("presign failed")); + } + + assert_eq!(results.len(), 8); + let calls = calls.lock().await; + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].len(), 8); + } + + #[tokio::test] + async fn not_supported_disables_future_presign_attempts() { + let batcher = test_batcher(Duration::from_millis(1)); + let calls = Arc::new(AtomicUsize::new(0)); + + let first = batcher + .presign(address(1), { + let calls = calls.clone(); + move |_addresses, _expires_in| { + let calls = calls.clone(); + async move { + calls.fetch_add(1, Ordering::Relaxed); + Err(ProtocolError::from(NotSupported { + operation: "direct download".to_string(), + })) + } + } + }) + .await; + assert!(first.expect_err("expected NotSupported").is_not_supported()); + + let second = batcher + .presign(address(2), |_addresses, _expires_in| async { + panic!("presign should be disabled after NotSupported") + }) + .await; + assert!( + second + .expect_err("expected disabled NotSupported") + .is_not_supported() + ); + assert_eq!(calls.load(Ordering::Relaxed), 1); + } +} diff --git a/lore-transport/src/grpc/mod.rs b/lore-transport/src/grpc/mod.rs index a928ab538..c099bdb6d 100644 --- a/lore-transport/src/grpc/mod.rs +++ b/lore-transport/src/grpc/mod.rs @@ -488,8 +488,8 @@ impl GRPCConnection { credentials.from_supplied_credentials(), ); - if let Some(auth) = self.auth.get(&key) { - return auth.clone(); + if let Some(auth_entry) = self.auth.get(&key) { + return auth_entry.clone(); } let auth = GRPCAuth::new( @@ -523,8 +523,8 @@ impl GRPCConnection { credentials.from_supplied_credentials(), ); - if let Some(auth) = self.auth.get(&key) { - return auth.clone(); + if let Some(auth_entry) = self.auth.get(&key) { + return auth_entry.clone(); } let auth = GRPCAuth::new_for_custom_resource( diff --git a/lore-transport/src/grpc/repository_client.rs b/lore-transport/src/grpc/repository_client.rs index e7ca7df6b..474dff075 100644 --- a/lore-transport/src/grpc/repository_client.rs +++ b/lore-transport/src/grpc/repository_client.rs @@ -70,6 +70,7 @@ impl RepositoryService { Ok(RepositoryData { id: repository.id.into(), name: repository.name, + default_branch_name: repository.default_branch_name, metadata: repository.metadata.into(), }) } @@ -132,6 +133,7 @@ impl RepositoryService { Ok(RepositoryData { id: repository.id.into(), name: repository.name, + default_branch_name: repository.default_branch_name, metadata: repository.metadata.into(), }) } else { @@ -158,6 +160,7 @@ impl RepositoryService { entries.push(RepositoryData { id: repository.id.into(), name: repository.name, + default_branch_name: repository.default_branch_name, metadata: repository.metadata.into(), }); } diff --git a/lore-transport/src/grpc/storage_client.rs b/lore-transport/src/grpc/storage_client.rs index b66a36588..747bff424 100644 --- a/lore-transport/src/grpc/storage_client.rs +++ b/lore-transport/src/grpc/storage_client.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; +use std::time::Duration; use bytes::BufMut; use bytes::Bytes; @@ -16,8 +17,10 @@ use lore_base::error::SlowDown; use lore_base::lore_debug; use lore_base::lore_error; use lore_base::lore_spawn_net; +use lore_base::lore_warn; use lore_base::types::Address; use lore_base::types::Context; +use lore_base::types::DirectDownload; use lore_base::types::Fragment; use lore_base::types::Hash; use lore_base::types::HealResult; @@ -37,6 +40,7 @@ use tonic::metadata::MetadataValue; use super::CORRELATION_ID_HEADER; use super::PARTITION_ID_KEY; use super::REPOSITORY_ID_KEY; +use crate::direct_download::DirectDownloadBatcher; use crate::error::ProtocolError; /// Translate a response's in-band `status` into a [`ProtocolError`], or `None` when the item @@ -426,6 +430,7 @@ pub struct StorageService { put_resolved_streams: StreamCache>, get_put_limiter: Semaphore, + direct_downloads: DirectDownloadBatcher, } fn inject_metadata(request: &mut tonic::Request, ctx: &GrpcSessionContext) { @@ -462,6 +467,7 @@ impl StorageService { get_resolved_streams: StreamCache::new(), put_resolved_streams: StreamCache::new(), get_put_limiter: Semaphore::new(INFLIGHT_COMMAND_LIMIT), + direct_downloads: DirectDownloadBatcher::new(), } } @@ -485,6 +491,31 @@ impl StorageService { ) -> Result<(Fragment, Bytes), ProtocolError> { lore_debug!("gRPC get fragment: {}", address); + match self + .direct_downloads + .presign(*address, |addresses, expires_in| async move { + self.presign_downloads(ctx, &addresses, expires_in).await + }) + .await + { + Ok(download) => match self.direct_downloads.download(download).await { + Ok(result) => return Ok(result), + Err(err) => { + lore_warn!( + "Direct gRPC download failed for {}, falling back to server get: {err}", + address + ); + } + }, + Err(ProtocolError::NotSupported(_)) => {} + Err(err) => { + lore_debug!( + "Direct gRPC presign failed for {}, falling back to server get: {err}", + address + ); + } + } + let _permit = self .get_put_limiter .acquire() @@ -522,6 +553,64 @@ impl StorageService { Ok((fragment, payload)) } + pub async fn presign_downloads( + &self, + ctx: &GrpcSessionContext, + addresses: &[Address], + expires_in: Duration, + ) -> Result, ProtocolError> { + if addresses.is_empty() { + return Ok(Vec::new()); + } + + lore_debug!("gRPC presign_download {} fragments", addresses.len()); + + let request = storage_v1::PresignDownloadRequest { + addresses: addresses.iter().map(model_v1::Address::from).collect(), + expires_in_seconds: expires_in.as_secs(), + }; + let mut client = StorageServiceClient::new(self.connection.channel()); + let mut req = tonic::Request::new(request); + inject_metadata(&mut req, ctx); + + let response = client + .presign_download(req) + .await + .map(|res| res.into_inner()) + .map_err(ProtocolError::from)?; + + response + .downloads + .into_iter() + .map(|download| { + let address = download + .address + .as_ref() + .ok_or_else(|| ProtocolError::internal("presign_download: missing address"))?; + let fragment = download + .fragment + .as_ref() + .ok_or_else(|| ProtocolError::internal("presign_download: missing fragment"))?; + let fragment = Fragment { + flags: fragment.flags, + size_payload: fragment.size_payload, + size_content: fragment.size_content, + }; + if let Err(reason) = lore_base::types::validate_fragment_response(&fragment) { + return Err(ProtocolError::internal(format!( + "presign_download: invalid fragment {fragment:?}: {reason}" + ))); + } + Ok(DirectDownload { + address: Address::from(address), + fragment, + url: download.url, + expires_at_epoch_seconds: download.expires_at_epoch_seconds, + }) + }) + .collect() + } + /// Fetch only the fragment metadata for an address. Same wire request as `get` (just an /// `Address`), but the server's response carries `Fragment` only — no payload bytes — so /// callers that don't need the payload skip the transfer cost. Used by the storage API's @@ -1420,6 +1509,20 @@ mod tests { Ok(Response::new(Box::pin(stream) as Self::PutStream)) } + async fn upload_content( + &self, + _request: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn presign_download( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + type CopyStream = ResponseStream; async fn copy( diff --git a/lore-transport/src/lib.rs b/lore-transport/src/lib.rs index bcf598202..b68e4977b 100644 --- a/lore-transport/src/lib.rs +++ b/lore-transport/src/lib.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT pub mod auth; pub mod connection; +pub(crate) mod direct_download; pub mod error; pub mod grpc; pub mod quic; diff --git a/lore-transport/src/quic/mod.rs b/lore-transport/src/quic/mod.rs index 670aac7c5..e43318c56 100644 --- a/lore-transport/src/quic/mod.rs +++ b/lore-transport/src/quic/mod.rs @@ -43,6 +43,7 @@ pub enum QuicServiceError { Failed = 3, NotFound = 4, Oversized = 5, + NotSupported = 6, SlowDown = 100, // service specific implementations can use 200-299 ImplementationReserved = RESERVED_ERROR_CODE_START, @@ -83,6 +84,8 @@ pub enum QuicClientError { NotFound, #[error("Oversized fragment rejected by server")] Oversized, + #[error("Operation not supported")] + NotSupported, } #[derive(Debug)] diff --git a/lore-transport/src/quic/response_reader.rs b/lore-transport/src/quic/response_reader.rs index cd84abfb2..79ea12353 100644 --- a/lore-transport/src/quic/response_reader.rs +++ b/lore-transport/src/quic/response_reader.rs @@ -321,6 +321,7 @@ fn handle_error(status: QuicErrorStatus) -> QuicClientError { x if x == QuicServiceError::NotAuthorized as u32 => QuicClientError::NotAuthorized, x if x == QuicServiceError::NotFound as u32 => QuicClientError::NotFound, x if x == QuicServiceError::Oversized as u32 => QuicClientError::Oversized, + x if x == QuicServiceError::NotSupported as u32 => QuicClientError::NotSupported, _ => QuicClientError::ServerError(status), } } diff --git a/lore-transport/src/quic/storage_service/client.rs b/lore-transport/src/quic/storage_service/client.rs index b8179740e..bdbfd9439 100644 --- a/lore-transport/src/quic/storage_service/client.rs +++ b/lore-transport/src/quic/storage_service/client.rs @@ -13,10 +13,12 @@ use bytes::BytesMut; use lore_base::error::Disconnected; use lore_base::error::NotAuthorized; use lore_base::error::NotFound; +use lore_base::error::NotSupported; use lore_base::error::Oversized; use lore_base::error::SlowDown; use lore_base::lore_debug; use lore_base::lore_trace; +use lore_base::lore_warn; use lore_base::types::Address; use lore_base::types::Context; use lore_base::types::Fragment; @@ -49,6 +51,7 @@ use super::super::storage_service::MAX_CHUNK_SIZE; use super::super::storage_service::auth::StorageClientAuth; use crate::connection::Connection; use crate::connection::SuppliedCredentials; +use crate::direct_download::DirectDownloadBatcher; use crate::error::ProtocolError; use crate::quic::client::CongestionAlgorithm; use crate::traits::Storage; @@ -72,6 +75,7 @@ pub struct StorageClient { quic: Arc, connection_establish: Semaphore, command_limit: Semaphore, + direct_downloads: DirectDownloadBatcher, sent: AtomicUsize, } @@ -117,6 +121,7 @@ impl StorageClient { counter: AtomicUsize::new(0), sent: AtomicUsize::new(0), command_limit: Semaphore::new(INFLIGHT_COMMAND_LIMIT), + direct_downloads: DirectDownloadBatcher::new(), } } @@ -249,6 +254,9 @@ impl ServiceClient for StorageClient { storage_service::command_name(&failed_request) ), }), + QuicClientError::NotSupported => ProtocolError::from(NotSupported { + operation: storage_service::command_name(&failed_request).to_string(), + }), _ => { let name = storage_service::command_name(&failed_request); ProtocolError::internal(format!( @@ -349,6 +357,32 @@ impl Storage for StorageClient { session_id: u32, address: &Address, ) -> Result<(Fragment, Bytes), ProtocolError> { + match self + .direct_downloads + .presign(*address, |addresses, expires_in| async move { + self.presign_downloads(session_id, &addresses, expires_in) + .await + }) + .await + { + Ok(download) => match self.direct_downloads.download(download).await { + Ok(result) => return Ok(result), + Err(err) => { + lore_warn!( + "Direct QUIC download failed for {}, falling back to server get: {err}", + address + ); + } + }, + Err(ProtocolError::NotSupported(_)) => {} + Err(err) => { + lore_debug!( + "Direct QUIC presign failed for {}, falling back to server get: {err}", + address + ); + } + } + let mut payload = send_normal_with_reconnect(self, Command::Get, session_id, || { [Bytes::default(), Bytes::from_owner(*address)] }) @@ -453,6 +487,32 @@ impl Storage for StorageClient { session_id: u32, address: &Address, ) -> Result<(Fragment, Bytes), ProtocolError> { + match self + .direct_downloads + .presign(*address, |addresses, expires_in| async move { + self.presign_downloads(session_id, &addresses, expires_in) + .await + }) + .await + { + Ok(download) => match self.direct_downloads.download(download).await { + Ok(result) => return Ok(result), + Err(err) => { + lore_warn!( + "Direct QUIC priority download failed for {}, falling back to server get: {err}", + address + ); + } + }, + Err(ProtocolError::NotSupported(_)) => {} + Err(err) => { + lore_debug!( + "Direct QUIC priority presign failed for {}, falling back to server get: {err}", + address + ); + } + } + let mut payload = send_high_priority_with_reconnect(self, Command::Get, session_id, || { [Bytes::default(), Bytes::from_owner(*address)] }) diff --git a/lore-transport/src/quic/storage_service/mod.rs b/lore-transport/src/quic/storage_service/mod.rs index 4625ecdb2..250579d46 100644 --- a/lore-transport/src/quic/storage_service/mod.rs +++ b/lore-transport/src/quic/storage_service/mod.rs @@ -56,6 +56,8 @@ pub enum Command { /// A zero `Address` hash removes the mapping instead of publishing one; the `Fragment` and /// payload are then ignored, since there is nothing to store. PutResolved = 13, + /// Batch request for short-lived direct-download URLs for immutable payloads. + PresignDownload = 14, } /// `flags` field of a [`Command::GetResolved`] request. Reserved; no bits are defined. @@ -91,6 +93,7 @@ impl TryFrom for Command { v if v == Command::GetMetadata as u8 => Ok(Command::GetMetadata), v if v == Command::GetResolved as u8 => Ok(Command::GetResolved), v if v == Command::PutResolved as u8 => Ok(Command::PutResolved), + v if v == Command::PresignDownload as u8 => Ok(Command::PresignDownload), _ => Err(UnknownCommand(value)), } } @@ -116,6 +119,7 @@ pub fn command_name(command: &Command) -> &'static str { Command::GetMetadata => "get_metadata", Command::GetResolved => "get_resolved", Command::PutResolved => "put_resolved", + Command::PresignDownload => "presign_download", } } diff --git a/lore-transport/src/traits.rs b/lore-transport/src/traits.rs index 37cbd5043..f24311fdd 100644 --- a/lore-transport/src/traits.rs +++ b/lore-transport/src/traits.rs @@ -2,9 +2,11 @@ // SPDX-License-Identifier: MIT use std::sync::Arc; use std::sync::Weak; +use std::time::Duration; use async_trait::async_trait; use bytes::Bytes; +use lore_base::error::NotSupported; use lore_base::types::*; use crate::connection::Connection; @@ -130,6 +132,19 @@ pub trait Storage: Send + Sync { Ok(fragment) } + /// Return short-lived direct-download URLs for a batch of immutable payloads. + async fn presign_downloads( + &self, + session_id: u32, + addresses: &[Address], + expires_in: Duration, + ) -> Result, ProtocolError> { + let _ = (session_id, addresses, expires_in); + Err(ProtocolError::from(NotSupported { + operation: "direct download".to_string(), + })) + } + /// Put the immutable fragment and optional payload for address async fn put( &self, diff --git a/lore-transport/src/types.rs b/lore-transport/src/types.rs index ca3a86d81..7f867a8aa 100644 --- a/lore-transport/src/types.rs +++ b/lore-transport/src/types.rs @@ -179,6 +179,7 @@ impl From for RevisionListStart { pub struct RepositoryData { pub id: RepositoryId, pub name: String, + pub default_branch_name: String, pub metadata: Hash, } diff --git a/lore/cbindgen.toml b/lore/cbindgen.toml index 690f185fc..754588bbf 100644 --- a/lore/cbindgen.toml +++ b/lore/cbindgen.toml @@ -171,6 +171,9 @@ include = [ "LoreBranchMergeStartArgs" = "lore_branch_merge_start_args_t" "LoreBranchMergeStartBeginEventData" = "lore_branch_merge_start_begin_event_data_t" "LoreBranchMergeStartEndEventData" = "lore_branch_merge_start_end_event_data_t" +"LorePathMergeRule" = "lore_path_merge_rule_t" +"LorePathMergeStrategy" = "lore_path_merge_strategy_t" +"LoreArray_lore_path_merge_rule" = "lore_path_merge_rule_array_t" "LoreCherryPickStartBeginEventData" = "lore_cherry_pick_start_begin_event_data_t" "LoreCherryPickStartEndEventData" = "lore_cherry_pick_start_end_event_data_t" "LoreCherryPickAbortBeginEventData" = "lore_cherry_pick_abort_begin_event_data_t" diff --git a/lore/src/branch.rs b/lore/src/branch.rs index 746b02e66..7b4700890 100644 --- a/lore/src/branch.rs +++ b/lore/src/branch.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use lore_error_set::prelude::*; use lore_macro::LoreArgs; +use lore_macro::ValidateText; use lore_revision::branch; use lore_revision::branch::BranchError; use lore_revision::branch::latest::ListOptions; @@ -11,6 +12,8 @@ use lore_revision::branch::merge::MergeError; use lore_revision::branch::merge::MergeIntoOptions; use lore_revision::branch::merge::MergeScope; use lore_revision::branch::merge::MergeStartOptions; +use lore_revision::branch::merge::PathMergeRule; +use lore_revision::branch::merge::PathMergeStrategy; use lore_revision::branch::push::PushOptions; use lore_revision::branch::reset::ResetError; use lore_revision::interface::LoreArray; @@ -28,6 +31,7 @@ use lore_revision::repository; use lore_revision::repository::BranchSwitchOptions; use lore_revision::repository::RepositoryContext; use lore_revision::repository::RepositoryWriteToken; +use lore_revision::util::path::RelativePath; use serde::Deserialize; use serde::Serialize; @@ -299,6 +303,42 @@ async fn list_local( .await } +/// cbindgen:prefix-with-name +/// cbindgen:rename-all=ScreamingSnakeCase +#[repr(C)] +/// Strategy used for a path during branch merge. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, ValidateText)] +#[serde(rename_all = "camelCase")] +pub enum LorePathMergeStrategy { + /// Merge the path normally. + #[default] + Merge = 0, + /// Keep the current target branch version for matching source changes. + KeepTarget = 1, + /// Exclude matching source changes from the merge. + Exclude = 2, +} + +impl From for PathMergeStrategy { + fn from(value: LorePathMergeStrategy) -> Self { + match value { + LorePathMergeStrategy::Merge => Self::Merge, + LorePathMergeStrategy::KeepTarget => Self::KeepTarget, + LorePathMergeStrategy::Exclude => Self::Exclude, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ValidateText)] +/// A repository-relative path merge strategy rule. +pub struct LorePathMergeRule { + /// Repository-relative path. Directories match descendants. + pub path: LoreString, + /// Strategy to apply when this rule is the most-specific match. + pub strategy: LorePathMergeStrategy, +} + #[repr(C)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, LoreArgs)] #[handler(merge_start_local)] @@ -314,6 +354,9 @@ pub struct LoreBranchMergeStartArgs { pub link: LoreString, /// Merge only the main repository, skipping all linked repositories pub ignore_links: u8, + /// Ordered per-path merge strategy rules. + #[serde(default)] + pub path_merge_rules: LoreArray, } /// Begins merging a source branch into the current branch, auto-committing if there are no conflicts. @@ -355,6 +398,24 @@ pub async fn merge_start( dispatch_call(globals, args, callback, merge_start_local).await } +fn convert_path_merge_rules( + repository: &RepositoryContext, + rules: &LoreArray, +) -> Result, MergeError> { + let repository_path = repository + .require_path() + .map_err(|err| MergeError::internal_with_context(err, "resolving repository path"))?; + let mut converted = Vec::with_capacity(rules.len()); + for rule in rules.as_slice() { + converted.push(PathMergeRule { + path: RelativePath::new_from_user_path(repository_path, rule.path.as_str()) + .forward::("invalid path merge rule")?, + strategy: rule.strategy.into(), + }); + } + Ok(converted) +} + async fn merge_start_local( globals: LoreGlobalArgs, args: LoreBranchMergeStartArgs, @@ -375,13 +436,16 @@ async fn merge_start_local( MergeScope::Link(link_str) }; - let options = MergeStartOptions { - message: args.message.to_string(), - no_commit: args.no_commit != 0, - scope, - }; - async move { + let path_merge_rules = + convert_path_merge_rules(&repository, &args.path_merge_rules)?; + let options = MergeStartOptions { + message: args.message.to_string(), + no_commit: args.no_commit != 0, + scope, + path_merge_rules, + }; + let branch = branch::resolve(repository.clone(), args.branch.as_str()) .await .forward::("resolving branch")?; diff --git a/lore/src/interface.rs b/lore/src/interface.rs index 877fd470d..43e7b75ea 100644 --- a/lore/src/interface.rs +++ b/lore/src/interface.rs @@ -1376,6 +1376,8 @@ pub extern "C" fn lore_branch_merge_restart_async( } pub type LoreBranchMergeStartArgs = crate::branch::LoreBranchMergeStartArgs; +pub type LorePathMergeRule = crate::branch::LorePathMergeRule; +pub type LorePathMergeStrategy = crate::branch::LorePathMergeStrategy; /// Start a merge from another branch into the current branch. /// diff --git a/lore/src/repository.rs b/lore/src/repository.rs index a4dfb43cb..206b226e4 100644 --- a/lore/src/repository.rs +++ b/lore/src/repository.rs @@ -386,6 +386,8 @@ pub struct LoreRepositoryCreateArgs { pub description: LoreString, /// Optional repository ID, set to empty string to generate a new ID pub id: LoreString, + /// Optional default branch name, set to empty string to use the Lore default + pub default_branch_name: LoreString, /// Whether to use the shared store instead of a local immutable store. Zero-initialized /// (`LORE_SHARED_STORE_MODE_INHERIT`) follows the machine's global setting. pub use_shared_store: LoreSharedStoreMode, @@ -463,6 +465,11 @@ async fn create_impl(args: &LoreRepositoryCreateArgs) -> Result<(), CreateError> } else { None }, + default_branch_name: if !args.default_branch_name.is_empty() { + Some(args.default_branch_name.to_string()) + } else { + None + }, shared_store_options: SharedStoreToUseConfig::from_cli_args( &global_config, args.use_shared_store, @@ -526,6 +533,11 @@ async fn create_with_metadata_impl( } else { None }, + default_branch_name: if !args.default_branch_name.is_empty() { + Some(args.default_branch_name.to_string()) + } else { + None + }, shared_store_options: SharedStoreToUseConfig::from_cli_args( &global_config, args.use_shared_store, diff --git a/lore/src/revision_tree/info.rs b/lore/src/revision_tree/info.rs index 71c651891..698ba8c17 100644 --- a/lore/src/revision_tree/info.rs +++ b/lore/src/revision_tree/info.rs @@ -100,7 +100,6 @@ async fn info_impl( }, async move |internal, args: LoreRevisionTreeInfoArgs| { let id = args.id; - let access = internal.access_shared().await; let state = access.state(); diff --git a/lore/src/revision_tree/list_children.rs b/lore/src/revision_tree/list_children.rs index 830532ea3..7b625209f 100644 --- a/lore/src/revision_tree/list_children.rs +++ b/lore/src/revision_tree/list_children.rs @@ -125,6 +125,11 @@ async fn resolve_listing_target( let Ok(node) = state.node(repository.clone(), node_id).await else { return Ok(None); }; + // A discarded slot reads back as a directory shape (no file or link + // bits); the node itself is gone (e.g. deleted through this handle). + if node.is_discarded() { + return Ok(None); + } if node.is_directory() { return Ok(Some((state, repository, node_id))); } diff --git a/lore/src/revision_tree/node_info.rs b/lore/src/revision_tree/node_info.rs index 580e12c00..889bf680a 100644 --- a/lore/src/revision_tree/node_info.rs +++ b/lore/src/revision_tree/node_info.rs @@ -139,6 +139,13 @@ async fn node_info_impl( return Err(invalid("node id is unknown")); }; + // A discarded slot keeps its name for history weaving; the node + // itself is gone (e.g. deleted through this handle). + if node.is_discarded() { + emit_node_info_error(id, LoreErrorCode::InvalidArguments); + return Err(invalid("node id resolves to a deleted node")); + } + let name = if node_id == ROOT_NODE { String::new() } else { diff --git a/lore/src/storage/mod.rs b/lore/src/storage/mod.rs index 37e6a2d47..af9f5edb9 100644 --- a/lore/src/storage/mod.rs +++ b/lore/src/storage/mod.rs @@ -276,7 +276,10 @@ pub(crate) fn aggregate_error_code( ) -> Option { fn severity(code: LoreErrorCode) -> u8 { match code { - LoreErrorCode::InvalidArguments => 4, + // Only revision-tree commits report BranchAdvanced; it never + // appears on a storage item, but rank it like a caller-actionable + // failure for completeness. + LoreErrorCode::InvalidArguments | LoreErrorCode::BranchAdvanced => 4, LoreErrorCode::Internal => 3, LoreErrorCode::SlowDown => 2, LoreErrorCode::AddressNotFound => 1, diff --git a/lore/tests/events.rs b/lore/tests/events.rs index b180ffd00..5a294c2e8 100644 --- a/lore/tests/events.rs +++ b/lore/tests/events.rs @@ -71,6 +71,7 @@ mod tests { repository_url: name.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }; diff --git a/lore/tests/ffi_error_events.rs b/lore/tests/ffi_error_events.rs index 07b93b304..4e48cd0fb 100644 --- a/lore/tests/ffi_error_events.rs +++ b/lore/tests/ffi_error_events.rs @@ -178,6 +178,7 @@ mod tests { repository_url: name.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), } diff --git a/lore/tests/file_write.rs b/lore/tests/file_write.rs index bc99d5746..b0abc3adf 100644 --- a/lore/tests/file_write.rs +++ b/lore/tests/file_write.rs @@ -36,6 +36,7 @@ mod tests { repository_url: url.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }; diff --git a/lore/tests/in_memory.rs b/lore/tests/in_memory.rs index cc5594bfa..d857de543 100644 --- a/lore/tests/in_memory.rs +++ b/lore/tests/in_memory.rs @@ -52,6 +52,7 @@ mod tests { repository_url: url.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }; @@ -148,6 +149,7 @@ mod tests { repository_url: url.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }; @@ -237,6 +239,7 @@ mod tests { repository_url: url.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }; diff --git a/lore/tests/metadata.rs b/lore/tests/metadata.rs index 6899995bf..8a644d272 100644 --- a/lore/tests/metadata.rs +++ b/lore/tests/metadata.rs @@ -55,6 +55,7 @@ mod tests { repository_url: url.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }; @@ -192,6 +193,7 @@ mod tests { repository_url: url.into(), id: LoreString::default(), description: LoreString::default(), + default_branch_name: LoreString::default(), use_shared_store: LoreSharedStoreMode::Disabled, shared_store_path: LoreString::default(), }; diff --git a/pyproject.toml b/pyproject.toml index 5427434a5..4415ce51c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dev = ["ruff>=0.9.6", "pyright>=1.1.396"] [tool.ruff] target-version = "py313" -extend-exclude = [".urc", ".lore"] +extend-exclude = [".urc", ".spacesync"] [tool.pytest.ini_options] addopts = "--dist loadgroup --tb=short -rsxX" diff --git a/scripts/test/lore.py b/scripts/test/lore.py index 0c685f0fe..678c64720 100644 --- a/scripts/test/lore.py +++ b/scripts/test/lore.py @@ -608,8 +608,38 @@ def branch_merge_start( link: str | None = None, dry_run: bool = False, ignore_links: bool = False, - **kwargs: Unpack[GlobalOptions], - ): + merge_strategy: str | list[str] | None = None, + keep_target: str | list[str] | Path | list[Path] | None = None, + exclude: str | list[str] | Path | list[Path] | None = None, + **kwargs: Unpack[GlobalOptions], + ): + def optional_string_list(value): + if value is None: + return [] + if isinstance(value, list): + return [str(item) for item in value] + return [str(value)] + + def optional_path_list(value): + return [self._fix_path(path) for path in optional_string_list(value)] + + def optional_strategy_list(value): + strategies = [] + for item in optional_string_list(value): + entries = item.split(",") + converted = [] + for entry in entries: + if ":" not in entry: + converted.append(entry) + continue + strategy, path = entry.split(":", 1) + converted.append(f"{strategy}:{self._fix_path(path)}") + strategies.append(",".join(converted)) + return strategies + + merge_strategy = optional_strategy_list(merge_strategy) + keep_target = optional_path_list(keep_target) + exclude = optional_path_list(exclude) return self.run( ["branch", "merge", "start"] + ([name] if name else []) @@ -618,7 +648,10 @@ def branch_merge_start( + (["--no-commit"] if no_commit else []) + (["--link", link] if link else []) + (["--dry-run"] if dry_run else []) - + (["--ignore-links"] if ignore_links else []), + + (["--ignore-links"] if ignore_links else []) + + [item for rule in merge_strategy for item in ["--merge-strategy", rule]] + + [item for path in keep_target for item in ["--keep-target", path]] + + [item for path in exclude for item in ["--exclude", path]], **kwargs, ) diff --git a/scripts/test/test_merge_into.py b/scripts/test/test_merge_into.py index 98241651a..ca181cbbc 100644 --- a/scripts/test/test_merge_into.py +++ b/scripts/test/test_merge_into.py @@ -119,3 +119,30 @@ def test_merge_into(new_lore_repo): repo.revision_sync() output = repo.revision_history() verify_signatures(output, 5) + + +def test_merge_into_after_unpushed_target_merge(new_lore_repo): + repo: Lore = new_lore_repo() + repo.write_commit_push( + "Initial main commit", + {"original.txt": "hello from main\n"}, + ) + + clone = repo.clone(name="merge-into-clone") + clone.branch_create("feature-branch") + clone.write_files({"feature.txt": "hello from feature\n"}) + clone.stage(scan=True) + clone.commit("Feature commit") + + clone.branch_merge_into("main", "First merge into main") + + with pytest.raises(MergeRequired): + clone.branch_merge_into("main", "Repeat merge into main") + + clone.branch_merge_start("main", message="Merge main into feature") + + clone.branch_merge_into("main", "Merge feature merge commit into main") + clone.branch_switch("main") + clone.sync() + + assert clone.file_exists("feature.txt") diff --git a/scripts/test/test_merge_path_strategy.py b/scripts/test/test_merge_path_strategy.py new file mode 100644 index 000000000..d86a11e96 --- /dev/null +++ b/scripts/test/test_merge_path_strategy.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: 2026 Epic Games, Inc. +# SPDX-License-Identifier: MIT +from pathlib import Path + +from lore import Lore + + +def write_text(repo: Lore, path: str, text: str): + absolute = Path(repo.path) / path + absolute.parent.mkdir(parents=True, exist_ok=True) + absolute.write_text(text, encoding="utf-8") + + +def read_text(repo: Lore, path: str) -> str: + return (Path(repo.path) / path).read_text(encoding="utf-8") + + +def commit_all(repo: Lore, message: str): + repo.file_stage(scan=True, offline=True) + repo.commit(message, offline=True) + + +def assert_no_conflict_sidecars(repo: Lore, path: str): + assert not repo.file_exists(path + "~mine") + assert not repo.file_exists(path + "~theirs") + assert not repo.file_exists(path + "~base") + + +def test_merge_keep_target_path_strategy_keeps_git_dir(new_lore_repo): + repo: Lore = new_lore_repo() + + write_text(repo, ".git/config", "base\n") + write_text(repo, "base.txt", "base\n") + commit_all(repo, "Initial commit") + + repo.branch_create("feature", offline=True) + write_text(repo, ".git/config", "feature\n") + write_text(repo, ".git/objects/aa/file", "feature object\n") + write_text(repo, "feature.txt", "feature\n") + commit_all(repo, "Feature changes") + + repo.branch_switch("main", offline=True) + write_text(repo, ".git/config", "target\n") + commit_all(repo, "Target changes") + + repo.branch_merge_start("feature", keep_target=".git", offline=True) + + assert read_text(repo, ".git/config") == "target\n" + assert not repo.file_exists(".git/objects/aa/file") + assert repo.file_exists("feature.txt") + assert_no_conflict_sidecars(repo, ".git/config") + + +def test_merge_strategy_comma_excludes_multiple_paths(new_lore_repo): + repo: Lore = new_lore_repo() + + write_text(repo, ".git/config", "base\n") + write_text(repo, "generated/base.txt", "base\n") + write_text(repo, "base.txt", "base\n") + commit_all(repo, "Initial commit") + + repo.branch_create("feature", offline=True) + write_text(repo, ".git/config", "feature\n") + write_text(repo, "generated/base.txt", "feature generated\n") + write_text(repo, "generated/cache.txt", "feature cache\n") + write_text(repo, "feature.txt", "feature\n") + commit_all(repo, "Feature changes") + + repo.branch_switch("main", offline=True) + write_text(repo, ".git/config", "target\n") + write_text(repo, "generated/base.txt", "target generated\n") + commit_all(repo, "Target changes") + + repo.branch_merge_start( + "feature", + merge_strategy="exclude:.git,exclude:generated", + offline=True, + ) + + assert read_text(repo, ".git/config") == "target\n" + assert read_text(repo, "generated/base.txt") == "target generated\n" + assert not repo.file_exists("generated/cache.txt") + assert repo.file_exists("feature.txt") + assert_no_conflict_sidecars(repo, ".git/config") + assert_no_conflict_sidecars(repo, "generated/base.txt")