diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 2b64aec..b6ad72c 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -128,19 +128,19 @@ jobs: - uses: actions/upload-artifact@v4 with: name: sprig_unix_amd64.deb - path: installer-scripts/unix/sprig_0.0.10_amd64.deb + path: installer-scripts/unix/sprig_0.0.11_amd64.deb - uses: actions/upload-artifact@v4 with: name: sprig_unix_amd64.rpm - path: installer-scripts/unix/sprig-0.0.10-1.x86_64.rpm + path: installer-scripts/unix/sprig-0.0.11-1.x86_64.rpm - uses: actions/upload-artifact@v4 with: name: sprig_unix_amd64.apk - path: installer-scripts/unix/sprig_0.0.10_x86_64.apk + path: installer-scripts/unix/sprig_0.0.11_x86_64.apk - uses: actions/upload-artifact@v4 with: name: sprig_unix_amd64.pkg.tar.zst - path: installer-scripts/unix/sprig-0.0.10-1-x86_64.pkg.tar.zst + path: installer-scripts/unix/sprig-0.0.11-1-x86_64.pkg.tar.zst - uses: actions/upload-artifact@v4 with: name: sprig-target-directory-unix diff --git a/Cargo.lock b/Cargo.lock index 52ce7a9..566a3fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aes" @@ -39,9 +39,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -54,33 +54,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.8" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", "once_cell_polyfill", @@ -112,7 +112,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -123,7 +123,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -134,9 +134,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "axum" @@ -238,14 +238,14 @@ dependencies = [ [[package]] name = "bridgectl" -version = "0.0.10" +version = "0.0.11" dependencies = [ "cat-dev", "clap", "fnv", "futures", "libc", - "log 0.0.10", + "log 0.0.11", "mac_address", "miette", "terminal_size", @@ -256,15 +256,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytemuck" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" [[package]] name = "byteorder" @@ -280,7 +280,7 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cat-dev" -version = "0.0.10" +version = "0.0.11" dependencies = [ "aes", "bitflags", @@ -316,25 +316,25 @@ dependencies = [ [[package]] name = "catlog" -version = "0.0.10" +version = "0.0.11" dependencies = [ "tokio", ] [[package]] name = "cc" -version = "1.2.25" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "shlex", ] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -342,6 +342,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "num-traits", +] + [[package]] name = "cipher" version = "0.4.4" @@ -354,9 +363,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -364,9 +373,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", @@ -374,32 +383,32 @@ dependencies = [ "strsim", "terminal_size", "unicase", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "configparser" @@ -506,6 +515,22 @@ dependencies = [ "typenum", ] +[[package]] +name = "dbg-generate-sata-wal-from-pcap" +version = "0.0.11" +dependencies = [ + "bytes", + "cat-dev", + "clap", + "fnv", + "log 0.0.11", + "miette", + "rtshark", + "tokio", + "tracing", + "valuable", +] + [[package]] name = "deranged" version = "0.4.0" @@ -523,7 +548,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -558,12 +583,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -574,7 +599,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "findbridge" -version = "0.0.10" +version = "0.0.11" dependencies = [ "cat-dev", "mac_address", @@ -584,9 +609,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "miniz_oxide", @@ -663,7 +688,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -708,7 +733,7 @@ dependencies = [ [[package]] name = "getbridgeconfig" -version = "0.0.10" +version = "0.0.11" dependencies = [ "cat-dev", "tokio", @@ -722,7 +747,7 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", ] [[package]] @@ -745,9 +770,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" dependencies = [ "atomic-waker", "bytes", @@ -755,7 +780,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.9.0", + "indexmap 2.10.0", "slab", "tokio", "tokio-util", @@ -770,9 +795,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" [[package]] name = "hdrhistogram" @@ -881,9 +906,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ "base64 0.22.1", "bytes", @@ -1024,12 +1049,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -1042,6 +1067,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1103,9 +1139,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "linux-raw-sys" @@ -1143,7 +1179,7 @@ dependencies = [ [[package]] name = "log" -version = "0.0.10" +version = "0.0.11" dependencies = [ "console-subscriber", "miette", @@ -1185,9 +1221,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memoffset" @@ -1225,7 +1261,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1242,9 +1278,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] @@ -1256,13 +1292,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] [[package]] name = "mionparamspace" -version = "0.0.10" +version = "0.0.11" dependencies = [ "cat-dev", "time", @@ -1271,7 +1307,7 @@ dependencies = [ [[package]] name = "mionps" -version = "0.0.10" +version = "0.0.11" dependencies = [ "cat-dev", "time", @@ -1411,9 +1447,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "owo-colors" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec" +checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" [[package]] name = "parking_lot" @@ -1438,6 +1474,14 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "pcfsserver" +version = "0.0.11" +dependencies = [ + "cat-dev", + "tokio", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1461,7 +1505,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1529,7 +1573,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1541,6 +1585,15 @@ dependencies = [ "prost", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.40" @@ -1552,9 +1605,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" @@ -1617,9 +1670,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ "bitflags", ] @@ -1670,9 +1723,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.18" +version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98ff6b0dbbe4d5a37318f433d4fc82babd21631f194d370409ceb2e40b2f0b5" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ "base64 0.22.1", "bytes", @@ -1683,11 +1736,9 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", - "ipnet", "js-sys", "log 0.4.27", "mime", - "once_cell", "percent-encoding", "pin-project-lite", "serde", @@ -1704,11 +1755,22 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rtshark" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "429e822591c8463006c175ac9d95f179163e64dfe44f075b07640ca4da37f500" +dependencies = [ + "chrono", + "quick-xml", + "semver", +] + [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustix" @@ -1774,6 +1836,12 @@ version = "3.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "584e070911c7017da6cb2eb0788d09f43d789029b5877d3e5ecc8acf86ceee21" +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + [[package]] name = "serde" version = "1.0.219" @@ -1791,7 +1859,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1820,7 +1888,7 @@ dependencies = [ [[package]] name = "setbridgeconfig" -version = "0.0.10" +version = "0.0.11" dependencies = [ "cat-dev", "tokio", @@ -1852,18 +1920,15 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" @@ -1921,9 +1986,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.101" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -1947,14 +2012,14 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "sysinfo" -version = "0.35.1" +version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79251336d17c72d9762b8b54be4befe38d2db56fbbc0241396d70f173c39d47a" +checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e" dependencies = [ "libc", "objc2-core-foundation", @@ -2013,7 +2078,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ "unicode-linebreak", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -2042,7 +2107,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2053,17 +2118,16 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -2097,17 +2161,19 @@ dependencies = [ [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "1140bb80481756a8cbe10541f37433b459c5aa1e727b4c020fbfebdc25bf3ec4" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", "tracing", @@ -2122,7 +2188,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2216,9 +2282,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags", "bytes", @@ -2257,20 +2323,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -2371,9 +2437,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "url" @@ -2415,7 +2481,7 @@ checksum = "4e3a32a9bcc0f6c6ccfd5b27bcf298c58e753bcc9eeff268157a303393183a6d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2455,9 +2521,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -2496,7 +2562,7 @@ dependencies = [ "log 0.4.27", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -2531,7 +2597,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2568,9 +2634,9 @@ dependencies = [ [[package]] name = "wide" -version = "0.7.32" +version = "0.7.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b5576b9a81633f3e8df296ce0063042a73507636cbe956c61133dd7034ab22" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" dependencies = [ "bytemuck", "safe_arch", @@ -2609,9 +2675,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.61.1" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", "windows-core", @@ -2639,7 +2705,7 @@ dependencies = [ "windows-interface", "windows-link", "windows-result", - "windows-strings 0.4.2", + "windows-strings", ] [[package]] @@ -2661,7 +2727,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2672,14 +2738,14 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-numerics" @@ -2693,13 +2759,13 @@ dependencies = [ [[package]] name = "windows-registry" -version = "0.4.0" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" dependencies = [ + "windows-link", "windows-result", - "windows-strings 0.3.1", - "windows-targets 0.53.0", + "windows-strings", ] [[package]] @@ -2711,15 +2777,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -2747,6 +2804,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2765,9 +2831,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.0" +version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" dependencies = [ "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", @@ -2919,28 +2985,28 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2960,7 +3026,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "synstructure", ] @@ -2994,5 +3060,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] diff --git a/Cargo.toml b/Cargo.toml index 011cc5d..76aaff3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,10 +2,12 @@ members = [ "cmd/bridgectl", "cmd/catlog", + "cmd/dbg-generate-sata-wal-from-pcap", "cmd/findbridge", "cmd/getbridgeconfig", "cmd/mionps", "cmd/mionparamspace", + "cmd/pcfsserver", "cmd/setbridgeconfig", "pkg/cat-dev", "pkg/log", @@ -14,6 +16,7 @@ resolver = "3" [workspace.dependencies] bytes = "^1.10.1" +clap = { version = "^4.5.40", features = ["color", "derive", "env", "error-context", "help", "suggestions", "unicode", "usage", "wrap_help"] } fnv = "^1.0.7" futures = "^0.3.31" mac_address = "^1.1.8" @@ -21,7 +24,7 @@ miette = { version = "^7.6.0", features = ["fancy"] } network-interface = "^2.0.1" time = "^0.3.41" tracing = { version = "^0.1.41", features = ["valuable"] } -tokio = { version = "^1.45.1", features = ["full", "tracing"] } +tokio = { version = "^1.46.0", features = ["full", "tracing"] } valuable = { version = "^0.1.1", features = ["derive"] } [workspace.package] @@ -29,7 +32,7 @@ authors = ["Cynthia "] edition = "2024" license = "MIT" repository = "https://github.com/rem-verse/sprig" -version = "0.0.10" +version = "0.0.11" [profile.release] codegen-units = 1 diff --git a/README.md b/README.md index a38ea65..f6c7692 100644 --- a/README.md +++ b/README.md @@ -75,10 +75,12 @@ Official Tool Replacements: installation version, and the firmware installed on your actual CAT-DEV. It is typically only used for diagnostics. - - [ ] `FSEmul`: FSEmul is the 'core' proccess for handling emulation of - various filesystem components for the CAT-DEV. Specifically - FSEmul handles various block level protocols (SDIO/ATAPI), - and provides information to `PCFSServer` +- [ ] `FSEmul`: FSEmul is the 'core' proccess for handling emulation of + various filesystem components for the CAT-DEV. Specifically + FSEmul handles various block level protocols (SDIO/ATAPI), + and provides information to `PCFSServer` + - [ ] `PCFSServer`: handles interacting with an actual filesystem with SATA + style apis like "create a folder", "create a file", etc. - [ ] `updatebridges`: a command used to update the firmware on a particular Host Bridge. - [ ] `imageuploader`: allow uploading mastered `WUMAD`/`WUM`'s to the internal diff --git a/cmd/bridgectl/Cargo.toml b/cmd/bridgectl/Cargo.toml index 77f078d..862988c 100644 --- a/cmd/bridgectl/Cargo.toml +++ b/cmd/bridgectl/Cargo.toml @@ -11,7 +11,7 @@ publish = false [dependencies] cat-dev = { default-features = false, features = ["clients", "serial", "scientists", "servers"], path = "../../pkg/cat-dev" } -clap = { version = "^4.5.39", features = ["color", "derive", "env", "error-context", "help", "suggestions", "unicode", "usage", "wrap_help"] } +clap.workspace = true fnv.workspace = true futures.workspace = true log = { path = "../../pkg/log" } diff --git a/cmd/bridgectl/src/commands/argv_helpers/fsemul_conf.rs b/cmd/bridgectl/src/commands/argv_helpers/fsemul_conf.rs index a8cd101..473ff5c 100644 --- a/cmd/bridgectl/src/commands/argv_helpers/fsemul_conf.rs +++ b/cmd/bridgectl/src/commands/argv_helpers/fsemul_conf.rs @@ -20,7 +20,13 @@ use crate::{ }; use cat_dev::fsemul::{FSEmulConfig, HostFilesystem}; use miette::miette; -use std::{path::PathBuf, sync::OnceLock}; +use std::{ + path::PathBuf, + sync::{ + OnceLock, + atomic::{AtomicBool, Ordering}, + }, +}; use tokio::sync::{RwLock, RwLockMappedWriteGuard, RwLockReadGuard, RwLockWriteGuard}; use tracing::{error, field::valuable, info}; @@ -29,6 +35,7 @@ static HOST_FILE_SYSTEM: OnceLock = OnceLock::new(); static FSEMUL_CONFIG_PATH: RwLock> = RwLock::const_new(None); static CAFE_DATA_PATH: RwLock> = RwLock::const_new(None); +static FORCE_UNIQUE_FDS: AtomicBool = AtomicBool::new(false); /// Initialize all of the stuff necessary for fetching from the bridge /// configuration file. @@ -62,6 +69,9 @@ pub async fn initialize_fsemul_config(fsemul_config_flags: &FSEmulConfigurationF ); } } + if fsemul_config_flags.force_unique_fds() { + FORCE_UNIQUE_FDS.store(true, Ordering::SeqCst); + } if let Some(cli_arg) = fsemul_config_flags.fsemul_config_path() { let mut locked_env_path = FSEMUL_CONFIG_PATH.write().await; @@ -250,7 +260,13 @@ async fn try_to_load_host_file_system() { let host_fs_path = read_env_path.as_ref().expect("impossible"); match futures::executor::block_on(HostFilesystem::from_cafe_dir(Some(host_fs_path.clone()))) { - Ok(state) => state, + Ok(mut state) => { + if FORCE_UNIQUE_FDS.load(Ordering::SeqCst) { + // This is guaranteed to work, the host filesystem _just_ created. + std::mem::drop(state.force_unique_fds()); + } + state + }, Err(cause) => { if SHOULD_LOG_JSON() { error!( @@ -404,7 +420,13 @@ async fn validate_host_file_system_is_populated() { match futures::executor::block_on(HostFilesystem::from_cafe_dir(Some( cafe_root_path.clone(), ))) { - Ok(state) => state, + Ok(mut state) => { + if FORCE_UNIQUE_FDS.load(Ordering::SeqCst) { + // This is guaranteed to work, the host filesystem _just_ created. + std::mem::drop(state.force_unique_fds()); + } + state + } Err(cause) => { if SHOULD_LOG_JSON() { error!( diff --git a/cmd/bridgectl/src/commands/argv_helpers/serial.rs b/cmd/bridgectl/src/commands/argv_helpers/serial.rs index 900ae2c..5642db6 100644 --- a/cmd/bridgectl/src/commands/argv_helpers/serial.rs +++ b/cmd/bridgectl/src/commands/argv_helpers/serial.rs @@ -1,30 +1,23 @@ use crate::{ SHOULD_LOG_JSON, - commands::argv_helpers::lease_fsemul_config_optionally, exit_codes::{ ARGV_SERIAL_CONFLICTING_ARGUMENTS, SERIAL_PORT_CONNECTION_FAILURE, SHOULD_NEVER_HAPPEN_FAILURE, }, - knobs::{ - cli::SharedSerialPortFlags, - env::{BRIDGECTL_SERIAL_PORT, SESSION_DEBUG_OUT_PORT}, - }, + knobs::{cli::SharedSerialPortFlags, env::BRIDGECTL_SERIAL_PORT}, utils::add_context_to, }; use cat_dev::serial::{AsyncSerialPort, SerialLines}; use miette::miette; -use std::{net::Ipv4Addr, path::PathBuf, time::Duration}; +use std::{path::PathBuf, time::Duration}; use tokio::{ io::{AsyncBufRead, BufReader}, - net::TcpStream, signal::ctrl_c as ctrl_c_signal, task::{Builder as TaskBuilder, JoinHandle}, + time::sleep, }; use tracing::{Instrument, debug, error, error_span, field::valuable, info, warn}; -/// The amount of times we'll try reconnecting to debug out. -const DEBUG_OUT_RETRY_COUNT: usize = 5_usize; - /// Determines if a positional argument that could be a path, or something /// not a path, should be interpreted as a path. /// @@ -69,7 +62,6 @@ pub fn should_interpret_arg_as_port_path(arg: Option<&String>) -> bool { /// for a serial port. /// - If we cannot open a handle/descriptor to the associated serial device. pub async fn coalesce_serial_ports( - mion_ip: Ipv4Addr, serial_port_flags: &SharedSerialPortFlags, serial_port_positional: Option<&PathBuf>, ) -> SerialLogger { @@ -112,18 +104,7 @@ pub async fn coalesce_serial_ports( } else if let Some(env) = BRIDGECTL_SERIAL_PORT.as_ref() { env } else { - let fsemul_port = lease_fsemul_config_optionally() - .await - .and_then(|config| config.get_debug_out_port()); - // Connect to the debug out port. - return SerialLogger::new_from_debug_out_port( - mion_ip, - serial_port_flags - .debug_out_port() - .or(*SESSION_DEBUG_OUT_PORT) - .or(fsemul_port) - .unwrap_or(6001), - ); + return SerialLogger::honk_shoo(); }; let port = match AsyncSerialPort::new(arg_to_take) { @@ -166,9 +147,6 @@ pub async fn coalesce_serial_ports( /// A logger capable of reading logs from a 'serial port' style logger. pub struct SerialLogger { - /// A port to connect over `DEBUG_OUT` which just shoves serial logs as - /// raw bytes. - debug_out: Option<(Ipv4Addr, u16)>, /// A wrapper around an actual physical connected serial port. /// /// This is a tuple type of (Async Serial Port Stream, Path to Serial Port). @@ -182,49 +160,54 @@ impl SerialLogger { #[must_use] pub const fn new_from_serial_port(port: AsyncSerialPort, path: PathBuf) -> Self { Self { - debug_out: None, serial_port: Some((port, path)), } } - /// Construct a logger based off a port to connect to `DEBUG_OUT`... + /// Construct a logger that does nothing but sleep... #[must_use] - pub const fn new_from_debug_out_port(mion_ip: Ipv4Addr, port: u16) -> Self { - Self { - debug_out: Some((mion_ip, port)), - serial_port: None, - } + pub const fn honk_shoo() -> Self { + Self { serial_port: None } + } + + #[must_use] + pub const fn has_serial_port(&self) -> bool { + self.serial_port.is_some() } /// Spawn a task that will watch for logs.... pub fn spawn_log_task(self) -> JoinHandle<()> { if let Some((port, port_path)) = self.serial_port { Self::spawn_serial_log_task(port, port_path) - } else if let Some((ip, port)) = self.debug_out { - Self::spawn_debug_out_task(ip, port) } else { - unreachable!("No way to construct a serial logger without debug_out, or serial_port") + Self::spawn_sleep_task() } } - /// Spawn a task that reads serial logs from the debug out port over, and over. + /// Spawn a task that just sleeps forever. #[allow(clippy::blocks_in_conditions)] - fn spawn_debug_out_task(ip: Ipv4Addr, port: u16) -> JoinHandle<()> { + fn spawn_sleep_task() -> JoinHandle<()> { match TaskBuilder::new() - .name("bridgectl::serial_log::debug_out_watcher") + .name("bridgectl::serial_log::honk_shoo") .spawn(async move { - tokio::select! { - () = Self::do_remote_serial_connect_and_process(ip, port) => {} - _ = ctrl_c_signal() => if SHOULD_LOG_JSON() { - info!( - id = "bridgectl::serial::debug_out_detected_ctrlc", - "ctrl-c has been hit, shutting down serial logger for DEBUG_OUT!", - ); - } else { - info!( - "Ctrl-C has been detected as being hit! Shutting down serial logger for DEBUG_OUT!" - ); + loop { + tokio::select! { + () = sleep(Duration::from_secs(u64::MAX)) => {} + _ = ctrl_c_signal() => { + if SHOULD_LOG_JSON() { + info!( + id = "bridgectl::serial::honk_shoo_detected_ctrlc", + "ctrl-c has been hit, shutting down empty serial logger!", + ); + } else { + info!( + "Ctrl-C has been detected as being hit! Shutting down empty serial logger!" + ); + } + + break; } + } } }) { Ok(port) => port, @@ -247,94 +230,6 @@ impl SerialLogger { } } - /// Actually spawn a connection to a `DEBUG_OUT` port, and process any log - /// messages that come in. - async fn do_remote_serial_connect_and_process(ip: Ipv4Addr, port: u16) { - info!( - id = "bridgectl::serial::debug_out::start_connection", - "Connecting to MION DEBUG_OUT port..." - ); - let mut stream_opt = None; - - for _ in 0..DEBUG_OUT_RETRY_COUNT { - stream_opt = - match tokio::time::timeout(Duration::from_secs(30), TcpStream::connect((ip, port))) - .await - { - Ok(Ok(stream)) => Some(stream), - Ok(Err(cause)) => { - if SHOULD_LOG_JSON() { - warn!( - alternatives = valuable(&[ - "You can always connect a USB to Serial Adapter to your cat-dev to get logs consistently", - ]), - id = "bridgectl::serial::debug_out::connection_failure", - ?cause, - "we could not connect to the Cat-Dev to listen for serial logs over DEBUG_OUT", - ); - } else { - warn!( - alternatives = valuable(&[ - "You can always connect a USB to Serial Adapter to your cat-dev to get logs consistently", - ]), - ?cause, - "we could not connect to the Cat-Dev to listen for serial logs over DEBUG_OUT", - ); - } - - None - } - Err(cause) => { - if SHOULD_LOG_JSON() { - warn!( - alternatives = valuable(&[ - "You can always connect a USB to Serial Adapter to your cat-dev to get logs consistently", - ]), - id = "bridgectl::serial::debug_out::connection_failure", - ?cause, - "we could not connect to the Cat-Dev to listen for serial logs over DEBUG_OUT", - ); - } else { - warn!( - alternatives = valuable(&[ - "You can always connect a USB to Serial Adapter to your cat-dev to get logs consistently", - ]), - ?cause, - "we could not connect to the Cat-Dev to listen for serial logs over DEBUG_OUT", - ); - } - - None - } - }; - - if stream_opt.is_some() { - break; - } - } - - let Some(stream) = stream_opt else { - info!( - id = "bridgectl::serial::debug_out::setup_connection", - "Failed to find debug out connection! Not trying again...." - ); - return; - }; - - info!( - id = "bridgectl::serial::debug_out::setup_connection", - "Connected to DEBUG_OUT! Now streaming logs..." - ); - - Self::do_serial_read_loop(BufReader::new(stream)) - .instrument(error_span!( - "bridgectl::serial::watch_debug_out", - debug_out.ip = %ip, - debug_out.port = port, - )) - .await; - } - /// Spawn a task that reads from a physical serial port over, and over again. #[allow(clippy::blocks_in_conditions)] fn spawn_serial_log_task(port: AsyncSerialPort, port_path: PathBuf) -> JoinHandle<()> { diff --git a/cmd/bridgectl/src/commands/boot/mod.rs b/cmd/bridgectl/src/commands/boot/mod.rs index ff31d50..2e27cce 100644 --- a/cmd/bridgectl/src/commands/boot/mod.rs +++ b/cmd/bridgectl/src/commands/boot/mod.rs @@ -73,7 +73,7 @@ use crate::{ cli::{FSEmulConfigurationFlags, SharedSerialPortFlags}, env::{ ATAPI_DISABLE_LOAD_BEARING_SLEEP, FSEMUL_DISABLE_REMOVAL, PCFS_DISABLE_CSR, - PCFS_DISABLE_FFIO, PCFS_DISABLE_LOAD_BEARING_SLEEP, PCFS_IS_SATA, + PCFS_DISABLE_FFIO, PCFS_DISABLE_LOAD_BEARING_SLEEP, PCFS_IS_SATA, SATA_WAL_LOG, SDIO_DISABLE_LOAD_BEARING_SLEEP, }, }, @@ -103,10 +103,9 @@ pub async fn handle_boot( let host_ip = get_host_bind_address().await; let is_modern_bridge = is_modern_bridge(bridge_ip).await; - let serial_task_handle = - coalesce_serial_ports(bridge_ip, &serial_port_args.0, serial_port_args.1) - .await - .spawn_log_task(); + let serial_task_handle = coalesce_serial_ports(&serial_port_args.0, serial_port_args.1) + .await + .spawn_log_task(); let (_info_request, setup_params, needs_pcfs) = validate_bridge_ready_for_booting( is_modern_bridge, @@ -137,8 +136,6 @@ pub async fn handle_boot( return; } - // TODO(mythra): properly serve a disc.... - turn_down_for_disc(bridge_ip).await; let file_system = lease_host_file_system().await; let final_atapi_port = serve_atapi( @@ -153,17 +150,8 @@ pub async fn handle_boot( fsemul_flags.disable_load_bearing_sleep_for_atapi() || *ATAPI_DISABLE_LOAD_BEARING_SLEEP, ) .await; - serve_sdio( - bridge_ip, - setup_params.as_ref(), - file_system, - get_sdio_control_port().await, - get_sdio_printf_port().await, - fsemul_flags.disable_load_bearing_sleep_for_sdio() || *SDIO_DISABLE_LOAD_BEARING_SLEEP, - ) - .await; - let will_use_sata = get_will_use_sata(disable_sata); + let will_use_sata = get_will_use_sata(disable_sata); let sata_port = if will_use_sata { let p = serve_sata( file_system, @@ -177,13 +165,28 @@ pub async fn handle_boot( fsemul_flags.disable_ffio() || *PCFS_DISABLE_FFIO, fsemul_flags.disable_csr() || *PCFS_DISABLE_CSR, fsemul_flags.disable_load_bearing_sleep_for_pcfs() || *PCFS_DISABLE_LOAD_BEARING_SLEEP, + fsemul_flags + .sata_wal_log() + .or(SATA_WAL_LOG.as_ref()) + .cloned(), ) .await; + Some(p) } else { None }; + serve_sdio( + bridge_ip, + setup_params.as_ref(), + file_system, + get_sdio_control_port().await, + get_sdio_printf_port().await, + fsemul_flags.disable_load_bearing_sleep_for_sdio() || *SDIO_DISABLE_LOAD_BEARING_SLEEP, + ) + .await; + wrap_power_on( is_modern_bridge, bridge_ip, diff --git a/cmd/bridgectl/src/commands/boot/sata.rs b/cmd/bridgectl/src/commands/boot/sata.rs index 5d72f01..ae3b9d2 100644 --- a/cmd/bridgectl/src/commands/boot/sata.rs +++ b/cmd/bridgectl/src/commands/boot/sata.rs @@ -18,7 +18,7 @@ use cat_dev::{ net::server::TCPServer, }; use miette::miette; -use std::net::Ipv4Addr; +use std::{net::Ipv4Addr, path::PathBuf}; use tokio::{signal::ctrl_c as ctrl_c_signal, task::Builder as TaskBuilder}; use tracing::{error, info}; @@ -26,6 +26,7 @@ use tracing::{error, info}; #[allow( // CLIPPY ILL THINK ABOUT IT. clippy::fn_params_excessive_bools, + clippy::too_many_arguments, )] pub async fn serve_sata( host_filesystem: &'static HostFilesystem, @@ -35,6 +36,7 @@ pub async fn serve_sata( disable_ffio: bool, disable_csr: bool, disable_load_bearing_sleep: bool, + wal_log_path: Option, ) -> u16 { let sata_server = match pcfs_sata_server( host_filesystem.clone(), @@ -43,6 +45,7 @@ pub async fn serve_sata( disable_ffio, disable_csr, disable_real_removal, + wal_log_path, *PCFS_OVERRIDE_LOAD_BEARING_SLEEP_MS, disable_load_bearing_sleep, None, diff --git a/cmd/bridgectl/src/commands/tail.rs b/cmd/bridgectl/src/commands/tail.rs index 7a28790..5e657bd 100644 --- a/cmd/bridgectl/src/commands/tail.rs +++ b/cmd/bridgectl/src/commands/tail.rs @@ -1,7 +1,7 @@ use crate::{ SHOULD_LOG_JSON, - commands::argv_helpers::{coalesce_serial_ports, get_targeted_bridge_ip}, - exit_codes::TAIL_COULD_NOT_SPAWN, + commands::argv_helpers::coalesce_serial_ports, + exit_codes::{TAIL_COULD_NOT_SPAWN, TAIL_NO_SERIAL_PORT}, knobs::cli::SharedSerialPortFlags, utils::add_context_to, }; @@ -14,12 +14,24 @@ pub async fn handle_tail( serial_port_positional: Option, serial_port_flags: SharedSerialPortFlags, ) { - let serial_reader = coalesce_serial_ports( - get_targeted_bridge_ip().await, - &serial_port_flags, - serial_port_positional.as_ref(), - ) - .await; + let serial_reader = + coalesce_serial_ports(&serial_port_flags, serial_port_positional.as_ref()).await; + + if !serial_reader.has_serial_port() { + if SHOULD_LOG_JSON() { + error!( + id = "bridgectl::tail::no_serial_port", + "tail requires a serial port to connect too for now" + ); + } else { + error!( + "\n{:?}", + miette!("Tailing currently requires an active connection to the serial port."), + ); + } + + std::process::exit(TAIL_NO_SERIAL_PORT); + } if let Err(cause) = serial_reader.spawn_log_task().await { if SHOULD_LOG_JSON() { diff --git a/cmd/bridgectl/src/exit_codes.rs b/cmd/bridgectl/src/exit_codes.rs index 3578770..c8b0963 100644 --- a/cmd/bridgectl/src/exit_codes.rs +++ b/cmd/bridgectl/src/exit_codes.rs @@ -57,6 +57,7 @@ pub const SET_PARAMS_INVALID_PARAMETER_VALUE: i32 = 77; pub const SET_PARAMS_FAILED_TO_SET_PARAMS: i32 = 78; pub const TAIL_COULD_NOT_SPAWN: i32 = 80; +pub const TAIL_NO_SERIAL_PORT: i32 = 81; pub const DUMP_EEPROM_FAILURE: i32 = 85; diff --git a/cmd/bridgectl/src/knobs/cli.rs b/cmd/bridgectl/src/knobs/cli.rs index 00e680e..90135f8 100644 --- a/cmd/bridgectl/src/knobs/cli.rs +++ b/cmd/bridgectl/src/knobs/cli.rs @@ -1052,6 +1052,24 @@ pub struct FSEmulConfigurationFlags { long_help = "If we should prefer the `fsemul.ini` file over the web configuration, this makes us act like nintendo's tools, but also means your configuration needs to be up to date." )] prefer_fsemul_over_network: bool, + #[arg( + long = "force-unique-fds", + visible_aliases = [ + "force_unique_fds", + "counter-fds", + "counter_fds" + ], + help = "Force unique file descriptors for files from HostFilesystem.", + long_help = "If you want an easier time debugging fds and your OS reuses them, set this flag to guarantee all files get unique fds.", + )] + force_unique_fds: bool, + #[arg( + long = "sata-wal-log", + alias = "sata_wal_log", + help = "Where we should create a SATA WAL log for all PCFS Sata requests.", + long_help = "Where we should create a SATA WAL log for all PCFS Sata requests, not that this does carry increased memory, and CPU time needing to be spent." + )] + sata_wal_log: Option, } impl FSEmulConfigurationFlags { #[must_use] @@ -1094,16 +1112,26 @@ impl FSEmulConfigurationFlags { self.fsemul_config_path.as_ref() } + #[must_use] + pub const fn force_unique_fds(&self) -> bool { + self.force_unique_fds + } + #[must_use] pub const fn prefer_fsemul_over_network(&self) -> bool { self.prefer_fsemul_over_network } + + #[must_use] + pub fn sata_wal_log(&self) -> Option<&PathBuf> { + self.sata_wal_log.as_ref() + } } impl Display for FSEmulConfigurationFlags { fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult { write!( fmt, - "FS Emulation Config Location Override Flag --fsemul-config-path: `{:?}`, --prefer-fsemul-over-network: `{}`, --cafe-dir: `{:?}`, --disable-csr: `{}`, --disable-ffio: `{}`, --disable-load-bearing-sleep-for-atapi: `{}`, --disable-load-bearing-sleep-for-pcfs: `{}`, --disable-load-bearing-sleep-for-sdio: `{}`, --disable-real-removal: `{}`", + "FS Emulation Config Location Override Flag --fsemul-config-path: `{:?}`, --prefer-fsemul-over-network: `{}`, --cafe-dir: `{:?}`, --disable-csr: `{}`, --disable-ffio: `{}`, --disable-load-bearing-sleep-for-atapi: `{}`, --disable-load-bearing-sleep-for-pcfs: `{}`, --disable-load-bearing-sleep-for-sdio: `{}`, --disable-real-removal: `{}`, --force-unique-fds: `{}`, --sata-wal-log: `{:?}`", self.fsemul_config_path, self.prefer_fsemul_over_network, self.cafe_dir, @@ -1113,6 +1141,8 @@ impl Display for FSEmulConfigurationFlags { self.disable_load_bearing_sleep_for_pcfs, self.disable_load_bearing_sleep_for_sdio, self.disable_real_removal, + self.force_unique_fds, + self.sata_wal_log, ) } } @@ -1126,6 +1156,8 @@ const FSEMUL_CONFIGURATION_FLAG_FIELDS: &[NamedField<'static>] = &[ NamedField::new("disable_load_bearing_sleep_for_pcfs"), NamedField::new("disable_load_bearing_sleep_for_sdio"), NamedField::new("disable_real_removal"), + NamedField::new("force_unique_fds"), + NamedField::new("sata_wal_log"), ]; impl Structable for FSEmulConfigurationFlags { fn definition(&self) -> StructDef<'_> { @@ -1158,6 +1190,13 @@ impl Valuable for FSEmulConfigurationFlags { Valuable::as_value(&self.disable_load_bearing_sleep_for_pcfs), Valuable::as_value(&self.disable_load_bearing_sleep_for_sdio), Valuable::as_value(&self.disable_real_removal), + Valuable::as_value(&self.force_unique_fds), + Valuable::as_value( + &self + .sata_wal_log + .as_ref() + .map(|pb| format!("{}", pb.display())), + ), ], )); } @@ -1291,38 +1330,24 @@ pub struct SharedSerialPortFlags { long_help = "The path to the serial port to use, on Windows you should use something like 'COM1', 'COM2', etc., on Linux this should be the full path to the device (conflicts with the positional argument)." )] serial_port_flag: Option, - #[arg( - long = "debug-out-port", - alias = "debug_out_port", - help = "A port override to determine where we should connect for `DEBUG_OUT` logs.", - long_help = "A port override to determine where we should connect for `DBEUG_OUT` logs, the default port is 6001." - )] - debug_out_port: Option, } impl SharedSerialPortFlags { #[must_use] pub fn serial_port_flag(&self) -> Option<&PathBuf> { self.serial_port_flag.as_ref() } - - #[must_use] - pub fn debug_out_port(&self) -> Option { - self.debug_out_port - } } impl Display for SharedSerialPortFlags { fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult { write!( fmt, - "Shared Serial Port Flags --serial-port-path: `{:?}`, --debug-out-port: `{:?}`", - self.serial_port_flag, self.debug_out_port, + "Shared Serial Port Flags --serial-port-path: `{:?}`", + self.serial_port_flag, ) } } -const SHARED_SERIAL_PORT_FLAG_FIELDS: &[NamedField<'static>] = &[ - NamedField::new("serial_port_flag"), - NamedField::new("debug_out_port"), -]; +const SHARED_SERIAL_PORT_FLAG_FIELDS: &[NamedField<'static>] = + &[NamedField::new("serial_port_flag")]; impl Structable for SharedSerialPortFlags { fn definition(&self) -> StructDef<'_> { StructDef::new_static( @@ -1339,15 +1364,12 @@ impl Valuable for SharedSerialPortFlags { fn visit(&self, visitor: &mut dyn Visit) { visitor.visit_named_fields(&NamedValues::new( SHARED_SERIAL_PORT_FLAG_FIELDS, - &[ - Valuable::as_value( - &self - .serial_port_flag - .as_ref() - .map(|p| format!("{}", p.display())), - ), - Valuable::as_value(&self.debug_out_port), - ], + &[Valuable::as_value( + &self + .serial_port_flag + .as_ref() + .map(|p| format!("{}", p.display())), + )], )); } } diff --git a/cmd/bridgectl/src/knobs/env.rs b/cmd/bridgectl/src/knobs/env.rs index fcd5ca6..cb45d98 100644 --- a/cmd/bridgectl/src/knobs/env.rs +++ b/cmd/bridgectl/src/knobs/env.rs @@ -267,6 +267,18 @@ pub static PCFS_DISABLE_LOAD_BEARING_SLEEP: LazyLock = LazyLock::new(|| { pub static PCFS_IS_SATA: LazyLock = LazyLock::new(|| env_var("USE_PCFS_OVER_SATA").ok().as_deref().unwrap_or("1") == "1"); +/// Determines where we should write a WAL log, and by consequence if we will. +/// +/// note: WAL logging currently doubles the parsing, memory, etc. for +/// requests/responses to SATA. Ideally we can lower this in the future but +/// you should be aware of it for now. +/// +/// Environment Variable Name: `SATA_WAL_LOG` +/// Expected Values: The path we can write to of a file the WAL log should be written. +/// Type: [`PathBuf`] +pub static SATA_WAL_LOG: LazyLock> = + LazyLock::new(|| env_var_os("SATA_WAL_LOG").map(PathBuf::from)); + /// Change the load bearing sleep time for your MION when communicating over /// SDIO. /// diff --git a/cmd/dbg-generate-sata-wal-from-pcap/Cargo.toml b/cmd/dbg-generate-sata-wal-from-pcap/Cargo.toml new file mode 100644 index 0000000..664d091 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "dbg-generate-sata-wal-from-pcap" +description = "A tool that can generate a SATA WAL log from a PCAPNG." +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +version.workspace = true +# This is a debugging tool for us, it doesn't need to live on CRATES.io +publish = false + +[dependencies] +bytes.workspace = true +cat-dev = { default-features = false, features = ["servers"], path = "../../pkg/cat-dev" } +clap.workspace = true +fnv.workspace = true +log = { path = "../../pkg/log" } +miette.workspace = true +rtshark = "^3.2.0" +tokio.workspace = true +tracing.workspace = true +valuable.workspace = true \ No newline at end of file diff --git a/cmd/dbg-generate-sata-wal-from-pcap/README.md b/cmd/dbg-generate-sata-wal-from-pcap/README.md new file mode 100644 index 0000000..c8766f6 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/README.md @@ -0,0 +1,24 @@ +# `dbg-generate-sata-wal-from-pcap` # + +- [ ] **Tool Re-Implementation** +- [ ] **Script** + +`dbg-generate-sata-wal-from-pcap` is a tool primarly built for debugging PCFS +Sata streams that have been captured in `pcap`, or `pcapng` format. This can +also make it useful for comparing against an official implementation. You can +capture the PCAP of the official SDK, and use it to generate a WAL log that you +can compare with a WAL log from Sprig. + +## Building ## + +In order to build you can follow the project instructions, or if you want to +build just this one single package you can use: `cargo build -p dbg-generate-sata-wal-from-pcap` +from the root directory of the project to build a debug version of the +application. It will be available at: `${project-dir}/target/debug/dbg-generate-sata-wal-from-pcap`, +or `${project-dir}/target/debug/dbg-generate-sata-wal-from-pcap.exe` if you are on windows. If you +want to build a release version that is fully optimized you want to use the +command: `cargo b --release -p dbg-generate-sata-wal-from-pcap`. It will be available at: +`${project-dir}/target/release/dbg-generate-sata-wal-from-pcap`, or +`${project-dir}/target/release/dbg-generate-sata-wal-from-pcap.exe` respectively. This project +should be compatible with any Rust version above: `1.63.0`, although it's +always safest to build with whatever the latest version of Rust is at the time. diff --git a/cmd/dbg-generate-sata-wal-from-pcap/build.rs b/cmd/dbg-generate-sata-wal-from-pcap/build.rs new file mode 100644 index 0000000..db0ef00 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/build.rs @@ -0,0 +1,105 @@ +//! Most of this build script comes from [NEARD](https://github.com/near/nearcore) +//! +//! Which is licensed under APACHE2/MIT at the time of copying. + +use std::io::Error as IoError; + +fn env(key: &str) -> Result { + println!("cargo:rerun-if-env-changed={key}"); + std::env::var_os(key).ok_or_else(|| format!("missing `{key}` environment variable")) +} + +/// Calls program with given arguments and returns its standard output. If +/// calling the program fails or it exits with non-zero exit status returns an +/// error. +fn command(prog: &str, args: &[&str], cwd: Option) -> Result, IoError> { + println!("cargo:rerun-if-env-changed=PATH"); + let mut command = std::process::Command::new(prog); + command.args(args); + command.stdout(std::process::Stdio::piped()); + command.stderr(std::process::Stdio::inherit()); + if let Some(cwd) = cwd { + command.current_dir(cwd); + } + let out = command.output()?; + if out.status.success() { + let mut stdout = out.stdout; + if let Some(b'\n') = stdout.last() { + stdout.pop(); + if let Some(b'\r') = stdout.last() { + stdout.pop(); + } + } + Ok(stdout) + } else if let Some(code) = out.status.code() { + Err(IoError::other(format!("{prog}: terminated with {code}"))) + } else { + Err(IoError::other(format!("{prog}: killed by signal"))) + } +} + +/// Expose the git version to dgswfp so it can print it out! +fn main() { + // Figure out git directory. Don’t just assume it’s ../.git because that + // doesn’t work with git work trees so use `git rev-parse --git-dir` instead. + let pkg_dir = + std::path::PathBuf::from(env("CARGO_MANIFEST_DIR").expect("need manifest directory")); + let git_dir = command("git", &["rev-parse", "--git-dir"], Some(pkg_dir)); + let git_dir = match git_dir { + Ok(git_dir) => { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStringExt; + std::path::PathBuf::from(std::ffi::OsString::from_vec(git_dir)) + } + + #[cfg(not(unix))] + { + // Windows paths are always guaranteed to be UTF paths. + std::path::PathBuf::from( + String::from_utf8(git_dir).expect("Failed to parse windows path as utf8!"), + ) + } + } + Err(msg) => { + // We’re probably not inside of a git repository so report git + // version as unknown. + println!("cargo:warning=unable to determine git version (not in git repository?)"); + println!("cargo:warning={msg}"); + println!("cargo:rustc-env=DGSWFP_BUILD=unknown"); + return; + } + }; + + // Make Cargo rerun us if currently checked out commit or the state of the + // working tree changes. We try to accomplish that by looking at a few + // crucial git state files. This probably may result in some false + // negatives but it’s best we’ve got. + for subpath in ["HEAD", "logs/HEAD", "index"] { + let path = git_dir + .join(subpath) + .canonicalize() + .expect("Failed to get canonical path to git directory"); + println!("cargo:rerun-if-changed={}", path.display()); + } + + // * --always → if there is no matching tag, use commit hash + // * --dirty=-sussy → append ‘-sussy’ if there are local changes + // * --tags → consider tags even if they are unannotated + // * --match=[0-9]* → only consider tags starting with a digit; this + // prevents tags such as `crates-0.14.0` from being considered + let args = &[ + "describe", + "--always", + "--dirty=-sussy", + "--tags", + "--match=[0-9]*", + ]; + let out = command("git", args, None).expect("Failed to run git describe!"); + let git_version = match String::from_utf8_lossy(&out) { + std::borrow::Cow::Borrowed(version) => version.trim().to_string(), + std::borrow::Cow::Owned(version) => panic!("git: invalid output: {version}"), + }; + + println!("cargo:rustc-env=DGSWFP_BUILD={git_version}"); +} diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/commands/help.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/help.rs new file mode 100644 index 0000000..3fb4c00 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/help.rs @@ -0,0 +1,181 @@ +//! Handles the help command, or when the help flag is specified on a +//! particular subcommand. +//! +//! We have to handle `help` ourselves, as opposed to FULLY relying on [`clap`] +//! so we can do things like printing the output in JSON. + +use crate::{ + SHOULD_LOG_JSON, + knobs::cli::{CliArguments, Subcommands}, +}; +use clap::{Arg, Command, CommandFactory}; +use tracing::{field::valuable, info}; +use valuable::Valuable; + +/// Actually process the help command. +/// +/// ## Panics +/// +/// - Techincally this function could panic if a subcommand could not be +/// matched on name alone, which should never happen. +pub fn handle_help(opt_sub_command: Option) { + let mut top_level_command = CliArguments::command(); + let mut subcommands_as_command = Subcommands::command(); + + if !SHOULD_LOG_JSON() { + if let Some(sub_command) = opt_sub_command { + let mut subcommands_as_command = Subcommands::command(); + let my_command = subcommands_as_command + .get_subcommands_mut() + .find(|potential_command| sub_command.name_matches(potential_command.get_name())) + .expect("internal error: recognized subcommand could not be matched on name?"); + info!("{}", my_command.render_long_help()); + } else { + info!("{}", top_level_command.render_long_help()); + } + return; + } + + let (command, is_top_level) = if let Some(sub_command) = opt_sub_command { + let my_command = subcommands_as_command + .get_subcommands_mut() + .find(|potential_command| sub_command.name_matches(potential_command.get_name())) + .expect(r#"{{"id": "dgswfp::help::internal_error", "cause": "internal error: recognized subcommand could not be matched on name?"}}"#); + (my_command, false) + } else { + (&mut top_level_command, true) + }; + + let args = command + .get_arguments() + .map(OwnedSubcommandOptionsHelpOutput::from) + .collect::>(); + let aliases = command + .get_all_aliases() + .map(ToOwned::to_owned) + .collect::>(); + let command_name = command.get_name().to_owned(); + let help = format!("{}", command.render_long_help()); + let options = command + .get_opts() + .map(OwnedSubcommandOptionsHelpOutput::from) + .collect::>(); + let positionals = command + .get_positionals() + .map(OwnedSubcommandOptionsHelpOutput::from) + .collect::>(); + let subcommands = command + .get_subcommands_mut() + .map(SubcommandHelpOutput::from) + .collect::>(); + + info!( + id = if is_top_level { + "dgswfp::help::top_level".to_owned() + } else { + format!("dgswfp::help::{command_name}") + }, + help.args = valuable(&args), + help.aliases = valuable(&aliases), + help.display_help_text = help, + help.options = valuable(&options), + help.positionals = valuable(&positionals), + help.name = command_name, + help.sub_commands = valuable(&subcommands), + ); +} + +#[derive(Debug, Valuable)] +struct SubcommandHelpOutput<'data> { + pub aliases: Vec<&'data str>, + pub args: Vec>, + pub has_subcommands: bool, + pub help_output: String, + pub name: &'data str, + pub options: Vec>, + pub positionals: Vec>, +} +impl<'data> From<&'data mut Command> for SubcommandHelpOutput<'data> { + fn from(value: &'data mut Command) -> SubcommandHelpOutput<'data> { + let help_output = format!("{}", value.render_help()); + let aliases = value.get_all_aliases().collect::>(); + let has_subcommands = value.has_subcommands(); + let name = value.get_name(); + let args = value + .get_arguments() + .map(SubcommandOptionsHelpOutput::from) + .collect::>(); + let options = value + .get_opts() + .map(SubcommandOptionsHelpOutput::from) + .collect::>(); + let positionals = value + .get_positionals() + .map(SubcommandOptionsHelpOutput::from) + .collect::>(); + + Self { + aliases, + args, + has_subcommands, + help_output, + name, + options, + positionals, + } + } +} + +#[derive(Debug, Valuable)] +struct SubcommandOptionsHelpOutput<'data> { + pub aliases: Option>, + pub default_values: Vec, + pub long_flag_name: Option<&'data str>, + pub short_aliases: Option>, + pub short_flag_name: Option, + pub option_help: Option, +} +impl<'data> From<&'data Arg> for SubcommandOptionsHelpOutput<'data> { + fn from(option: &'data Arg) -> SubcommandOptionsHelpOutput<'data> { + Self { + aliases: option.get_all_aliases(), + default_values: option + .get_default_values() + .iter() + .map(|dv| format!("{}", dv.to_string_lossy())) + .collect(), + long_flag_name: option.get_long(), + short_aliases: option.get_all_short_aliases(), + short_flag_name: option.get_short(), + option_help: option.get_long_help().map(|sstr| format!("{sstr}")), + } + } +} + +#[derive(Debug, Valuable)] +struct OwnedSubcommandOptionsHelpOutput { + pub aliases: Option>, + pub default_values: Vec, + pub long_flag_name: Option, + pub short_aliases: Option>, + pub short_flag_name: Option, + pub option_help: Option, +} +impl<'data> From<&'data Arg> for OwnedSubcommandOptionsHelpOutput { + fn from(option: &'data Arg) -> OwnedSubcommandOptionsHelpOutput { + Self { + aliases: option + .get_all_aliases() + .map(|value| value.into_iter().map(ToOwned::to_owned).collect::>()), + default_values: option + .get_default_values() + .iter() + .map(|dv| format!("{}", dv.to_string_lossy())) + .collect(), + long_flag_name: option.get_long().map(ToOwned::to_owned), + short_aliases: option.get_all_short_aliases(), + short_flag_name: option.get_short(), + option_help: option.get_long_help().map(|sstr| format!("{sstr}")), + } + } +} diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/commands/mod.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/mod.rs new file mode 100644 index 0000000..88af521 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/mod.rs @@ -0,0 +1,11 @@ +//! A thin module wrapper that contains all the different files that each +//! handle one command. + +mod help; +mod padlog; +mod sata_wal; +pub mod utils; + +pub use help::*; +pub use padlog::*; +pub use sata_wal::*; diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/commands/padlog.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/padlog.rs new file mode 100644 index 0000000..9d33f95 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/padlog.rs @@ -0,0 +1,600 @@ +//! Generate a "PADLOG". +//! +//! A PADLOG is built to identify read-file paddings when dealing with a single +//! stream of inputs, like talking to a real cat-dev. For multiple sessions you +//! would need to use something like `SessionManager` to have multiple cat-dev's +//! on multiple unique ports. + +use crate::{ + SHOULD_LOG_JSON, + commands::utils::{PacketOnPort, PacketsWithDataOnPort, validate_pcap_path_constraints}, + exit_codes::{ + PADLOG_CANT_CREATE_LOG, PADLOG_FLUSH_FAILURE, PADLOG_NAGLE_FAILURE, PADLOG_WRITE_FAILURE, + }, + utils::add_context_to, +}; +use bytes::{Buf, Bytes, BytesMut}; +use cat_dev::{ + fsemul::pcfs::sata::{ + proto::{ + SataFileDescriptorResult, SataOpenFilePacketBody, SataPingPacketBody, + SataReadFilePacketBody, SataRequest, SataResponse, SataWriteFilePacketBody, + }, + server::connection_flags::SataConnectionFlags, + }, + net::models::{Endianness, NagleGuard}, +}; +use fnv::FnvHashMap; +use miette::miette; +use std::{ + cmp::Ordering as CompareOrdering, + collections::{VecDeque, hash_map::Entry}, + path::Path, +}; +use tokio::{ + fs::File, + io::{AsyncWriteExt, BufWriter, Error as AsyncIOError}, +}; +use tracing::{Instrument, error, error_span, info}; + +/// Handle generating a WAL Log from a particular pcap. +pub async fn handle_padlog(pcap_path: &Path, log_path: &Path, sata_port: u16) { + let final_path = validate_pcap_path_constraints(pcap_path); + let writer = get_log_writer(log_path).await; + let iterator = PacketsWithDataOnPort::new(&final_path, sata_port); + + process_packets(iterator, writer) + .instrument(error_span!( + "dgswfp::command::padlog::process_packets", + pcap.path = final_path, + sata.port = sata_port, + )) + .await; +} + +#[allow( + // TODO(mythra): fix + clippy::too_many_lines, + clippy::type_complexity, +)] +async fn process_packets(packet_stream: PacketsWithDataOnPort, mut writer: BufWriter) { + let mut stream_buffers: FnvHashMap< + u64, + ( + SataConnectionFlags, + (BytesMut, usize), + (VecDeque, Option<(i32, BytesMut, usize, usize)>), + ), + > = FnvHashMap::default(); + let mut fd_map: FnvHashMap = FnvHashMap::default(); + + for pkt in packet_stream { + if let Entry::Vacant(entry) = stream_buffers.entry(pkt.stream_id()) { + entry.insert(( + // Start as (false, false) as it's the only ping we can't + // really auto detect. + // + // Every other PCFS implementation will start with a ping + // which if any flags are registered wil be picked up. + SataConnectionFlags::new_with_flags(false, false), + (BytesMut::new(), 0_usize), + (VecDeque::new(), None), + )); + } + + let (cflags, cache, (cached_file_name, response_buff)) = stream_buffers + .get_mut(&pkt.stream_id()) + .expect("impossible: always created"); + do_packet_processing( + pkt, + &mut fd_map, + cflags, + cache, + cached_file_name, + response_buff, + &mut writer, + ) + .await; + } + + for (_sid, (_cflags, _cache, (_cached_file_name, mut response_cache))) in stream_buffers { + if let Some((fd, cached_read_file, expected_size, flags_size)) = response_cache.take() { + let mut final_rf = cached_read_file.freeze(); + // 32 bytes read file header + // 4 bytes file len + let read_file_header = final_rf.split_to(36); + + match final_rf.len().cmp(&expected_size) { + CompareOrdering::Greater => { + check_write( + writer + .write_all( + format!( + "OVER({}): {} [{}]", + flags_size < expected_size, + final_rf.len() - expected_size, + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } + CompareOrdering::Less => { + check_write( + writer + .write_all( + format!( + "UNDR({}): {} | {} [{}]", + flags_size < expected_size, + expected_size - final_rf.len(), + if final_rf.len() > flags_size { + format!("+{}", final_rf.len() - flags_size) + } else { + format!("-{}", flags_size - final_rf.len()) + }, + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } + CompareOrdering::Equal => { + let mut padded_bytes = 0_usize; + for byte in final_rf.iter().rev() { + if *byte == 0xCD { + padded_bytes += 1; + } else { + break; + } + } + + if padded_bytes == 0 { + check_write( + writer + .write_all( + format!( + "UPAD({}): {} [{}]", + flags_size < expected_size, + final_rf.len(), + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } else { + check_write( + writer + .write_all( + format!( + "PADD({}): {}/{} [{}]", + flags_size < expected_size, + final_rf.len() - padded_bytes, + padded_bytes, + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } + } + } + + if read_file_header.starts_with(&[0xC4, 0x00, 0x24, 0x02, 0xE8, 0xEF, 0x24, 0x02]) { + check_write(writer.write_all(b" -> C4 Known Header").await); + } else if read_file_header.starts_with(&[0; 8]) { + check_write(writer.write_all(b" -> 00 Known Header").await); + } else { + check_write( + writer + .write_all(b" -> ?? Unknown Header: {read_file_header:02x?}") + .await, + ); + } + } + } + + do_flush(&mut writer).await; +} + +#[allow( + // TODO(mythra): fix + clippy::too_many_lines, + clippy::type_complexity, +)] +async fn do_packet_processing( + pkt: PacketOnPort, + fd_map: &mut FnvHashMap, + cflags: &SataConnectionFlags, + req_nagle_cache: &mut (BytesMut, usize), + cached_file_names: &mut VecDeque, + response_cache: &mut Option<(i32, BytesMut, usize, usize)>, + writer: &mut BufWriter, +) { + if !pkt.is_request() { + if let Some(to_extend) = response_cache.as_mut() { + // Part of a read file response. + to_extend.1.extend(pkt.data()); + } else if let Some(file_name) = cached_file_names.pop_front() { + let Ok(response) = + SataResponse::::try_from(pkt.data().clone()) + else { + return; + }; + + if let Ok(fd) = response.body().result() { + fd_map.insert(fd, file_name); + } + } + + return; + } + + req_nagle_cache.0.extend(pkt.data()); + if let Some((fd, cached_read_file, expected_size, flags_size)) = response_cache.take() { + let mut final_rf = cached_read_file.freeze(); + // 32 bytes read file header + // 4 bytes file len + let read_file_header = final_rf.split_to(36); + + match final_rf.len().cmp(&expected_size) { + CompareOrdering::Greater => { + check_write( + writer + .write_all( + format!( + "OVER({}): {} [{}]", + flags_size < expected_size, + final_rf.len() - expected_size, + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } + CompareOrdering::Less => { + check_write( + writer + .write_all( + format!( + "UNDR({}): {} | {} [{}]", + flags_size < expected_size, + expected_size - final_rf.len(), + if final_rf.len() > flags_size { + format!("+{}", final_rf.len() - flags_size) + } else { + format!("-{}", flags_size - final_rf.len()) + }, + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } + CompareOrdering::Equal => { + let mut padded_bytes = 0_usize; + for byte in final_rf.iter().rev() { + if *byte == 0xCD { + padded_bytes += 1; + } else { + break; + } + } + + if padded_bytes == 0 { + check_write( + writer + .write_all( + format!( + "UPAD({}): {} [{}]", + flags_size < expected_size, + final_rf.len(), + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } else { + check_write( + writer + .write_all( + format!( + "PADD({}): {}/{} [{}]", + flags_size < expected_size, + final_rf.len() - padded_bytes, + padded_bytes, + fd_map.get(&fd).cloned().unwrap_or_default(), + ) + .as_bytes(), + ) + .await, + ); + } + } + } + + if read_file_header.starts_with(&[0xC4, 0x00, 0x24, 0x02, 0xE8, 0xEF, 0x24, 0x02]) { + check_write(writer.write_all(b" -> C4 Known Header").await); + } else if read_file_header.starts_with(&[0; 8]) { + check_write(writer.write_all(b" -> 00 Known Header").await); + } else { + check_write( + writer + .write_all(b" -> ?? Unknown Header: {read_file_header:02x?}") + .await, + ); + } + } + + let nagle_guard = NagleGuard::U32LengthPrefixed(Endianness::Big, Some(0x20)); + 'state_loop: loop { + if req_nagle_cache.1 > 0 { + if req_nagle_cache.0.len() < req_nagle_cache.1 { + break; + } + + (&mut req_nagle_cache.0).take(req_nagle_cache.1); + req_nagle_cache.1 = 0; + } else { + while let Ok(Some((start_of_packet, end_of_packet))) = + nagle_guard.split(&req_nagle_cache.0) + { + let remaining_buff = req_nagle_cache.0.split_off(end_of_packet); + let _start_of_buff = req_nagle_cache.0.split_to(start_of_packet); + let body = req_nagle_cache.0.clone().freeze(); + req_nagle_cache.0 = remaining_buff; + + if let Some(skip_amount) = + process_request(cflags, cached_file_names, response_cache, &body) + { + req_nagle_cache.1 = skip_amount; + continue 'state_loop; + } + } + } + break; + } +} + +#[allow( + // TODO(mythra): fix + clippy::too_many_lines, +)] +fn process_request( + cflags: &SataConnectionFlags, + cached_file_names: &mut VecDeque, + response_cache: &mut Option<(i32, BytesMut, usize, usize)>, + packet: &Bytes, +) -> Option { + // Can't sniff a command type out of this... + if packet.len() < 0x34 { + return None; + } + + let command_type = u32::from_be_bytes([packet[0x30], packet[0x31], packet[0x32], packet[0x33]]); + if command_type == 5 { + let parsed_data = match SataRequest::::try_from(packet.clone()) { + Ok(d) => d, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::open_file_parse_failure", + packet = format!("{packet:02x?}"), + "Failed to parse open file request, so cannot determine correct nagle length!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!( + "Failed to parse open file request, so cannot determine correct nagle length!" + ), + [cause.into()].into_iter(), + ), + ); + } + + std::process::exit(PADLOG_NAGLE_FAILURE); + } + }; + + cached_file_names.push_back(parsed_data.body().path().to_owned()); + } else if command_type == 6 { + // We want to determine the padding of this. + assert!( + !response_cache.is_some(), + "TODO(mythra): double response cache" + ); + let parsed_data = match SataRequest::::try_from(packet.clone()) { + Ok(d) => d, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::read_file_parse_failure", + packet = format!("{packet:02x?}"), + "Failed to parse read file request, so cannot determine correct nagle length!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!( + "Failed to parse read file request, so cannot determine correct nagle length!" + ), + [cause.into()].into_iter(), + ), + ); + } + + std::process::exit(PADLOG_NAGLE_FAILURE); + } + }; + + let total_size = parsed_data.body().block_size() * parsed_data.body().block_count(); + + _ = response_cache.insert(( + parsed_data.body().file_descriptor(), + BytesMut::new(), + usize::try_from(total_size).unwrap_or(usize::MAX), + usize::try_from(cflags.first_read_size()).unwrap_or(usize::MAX), + )); + } else if command_type == 7 { + // This is a write file which will break our normal nagle logic. + // Pre-calculate how many bytes we'll send and mark it. + let parsed_data = match SataRequest::::try_from(packet.clone()) { + Ok(d) => d, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::write_file_parse_failure", + packet = format!("{packet:02x?}"), + "Failed to parse write file request, so cannot determine correct nagle length!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!( + "Failed to parse write file request, so cannot determine correct nagle length!" + ), + [cause.into()].into_iter(), + ), + ); + } + + std::process::exit(PADLOG_NAGLE_FAILURE); + } + }; + + return Some( + usize::try_from(parsed_data.body().block_size()).unwrap_or(usize::MAX) + * usize::try_from(parsed_data.body().block_count()).unwrap_or(usize::MAX), + ); + } else if command_type == 0x14 { + // This is a ping record first read file size / write file size + let parsed_data = match SataRequest::::try_from(packet.clone()) { + Ok(d) => d, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::ping_parse_failure", + packet = format!("{packet:02x?}"), + "Failed to parse ping request, so cannot determine correct nagle length!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!( + "Failed to parse ping request, so cannot determine correct nagle length!" + ), + [cause.into()].into_iter(), + ), + ); + } + + std::process::exit(PADLOG_NAGLE_FAILURE); + } + }; + + info!( + read_size = parsed_data.command_info().user().0, + write_size = parsed_data.command_info().user().1, + "Updating Connection Read/Write Size", + ); + cflags.set_first_read_size(parsed_data.command_info().user().0); + cflags.set_first_write_size(parsed_data.command_info().user().1); + } + + None +} + +/// Create a buffered writer to a particular log file. +async fn get_log_writer(log_path: &Path) -> BufWriter { + let file = match File::create_new(log_path).await { + Ok(fd) => fd, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::padlog::cannot_open_log", + log.path = %log_path.display(), + "Could not create destination log file!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("could not create destination log file"), + [ + miette!("{cause:?}"), + miette!( + "Please ensure the LOG location you specified is correct, and does not exist: {}", + log_path.display(), + ), + ] + .into_iter(), + ), + ); + } + + std::process::exit(PADLOG_CANT_CREATE_LOG); + } + }; + + BufWriter::new(file) +} + +async fn do_flush(writer: &mut BufWriter) { + if let Err(cause) = writer.flush().await { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::padlog::cannot_flush_log_file", + "Could not write all data to our log file, may be corrupt!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("could not write all data to log file, may be corrupt"), + [miette!("{cause:?}"),].into_iter(), + ), + ); + } + + std::process::exit(PADLOG_FLUSH_FAILURE); + } +} + +fn check_write(result: Result<(), AsyncIOError>) { + if let Err(cause) = result { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::padlog::cannot_write_to_buffer", + "Could not write data to our in memory buffer to later flush to a file, OOM?", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("We couldn't write data to our buffered writer, to then flush? OOM?"), + [miette!("{cause:?}"),].into_iter(), + ), + ); + } + + std::process::exit(PADLOG_WRITE_FAILURE); + } +} diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/commands/sata_wal.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/sata_wal.rs new file mode 100644 index 0000000..bc80a9d --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/sata_wal.rs @@ -0,0 +1,458 @@ +//! Handles generation of a SATA WAL log from a PCAPNG. + +use crate::{ + SHOULD_LOG_JSON, + commands::utils::{PacketOnPort, PacketsWithDataOnPort, validate_pcap_path_constraints}, + exit_codes::{GENERATE_CANT_CREATE_WAL, GENERATE_NAGLE_FAILURE}, + utils::add_context_to, +}; +use bytes::{Bytes, BytesMut}; +use cat_dev::{ + fsemul::pcfs::sata::{ + proto::{SataPingPacketBody, SataReadFilePacketBody, SataRequest, SataWriteFilePacketBody}, + server::{connection_flags::SataConnectionFlags, wal::WriteAheadLog}, + }, + net::models::{Endianness, NagleGuard}, +}; +use fnv::FnvHashMap; +use miette::miette; +use std::{ + collections::hash_map::Entry, + path::{Path, PathBuf}, + time::Duration, +}; +use tokio::time::sleep; +use tracing::{Instrument, debug, error, error_span, info}; + +/// Handle generating a WAL Log from a particular pcap. +pub async fn handle_generate(pcap_path: PathBuf, wal_path: PathBuf, sata_port: u16) { + let final_path = validate_pcap_path_constraints(&pcap_path); + let stream = PacketsWithDataOnPort::new(&final_path, sata_port); + let wal = get_wal(&wal_path); + + process_packets(stream, wal) + .instrument(error_span!( + "dgswfp::command::sata_wal::process_packets", + pcap.path = final_path, + sata.port = sata_port, + )) + .await; +} + +#[allow( + // TODO(mythra): fix + clippy::type_complexity, + clippy::too_many_lines, +)] +async fn process_packets(packet_stream: PacketsWithDataOnPort, wal: WriteAheadLog) { + let mut stream_buffers: FnvHashMap< + u64, + ( + SataConnectionFlags, + (SataStreamState, BytesMut), + (SataStreamState, BytesMut), + ), + > = FnvHashMap::default(); + + for pkt in packet_stream { + if let Entry::Vacant(entry) = stream_buffers.entry(pkt.stream_id()) { + wal.record_open_stream(pkt.stream_id()).await; + entry.insert(( + // Start as (false, false) as it's the only ping we can't + // really auto detect. + // + // Every other PCFS implementation will start with a ping + // which if any flags are registered wil be picked up. + SataConnectionFlags::new_with_flags(false, false), + (SataStreamState::ProcessingPackets, BytesMut::new()), + (SataStreamState::ProcessingPackets, BytesMut::new()), + )); + } + + let ref_buffers = stream_buffers + .get_mut(&pkt.stream_id()) + .expect("impossible: always created"); + let (flags, cache, state, other_state) = if pkt.is_request() { + let mut_borrow: &mut ( + SataConnectionFlags, + (SataStreamState, BytesMut), + (SataStreamState, BytesMut), + ) = ref_buffers; + + ( + &mut_borrow.0, + &mut mut_borrow.1.1, + &mut mut_borrow.1.0, + &mut mut_borrow.2.0, + ) + } else { + let mut_borrow: &mut ( + SataConnectionFlags, + (SataStreamState, BytesMut), + (SataStreamState, BytesMut), + ) = ref_buffers; + + ( + &mut_borrow.0, + &mut mut_borrow.2.1, + &mut mut_borrow.2.0, + &mut mut_borrow.1.0, + ) + }; + + do_packet_processing(pkt, cache, flags, state, other_state, &wal).await; + } + + info!( + id = "dgswfp::generate::flush_notice", + "Packets processed... flushing WAL" + ); + // Ensure flush happens by ensuring enough seconds have passed that a close + // stream will trigger a flush. + sleep(Duration::from_secs(3)).await; + for sid in stream_buffers.keys() { + wal.record_close_stream(*sid).await; + } +} + +async fn do_packet_processing( + pkt: PacketOnPort, + nagle_cache: &mut BytesMut, + cflags: &SataConnectionFlags, + state: &mut SataStreamState, + other_state: &mut SataStreamState, + wal: &WriteAheadLog, +) { + // Always add ourselves to whatever cache we already have. + // We'll split below... + nagle_cache.extend(pkt.data()); + + let nagle_guard = NagleGuard::U32LengthPrefixed(Endianness::Big, Some(0x20)); + 'stream_state_loop: loop { + if let SataStreamState::NeedsAtLeast(fd, needed) = state { + if nagle_cache.len() < *needed { + // Sniff out read file errors... + if !pkt.is_request() && nagle_cache.starts_with(&[0xC4_u8, 0x00, 0xFE, 0x00, 0x20]) + { + *needed = 8 + 24 + 4; + } + + if nagle_cache.len() < *needed { + debug!("waiting for more data in NeedsAtLeast...."); + return; + } + } + let next_item = nagle_cache.split_to(*needed).freeze(); + + if pkt.is_request() { + wal.record_oob_file_write_read(pkt.stream_id(), *fd, *needed) + .await; + } else { + wal.record_response(pkt.stream_id(), next_item).await; + } + *state = SataStreamState::ProcessingPackets; + continue; + } + if let SataStreamState::NeedsAtLeastCheckAt(fd, needed_at) = state { + // Sniff out read file errors... + if !pkt.is_request() && nagle_cache.starts_with(&[0xC4_u8, 0x00, 0xFE, 0x00, 0x20]) { + let needed = 8 + 24 + 4; + if nagle_cache.len() < needed { + debug!("waiting for more data in error'd NeedsAtLeastCheckAt"); + return; + } + } + + if nagle_cache.len() < *needed_at + 4 { + debug!("waiting for more data for length check in NeedsAtLeastCheckAt...."); + return; + } + let read_bytes_or_file_size = u32::from_be_bytes([ + nagle_cache[*needed_at], + nagle_cache[*needed_at + 1], + nagle_cache[*needed_at + 2], + nagle_cache[*needed_at + 3], + ]); + let read_bytes_or_file_size_size = + usize::try_from(read_bytes_or_file_size).unwrap_or(usize::MAX); + + // Okay, so we've got a file length, or read length. So we now need to + // check. Are we going to do padding? We know to hit this branch... + // you _MUST_ have total_read_size > read_file.len(). So now, we need + // to check the _other_ condition for padding. Is + // file_size < read_file.len(). + if read_bytes_or_file_size < cflags.first_read_size() { + // Okay so we're large enough that we got into this state, but we're not + // large enough to ignore padding. So switch our state, and pad. + if pkt.is_request() { + unreachable!(); + } else { + *state = SataStreamState::NeedsAtLeast( + *fd, + 0x20 + 0x4 + + usize::try_from(cflags.first_read_size()).unwrap_or(usize::MAX), + ); + } + + continue; + } + + if nagle_cache.len() < *needed_at + 4 + read_bytes_or_file_size_size { + debug!("waiting for more data in NeedsAtLeastCheckAt...."); + return; + } + + let next_item = nagle_cache + .split_to(*needed_at + 4 + read_bytes_or_file_size_size) + .freeze(); + if pkt.is_request() { + // This branch is only taken in read file, not write file + unreachable!(); + } else { + wal.record_response(pkt.stream_id(), next_item).await; + } + *state = SataStreamState::ProcessingPackets; + continue; + } + + while let Ok(Some((start_of_packet, end_of_packet))) = nagle_guard.split(nagle_cache) { + let remaining_buff = nagle_cache.split_off(end_of_packet); + let _start_of_buff = nagle_cache.split_to(start_of_packet); + let body = nagle_cache.clone().freeze(); + *nagle_cache = remaining_buff; + + let re_loop = if pkt.is_request() { + process_request(pkt.stream_id(), body, cflags, state, other_state, wal).await + } else { + process_response(pkt.stream_id(), body, cflags, state, other_state, wal).await + }; + + if re_loop { + continue 'stream_state_loop; + } + } + break; + } +} + +#[allow( + // TODO(mythra): fix + clippy::too_many_lines, +)] +async fn process_request( + stream_id: u64, + packet: Bytes, + cflags: &SataConnectionFlags, + state: &mut SataStreamState, + response_state: &mut SataStreamState, + wal: &WriteAheadLog, +) -> bool { + wal.record_request(stream_id, packet.clone()).await; + // Can't sniff a command type out of this... + if packet.len() < 0x34 { + return false; + } + + let command_type = u32::from_be_bytes([packet[0x30], packet[0x31], packet[0x32], packet[0x33]]); + if command_type == 6 && cflags.ffio_enabled() { + // This is a read file, which will break normal nagle logic. + // Let's pre-calculate the size the server should send, and + // mark it. + let parsed_data = match SataRequest::::try_from(packet.clone()) { + Ok(d) => d, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::read_file_parse_failure", + packet = format!("{packet:02x?}"), + "Failed to parse read file request, so cannot determine correct nagle length!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!( + "Failed to parse read file request, so cannot determine correct nagle length!" + ), + [cause.into()].into_iter(), + ), + ); + } + + std::process::exit(GENERATE_NAGLE_FAILURE); + } + }; + + let total_size = parsed_data.body().block_size() * parsed_data.body().block_count(); + if total_size > cflags.first_read_size() { + *response_state = SataStreamState::NeedsAtLeastCheckAt( + parsed_data.body().file_descriptor(), + 0x20_usize, + ); + } else { + *response_state = SataStreamState::NeedsAtLeast( + parsed_data.body().file_descriptor(), + 0x20_usize + + 0x4_usize + (usize::try_from(parsed_data.body().block_size()) + .unwrap_or(usize::MAX) + * usize::try_from(parsed_data.body().block_count()).unwrap_or(usize::MAX)), + ); + } + } else if command_type == 7 { + // This is a write file which will break our normal nagle logic. + // Pre-calculate how many bytes we'll send and mark it. + let parsed_data = match SataRequest::::try_from(packet.clone()) { + Ok(d) => d, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::write_file_parse_failure", + packet = format!("{packet:02x?}"), + "Failed to parse write file request, so cannot determine correct nagle length!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!( + "Failed to parse write file request, so cannot determine correct nagle length!" + ), + [cause.into()].into_iter(), + ), + ); + } + + std::process::exit(GENERATE_NAGLE_FAILURE); + } + }; + + *state = SataStreamState::NeedsAtLeast( + parsed_data.body().file_descriptor(), + usize::try_from(parsed_data.body().block_size()).unwrap_or(usize::MAX) + * usize::try_from(parsed_data.body().block_count()).unwrap_or(usize::MAX), + ); + // We modified our state, we need to loop! + return true; + } else if command_type == 0x14 { + // This is a ping record first read file size / write file size + let parsed_data = match SataRequest::::try_from(packet.clone()) { + Ok(d) => d, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::ping_parse_failure", + packet = format!("{packet:02x?}"), + "Failed to parse ping request, so cannot determine correct nagle length!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!( + "Failed to parse ping request, so cannot determine correct nagle length!" + ), + [cause.into()].into_iter(), + ), + ); + } + + std::process::exit(GENERATE_NAGLE_FAILURE); + } + }; + + info!( + read_size = parsed_data.command_info().user().0, + write_size = parsed_data.command_info().user().1, + "Updating Connection Read/Write Size", + ); + cflags.set_first_read_size(parsed_data.command_info().user().0); + cflags.set_first_write_size(parsed_data.command_info().user().1); + } + + false +} + +async fn process_response( + stream_id: u64, + packet: Bytes, + cflags: &SataConnectionFlags, + _state: &mut SataStreamState, + _request_state: &mut SataStreamState, + wal: &WriteAheadLog, +) -> bool { + // Try to sniff out pings... + if packet.len() == 0x28 { + if packet.ends_with(&0xCAFE_0003_u32.to_be_bytes()) { + cflags.set_csr_enabled(true); + cflags.set_ffio_enabled(true); + } else if packet.ends_with(&0xCAFE_0002_u32.to_be_bytes()) { + cflags.set_csr_enabled(true); + cflags.set_ffio_enabled(false); + } else if packet.ends_with(&0xCAFE_0001_u32.to_be_bytes()) { + cflags.set_csr_enabled(false); + cflags.set_ffio_enabled(true); + } + } + + wal.record_response(stream_id, packet.clone()).await; + + false +} + +/// Get a reference to a live write ahead log. +fn get_wal(wal_path: &Path) -> WriteAheadLog { + match WriteAheadLog::new(wal_path.to_path_buf()) { + Ok(wal) => wal, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + ?cause, + id = "dgswfp::generate::cannot_open_wal", + wal.path = %wal_path.display(), + "Failed to create a WAL to write!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("cannot open and generate wal"), + [ + cause.into(), + miette!( + "The wal path we would've written to is: {}", + wal_path.display(), + ), + ] + .into_iter(), + ), + ); + } + + std::process::exit(GENERATE_CANT_CREATE_WAL); + } + } +} + +enum SataStreamState { + /// The 'default' state, just reading packets. + ProcessingPackets, + /// We requested a file read, and need to break NAGLE logic + /// to read a file of N bytes for our response. + /// + /// IF this is a request side of the stream in this state it means + /// we are in write file, and should read N extra bytes and call register oob + /// write extra read. + /// + /// IF this is a response side of the stream, it means we need to read N bytes + /// rather than the normal NAGLE length, and call that as one 'response' packet. + NeedsAtLeast(i32, usize), + /// We requested a file read, but one that allows for truncating a response rather + /// than padding it out. + /// + /// In this case we need to actually read the repsonse to know how many bytes to + /// read. it'll be at usize location in the packet. + NeedsAtLeastCheckAt(i32, usize), +} diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/commands/utils.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/utils.rs new file mode 100644 index 0000000..236b939 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/commands/utils.rs @@ -0,0 +1,258 @@ +//! Utilities that are common across multiple commands. + +use crate::{ + SHOULD_LOG_JSON, + exit_codes::{ + ARGV_PCAP_DOES_NOT_EXIST, ARGV_PCAP_PATH_NOT_UTF8, UTILS_CANNOT_SPAWN_TSHARK, + UTILS_TSHARK_READ_FAILURE, + }, + utils::add_context_to, +}; +use bytes::{BufMut, Bytes, BytesMut}; +use miette::miette; +use rtshark::{Metadata as RtMetadata, RTShark, RTSharkBuilder}; +use std::{iter::Iterator, path::Path}; +use tracing::{debug, error}; + +/// Validate that the PCAP file exists, and is stored on a UTF-8 path. +pub fn validate_pcap_path_constraints(pcap_path: &Path) -> String { + if !pcap_path.exists() || !pcap_path.is_file() { + if SHOULD_LOG_JSON() { + error!( + id = "dgswfp::utils::no_source_pcap", + pcap.path = %pcap_path.display(), + "Source PCAP is not an existing file, cannot parse!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("cannot parse a PCAP that is not an existing file"), + [miette!( + "Please ensure the PCAP location you specified is correct: {}", + pcap_path.display(), + )] + .into_iter(), + ), + ); + } + + std::process::exit(ARGV_PCAP_DOES_NOT_EXIST); + } + + let Some(existing_path) = pcap_path.to_str() else { + if SHOULD_LOG_JSON() { + error!( + id = "dgswfp::utils::pcap_path_not_utf8", + pcap.path = %pcap_path.display(), + "Source PCAP path must be representable as a UTF-8 string!", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("cannot parse PCAP whose path is not fully UTF-8!"), + [miette!( + "Please move the PCAP file into a UTF-8 compatible path: {}", + pcap_path.display(), + )] + .into_iter(), + ), + ); + } + + std::process::exit(ARGV_PCAP_PATH_NOT_UTF8); + }; + + existing_path.to_owned() +} + +/// A stream over all the packets with data happening on a particular port. +/// +/// This for now is just a thin wrapper around an rtshark iterator, and stream +/// that will filter out any packet that is not on the correct port. Then it +/// will map and return just the actual payload along with whether it's +/// incoming, or outgoing. +pub struct PacketsWithDataOnPort { + /// If we've already finished our stream. + finished: bool, + /// The port to filter data on. + port: u16, + /// The underlying command we will end up iterating ontop of. + underlying_command: RTShark, +} + +impl PacketsWithDataOnPort { + /// Create a new iterator over all the packets coming into/going out of a + /// port. + #[must_use] + pub fn new(pcap: &str, port: u16) -> Self { + let underlying_command = match RTSharkBuilder::builder() + .input_path(pcap) + .disable_protocol("ALL") + .enable_protocol("eth") + .enable_protocol("ip") + .enable_protocol("tcp") + .display_filter(&format!("tcp.port == {port} && data.len > 0")) + .spawn() + { + Ok(builder) => builder, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + id = "dgswfp::utils::pcap_spawn_failure", + pcap.path = pcap, + "failed to spawn tshark, and read from PCAP/PCAPNG.", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("Failed to spawn TSHARK, and read from PCAP/PCAPNG."), + [miette!("{cause:?}")].into_iter(), + ), + ); + } + + std::process::exit(UTILS_CANNOT_SPAWN_TSHARK); + } + }; + + Self { + finished: false, + port, + underlying_command, + } + } +} + +impl Iterator for PacketsWithDataOnPort { + type Item = PacketOnPort; + + fn next(&mut self) -> Option { + if self.finished { + return None; + } + + loop { + match self.underlying_command.read() { + Ok(Some(pkt)) => { + // Just cause we have a packet doesn't mean it can be used... + let Some(tcp_layer) = pkt.layer_name("tcp") else { + debug!("packet is missing TCP layer! skipping..."); + continue; + }; + + let Some(srcport) = tcp_layer + .metadata("tcp.srcport") + .and_then(|val| val.value().parse::().ok()) + else { + debug!("packet is missing `tcp.srcport`"); + continue; + }; + let Some(dstport) = tcp_layer + .metadata("tcp.dstport") + .and_then(|val| val.value().parse::().ok()) + else { + debug!("packet is missing `tcp.dstport`"); + continue; + }; + + if srcport != self.port && dstport != self.port { + debug!("packet is not incoming to the correct port"); + continue; + } + let is_request = dstport == self.port; + + let Some(sid) = tcp_layer + .metadata("tcp.stream") + .and_then(|val| val.value().parse::().ok()) + else { + debug!("packet is missing `tcp.stream`"); + continue; + }; + let Some(payload) = tcp_layer + .metadata("tcp.payload") + .map(RtMetadata::raw_value) + .map(string_to_hex_bytes) + else { + debug!("packet is missing `tcp.payload`"); + continue; + }; + + return Some(PacketOnPort::new(payload, is_request, sid)); + } + Ok(None) => { + self.finished = true; + return None; + } + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + id = "dgswfp::utils::pcap_read_failure", + "failed to read from pcap", + ); + } else { + error!( + "\n{:?}", + add_context_to( + miette!("Failed to read data from the PCAP!"), + [miette!("{cause:?}")].into_iter(), + ), + ); + } + + std::process::exit(UTILS_TSHARK_READ_FAILURE); + } + } + } + } +} + +/// An actual packet that has come in on a specific port. +#[derive(Debug)] +pub struct PacketOnPort { + /// The actual data of the packet. + data: Bytes, + /// If the packet was a request (e.g. dstport is equal to port). + is_request: bool, + /// The ID of the stream this packet is on the port. + stream_id: u64, +} + +impl PacketOnPort { + /// Create a new packet representation. + #[must_use] + pub const fn new(data: Bytes, is_request: bool, stream_id: u64) -> Self { + Self { + data, + is_request, + stream_id, + } + } + + #[must_use] + pub const fn data(&self) -> &Bytes { + &self.data + } + #[must_use] + pub const fn is_request(&self) -> bool { + self.is_request + } + #[must_use] + pub const fn stream_id(&self) -> u64 { + self.stream_id + } +} + +fn string_to_hex_bytes(data: &str) -> Bytes { + let mut result = BytesMut::with_capacity(data.len() / 2); + + for chunk in data.chars().collect::>().chunks(2) { + let new_byte = + u8::from_str_radix(&format!("{}{}", chunk[0], chunk[1]), 16).expect("bad byte!"); + result.put_u8(new_byte); + } + + result.freeze() +} diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/exit_codes.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/exit_codes.rs new file mode 100644 index 0000000..8ccc039 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/exit_codes.rs @@ -0,0 +1,22 @@ +//! Just a list of all the exit codes in our process. + +pub const NOT_YET_IMPLEMENTED: i32 = 1; +pub const LOGGING_HANDLER_INSTALL_FAILURE: i32 = 2; +pub const SHOULD_NEVER_HAPPEN_FAILURE: i32 = 3; +pub const FAILED_TO_WRITE_TO_DISK: i32 = 4; + +pub const ARGV_PARSE_FAILURE: i32 = 10; +pub const ARGV_NO_COMMAND_SPECIFIED: i32 = 11; +pub const ARGV_PCAP_DOES_NOT_EXIST: i32 = 12; +pub const ARGV_PCAP_PATH_NOT_UTF8: i32 = 13; + +pub const UTILS_TSHARK_READ_FAILURE: i32 = 20; +pub const UTILS_CANNOT_SPAWN_TSHARK: i32 = 21; + +pub const GENERATE_CANT_CREATE_WAL: i32 = 30; +pub const GENERATE_NAGLE_FAILURE: i32 = 32; + +pub const PADLOG_CANT_CREATE_LOG: i32 = 40; +pub const PADLOG_NAGLE_FAILURE: i32 = 42; +pub const PADLOG_FLUSH_FAILURE: i32 = 43; +pub const PADLOG_WRITE_FAILURE: i32 = 44; diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/cli.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/cli.rs new file mode 100644 index 0000000..36d388e --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/cli.rs @@ -0,0 +1,154 @@ +//! Defines the command line interface a.k.a. all the arguments & flags. + +use clap::Parser; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[clap(disable_help_flag = true, disable_help_subcommand = true)] +#[command( + about, + author, + name = "dbg-generate-sata-wal-from-pcap", + propagate_version = true, + version +)] +pub struct CliArguments { + #[command(subcommand)] + pub commands: Option, + #[arg( + global = true, + short = 'h', + long = "help", + help = "Display the help page for your command rather than running it.", + long_help = "Show the help output for either the top level cli, or a particular subcommand. This will always be prioritized." + )] + pub help: bool, + #[arg( + global = true, + short = 'j', + long = "json", + help = "Ensures all logging comes out in JSON instead of text.", + long_help = "Switch all logging and output to JSON for machine parsable output. NOTE: there is no necissarily guaranteed structure, though we will not break it unnecissarily." + )] + pub json: bool, +} + +#[derive(Parser, Debug)] +#[clap(disable_help_flag = true, disable_help_subcommand = true)] +pub enum Subcommands { + /// Generate a SATA Pad Log. + #[command( + name = "generate-sata-padlog", + visible_aliases = [ + "generate_sata_padlog", + "generate-sata-pad", + "generate_sata_pad", + "gen-sata-pad", + "gen_sata_pad", + "sata-pad", + "sata_pad", + "gsp", + "sp", + ], + )] + GenerateSataPadlog { + #[arg( + short = 'p', + long = "sata-port", + visible_alias = "sata_port", + default_value_t = 7500 + )] + sata_port: u16, + #[arg( + index = 1, + help = "The PCAPNG file to read.", + long_help = "The PCAPNG to generate a SATA WAL log from." + )] + pcap: PathBuf, + #[arg( + index = 2, + help = "The Padlog file to write.", + long_help = "The Padlog file to write too." + )] + padlog: PathBuf, + }, + /// Generate a SATA WAL (Write-Ahead Log). + #[command( + name = "generate-sata-wal", + visible_aliases = [ + "generate_sata_wal", + "gen-sata-wal", + "gen_sata_wal", + "sata-wal", + "sata_wal", + "gsw", + "sw", + ], + )] + GenerateSataWAL { + #[arg( + short = 'p', + long = "sata-port", + visible_alias = "sata_port", + default_value_t = 7500 + )] + sata_port: u16, + #[arg( + index = 1, + help = "The PCAPNG file to read.", + long_help = "The PCAPNG to generate a SATA WAL log from." + )] + pcap: PathBuf, + #[arg( + index = 2, + help = "The WAL file to write.", + long_help = "The location where we should write the final WAL." + )] + wal: PathBuf, + }, + /// An alternative to `-h`, or `--help` to show the help for the top level CLI. + #[command(name = "help")] + Help {}, +} +impl Subcommands { + /// If this subcommand matches a particular name. + #[allow(unused)] + #[must_use] + pub fn name_matches(&self, name: &str) -> bool { + match self { + Self::GenerateSataPadlog { + sata_port, + pcap, + padlog, + } => [ + "generate-sata-padlog", + "generate_sata_padlog", + "generate-sata-pad", + "generate_sata_pad", + "gen-sata-pad", + "gen_sata_pad", + "sata-pad", + "sata_pad", + "gsp", + "sp", + ] + .contains(&name), + Self::GenerateSataWAL { + sata_port, + pcap, + wal, + } => [ + "generate-sata-wal", + "generate_sata_wal", + "gen-sata-wal", + "gen_sata_wal", + "sata-wal", + "sata_wal", + "gsw", + "sw", + ] + .contains(&name), + Self::Help {} => name == "help", + } + } +} diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/env.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/env.rs new file mode 100644 index 0000000..0094f13 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/env.rs @@ -0,0 +1,11 @@ +//! The list of environment variables that influence behavior for `dbg-generate-sata-wal-from-pcap`. + +use std::{env::var as env_var, sync::LazyLock}; + +/// Another way of configuring `dbg-generate-sata-wal-from-pcap` to output it's data in JSON. +/// +/// Environment Variable Name: `DGSWFP_OUTPUT_JSON` +/// Expected Values: ("1" or "0"), and ("true" or "false") +/// Type: Boolean +pub static USE_JSON_OUTPUT: LazyLock = + LazyLock::new(|| env_var("DGSWFP_OUTPUT_JSON").is_ok_and(|var| var == "1" || var == "true")); diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/mod.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/mod.rs new file mode 100644 index 0000000..22df6de --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/knobs/mod.rs @@ -0,0 +1,7 @@ +//! The series of knobs that you can use to configure for `dbg-generate-sata-wal-from-pcap`. +//! +//! NOTE: this doesn't include any flags potentially included in shared +//! libraries like those used for [`log`]. + +pub mod cli; +pub mod env; diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/main.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/main.rs new file mode 100644 index 0000000..88ec191 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/main.rs @@ -0,0 +1,175 @@ +pub mod commands; +pub mod exit_codes; +pub mod knobs; +pub mod utils; + +use crate::{ + commands::{handle_generate, handle_help, handle_padlog}, + exit_codes::{ + ARGV_NO_COMMAND_SPECIFIED, ARGV_PARSE_FAILURE, LOGGING_HANDLER_INSTALL_FAILURE, + SHOULD_NEVER_HAPPEN_FAILURE, + }, + knobs::{ + cli::{CliArguments, Subcommands}, + env::USE_JSON_OUTPUT, + }, + utils::add_context_to, +}; +use clap::{Error as ClapError, Parser, error::ErrorKind as ClapErrorKind}; +use log::install_logging_handlers; +use miette::{IntoDiagnostic, miette}; +use tracing::{error, info}; + +/// Whether or not we're logging in JSON. +static mut USE_JSON: bool = false; + +/// Whether or not we should log in JSON. +/// +/// Wrapper around the "unsafe" static mutable. This is guaranteed to be +/// safe as it's initialized as the very first thing to be used in the +/// program, and guaranteed to not change after that. +#[allow(non_snake_case)] +#[inline] +#[must_use] +pub fn SHOULD_LOG_JSON() -> bool { + unsafe { USE_JSON } +} + +#[tokio::main] +async fn main() { + let (argv, use_json) = bootstrap_cli(); + unsafe { + USE_JSON = use_json; + } + + if argv.help || argv.commands.is_none() || matches!(argv.commands, Some(Subcommands::Help {})) { + let should_error = !argv.help && argv.commands.is_none(); + handle_help(argv.commands); + std::process::exit(if should_error { + ARGV_NO_COMMAND_SPECIFIED + } else { + 0 + }); + } + + let Some(sub_command) = argv.commands else { + if use_json { + error!( + id = "dgswfp::help::internal", + cause = "Didn't call help even when subcommands was none?" + ); + } else { + error!( + "\n{:?}", + miette!( + "internal error: Failed to specify a single command, and didn't call `help` handler?" + ), + ); + } + std::process::exit(SHOULD_NEVER_HAPPEN_FAILURE); + }; + + match sub_command { + Subcommands::GenerateSataWAL { + sata_port, + pcap, + wal, + } => { + handle_generate(pcap, wal, sata_port).await; + } + // Help is handled above. + Subcommands::Help {} => unreachable!(), + Subcommands::GenerateSataPadlog { + sata_port, + pcap, + padlog, + } => { + handle_padlog(&pcap, &padlog, sata_port).await; + } + } +} + +fn bootstrap_cli() -> (CliArguments, bool) { + let args_opt = CliArguments::try_parse(); + + let use_json_cli = args_opt.as_ref().map_or_else( + |_error| { + let mut use_json = false; + + // Try to identify if the user is wanting to use JSON, even when argument + // parsing itself fails. + for arg in std::env::args() { + if arg.as_str() == "-j" || arg.as_str() == "--json" { + use_json = true; + break; + } + } + + use_json + }, + |args| args.json, + ); + let use_json = *USE_JSON_OUTPUT || use_json_cli; + + if let Err(cause) = install_logging_handlers(use_json) { + // We have to use a custom panic script here, because logging isn't setup yet. + if use_json { + println!( + r#"{{"id": "dgswfp::logging::install_failure", "inner_display_error": "{}", "message": "Failed to install the logging handlers!"}}"#, + format!("{cause:?}").replace('"', "\\\"") + ); + } else { + println!("Failed to install the logging handler to setup logging:\n{cause:?}"); + } + std::process::exit(LOGGING_HANDLER_INSTALL_FAILURE); + } + + match args_opt { + Ok(args) => (args, use_json), + Err(cause) => { + if cause.kind() == ClapErrorKind::DisplayVersion { + if use_json { + info!( + id = "dgswfp::cli::print_version", + version = format!( + "{} ({})", + format!("{}", cause.render()).trim(), + option_env!("DGSWFP_BUILD").unwrap_or("unknown") + ), + ); + } else { + info!( + "{}", + format!( + "{} ({})", + format!("{}", cause.render()).trim(), + option_env!("DGSWFP_BUILD").unwrap_or("unknown") + ), + ); + } + + std::process::exit(0); + } + + if use_json { + error!( + id = "dgswfp::cli::arg_parse_failure", + error.kind = %cause.kind(), + error.context = ?cause.context().map(|(kind, value)| format!("{kind}: {value}")).collect::>(), + error.rendered = %cause.render(), + "Failed parsing CLI arguments" + ); + } else { + error!( + "\n{:?}", + add_context_to( + Err::<(), ClapError>(cause).into_diagnostic().unwrap_err(), + [miette!("Failed parsing CLI arguments!")].into_iter(), + ), + ); + } + + std::process::exit(ARGV_PARSE_FAILURE); + } + } +} diff --git a/cmd/dbg-generate-sata-wal-from-pcap/src/utils.rs b/cmd/dbg-generate-sata-wal-from-pcap/src/utils.rs new file mode 100644 index 0000000..57da3a1 --- /dev/null +++ b/cmd/dbg-generate-sata-wal-from-pcap/src/utils.rs @@ -0,0 +1,29 @@ +//! Utility functions that don't have one place that they should live. + +use miette::Report; + +/// Add context to a specific error, where you can have like a list of +/// suggestions. +/// +/// NOTE: we cannot reassign a reports severity, so your last items severity +/// is where the real severity gets taken. +pub fn add_context_to( + original_error: Report, + suggestions: impl DoubleEndedIterator, +) -> Report { + let mut latest_error: Option = None; + + for suggestion in suggestions.rev() { + if let Some(last_error) = latest_error { + latest_error = Some(last_error.wrap_err(suggestion)); + } else { + latest_error = Some(suggestion); + } + } + + if let Some(latest) = latest_error { + latest.wrap_err(original_error) + } else { + original_error + } +} diff --git a/cmd/pcfsserver/Cargo.toml b/cmd/pcfsserver/Cargo.toml new file mode 100644 index 0000000..e098930 --- /dev/null +++ b/cmd/pcfsserver/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "pcfsserver" +description = "A re-implementation of the Cafe SDK's Host Bridge Software tool `PCFSServer.exe`." +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +version.workspace = true +# Is a CLI tool with a potentially common name, people can build it/use our packages with scripts included. +publish = false + +[dependencies] +cat-dev = { default-features = false, features = ["servers"], path = "../../pkg/cat-dev" } +tokio.workspace = true \ No newline at end of file diff --git a/cmd/pcfsserver/README.md b/cmd/pcfsserver/README.md new file mode 100644 index 0000000..9e6bd08 --- /dev/null +++ b/cmd/pcfsserver/README.md @@ -0,0 +1,32 @@ +# `PCFSServer` # + +- [x] **Tool Re-Implementation** +- [ ] **Script** + +***note: PCFSServer is currently not implemented as we work on finalizing +reversing the server schema.*** + +PCFSServer is part of the FSEmulation toolchain that specifically allows +querying paths, creating files/folders, etc. With normal unix style paths, +modes, etc. + +## Building ## + +In order to build you can follow the project instructions, or if you want to +build just this one single package you can use: `cargo build -p pcfsserver` +from the root directory of the project to build a debug version of the +application. It will be available at: `${project-dir}/target/debug/pcfsserver`, +or `${project-dir}/target/debug/pcfsserver.exe` if you are on windows. If you +want to build a release version that is fully optimized you want to use the +command: `cargo b --release -p pcfsserver`. It will be available at: +`${project-dir}/target/release/pcfsserver`, or +`${project-dir}/target/release/pcfsserver.exe` respectively. This project +should be compatible with any Rust version above: `1.63.0`, although it's +always safest to build with whatever the latest version of Rust is at the time. + +## Known Issues ## + +There are several known issues with `pcfsserver` that have been intentionally +preserved for compatability. We describe the workaround for these issues that +you can use to hopefully get the data you want. + diff --git a/cmd/pcfsserver/src/main.rs b/cmd/pcfsserver/src/main.rs new file mode 100644 index 0000000..1dd1c2e --- /dev/null +++ b/cmd/pcfsserver/src/main.rs @@ -0,0 +1,41 @@ +use cat_dev::fsemul::{HostFilesystem, pcfs::sata::server::pcfs_sata_server}; +use std::{net::Ipv4Addr, str::FromStr}; +use tokio::runtime::Runtime; + +fn main() { + let Ok(runtime) = Runtime::new() else { + println!("TODO(mythra): thread pool spin up failure"); + return; + }; + runtime.block_on(serve()); +} + +async fn serve() { + let fs = HostFilesystem::from_cafe_dir(None) + .await + .expect("TODO HOSTFS FAILURE"); + let server = pcfs_sata_server( + fs, + std::env::args() + .next() + .and_then(|ipstr| Ipv4Addr::from_str(&ipstr).ok()), + None, + false, + false, + false, + None, + None, + true, + None, + true, + true, + ) + .await + .expect("TODO failed to spin up server"); + println!( + "TODO mythra debug: server on: {}:{}", + server.ip(), + server.port() + ); + server.bind().await.expect("TODO Failed to serve server"); +} diff --git a/installer-scripts/osx/distribution.arm.xml b/installer-scripts/osx/distribution.arm.xml index d8affe7..235ee9f 100644 --- a/installer-scripts/osx/distribution.arm.xml +++ b/installer-scripts/osx/distribution.arm.xml @@ -12,5 +12,5 @@ - + diff --git a/installer-scripts/osx/distribution.intel.xml b/installer-scripts/osx/distribution.intel.xml index 6cdb86f..08a0c87 100644 --- a/installer-scripts/osx/distribution.intel.xml +++ b/installer-scripts/osx/distribution.intel.xml @@ -12,5 +12,5 @@ - + diff --git a/installer-scripts/osx/package.sh b/installer-scripts/osx/package.sh index e6b81f4..2c11042 100755 --- a/installer-scripts/osx/package.sh +++ b/installer-scripts/osx/package.sh @@ -26,13 +26,15 @@ cp ../../../cmd/getbridgetype/sh/getbridgetype ./ cp ../../../cmd/setbridge/sh/setbridge ./ cp ../../../target/release/mionps ./ cp ../../../target/release/mionparamspace ./ +cp ../../../target/release/pcfsserver ./ +cp ../../../target/release/dbg-generate-sata-wal-from-pcap ./ cp ../../../pkg/cat-dev/licenses/serial2-tokio-rs-apache.md ./ cp ../../../pkg/cat-dev/licenses/serial2-tokio-rs-bsd.md ./ cp ../../../LICENSE ./ cd ../ echo "Done! Building...." -pkgbuild --root ./working-dir/ --identifier "dev.rem-verse.sprig" --version "0.0.10" --install-location "/usr/local/bin" sprig.pkg +pkgbuild --root ./working-dir/ --identifier "dev.rem-verse.sprig" --version "0.0.11" --install-location "/usr/local/bin" sprig.pkg echo "Done! Preparing Distribution Directory..." mkdir working-dir-pkg diff --git a/installer-scripts/unix/nfpm.yaml b/installer-scripts/unix/nfpm.yaml index 0452e16..2e55aab 100644 --- a/installer-scripts/unix/nfpm.yaml +++ b/installer-scripts/unix/nfpm.yaml @@ -6,7 +6,7 @@ homepage: "https://github.com/rem-verse/sprig" license: "MIT" maintainer: "Cynthia " vendor: "RemVerse" -version: "v0.0.10" +version: "v0.0.11" arch: "amd64" platform: "linux" @@ -41,6 +41,10 @@ contents: dst: /usr/local/bin/mionps - src: ../../target/release/catlog dst: /usr/local/bin/catlog + - src: ../../target/release/pcfsserver + dst: /usr/local/bin/pcfsserver + - src: ../../target/release/dbg-generate-sata-wal-from-pcap + dst: /usr/local/bin/dbg-generate-sata-wal-from-pcap - dst: /usr/share/licenses/sprig type: dir - src: ../../pkg/cat-dev/licenses/serial2-tokio-rs-apache.md diff --git a/installer-scripts/win/sprig.wxs b/installer-scripts/win/sprig.wxs index fce2425..9dcaa12 100644 --- a/installer-scripts/win/sprig.wxs +++ b/installer-scripts/win/sprig.wxs @@ -6,7 +6,7 @@ @@ -55,6 +55,11 @@ + + + + + @@ -81,6 +86,8 @@ + + diff --git a/pkg/cat-dev/Cargo.toml b/pkg/cat-dev/Cargo.toml index 53090f3..aeeb2ae 100644 --- a/pkg/cat-dev/Cargo.toml +++ b/pkg/cat-dev/Cargo.toml @@ -12,7 +12,7 @@ default = ["clients", "serial", "servers"] clients = ["bitflags", "form_urlencoded", "local-ip-address", "mac_address", "network-interface", "rand", "reqwest", "tower", "wide"] scientists = [] serial = ["libc", "pin-project-lite", "windows"] -servers = ["bitflags", "local-ip-address", "rand", "sysinfo", "tower", "walkdir", "wide"] +servers = ["bitflags", "local-ip-address", "rand", "sysinfo", "tower", "wide"] [dependencies] aes = "^0.8.4" @@ -29,7 +29,7 @@ mac_address = { optional = true, workspace = true } miette.workspace = true network-interface = { optional = true, workspace = true } pin-project-lite = { version = "^0.2.16", optional = true } -reqwest = { version = "^0.12.18", default-features = false, features=["charset", "macos-system-configuration"], optional = true } +reqwest = { version = "^0.12.22", default-features = false, features=["charset", "macos-system-configuration"], optional = true } rand = { version = "^0.9.0", optional = true } scc = "^2.3.4" sysinfo = { version = "^0.35.1", default-features = false, features=["disk", "linux-tmpfs"], optional = true } @@ -39,15 +39,15 @@ tokio.workspace = true tokio-util = { version = "^0.7.15", default-features = false, features = ["codec"] } tower = { version = "^0.5.2", default-features = false, features = ["util", "make"], optional = true } valuable.workspace = true -walkdir = { version = "^2.5.0", optional = true } -wide = { version = "^0.7.32", optional = true } +walkdir = "^2.5.0" +wide = { version = "^0.7.33", optional = true } whoami = "^1.6.0" [target.'cfg(any(target_os = "linux", target_os = "freebsd", target_os = "openbsd", target_os = "netbsd", target_os = "macos"))'.dependencies] libc = { version = "^0.2", optional = true } [target.'cfg(target_os = "windows")'.dependencies] -windows = { version = "^0.61.1", default-features = false, features=["Win32_Devices_Communication", "Win32_Networking_WinSock", "Win32_Security", "Win32_Storage_FileSystem", "Win32_System_IO", "Win32_System_Registry", "Win32_System_Threading", "std"], optional = true } +windows = { version = "^0.61.3", default-features = false, features=["Win32_Devices_Communication", "Win32_Networking_WinSock", "Win32_Security", "Win32_Storage_FileSystem", "Win32_System_IO", "Win32_System_Registry", "Win32_System_Threading", "std"], optional = true } [dev-dependencies] tempfile = "^3.20.0" diff --git a/pkg/cat-dev/src/errors.rs b/pkg/cat-dev/src/errors.rs index c90d966..2b7c398 100644 --- a/pkg/cat-dev/src/errors.rs +++ b/pkg/cat-dev/src/errors.rs @@ -13,8 +13,6 @@ use miette::{Diagnostic, Report}; use std::{ffi::FromBytesUntilNulError, str::Utf8Error, string::FromUtf8Error, time::Duration}; use thiserror::Error; use tokio::{io::Error as IoError, task::JoinError}; - -#[cfg(feature = "servers")] use walkdir::Error as WalkdirError; #[cfg(feature = "clients")] @@ -101,6 +99,7 @@ pub enum CatBridgeError { #[derive(Error, Diagnostic, Debug)] pub enum APIError { /// Common network related API errors. + #[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] #[error(transparent)] #[diagnostic(transparent)] @@ -122,6 +121,7 @@ pub enum APIError { NoHostIpFound, } +#[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] impl From for CatBridgeError { fn from(value: CommonNetAPIError) -> Self { @@ -131,7 +131,6 @@ impl From for CatBridgeError { /// Trying to interact with the filesystem has resulted in an error. #[derive(Error, Diagnostic, Debug)] -#[non_exhaustive] pub enum FSError { /// We need a place to read/store a list of all the bridges on your host. /// @@ -164,7 +163,6 @@ pub enum FSError { #[error("Error writing/reading data from the filesystem: {0}")] #[diagnostic(code(cat_dev::fs::io))] IO(#[from] IoError), - #[cfg(feature = "servers")] #[error("Error iterating through folder: {0:?}")] #[diagnostic(code(cat_dev::fs::iterating_folder_error))] IteratingFolderError(#[from] WalkdirError), @@ -207,11 +205,13 @@ pub enum NetworkError { #[error("Failed to bind to a local address to receive packets.")] #[diagnostic(code(cat_dev::net::bind_failure))] BindFailure, + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] #[error(transparent)] #[diagnostic(transparent)] CommonClient(#[from] CommonNetClientNetworkError), /// An error has occurred in our common network framework. + #[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] #[error(transparent)] #[diagnostic(transparent)] @@ -222,6 +222,7 @@ pub enum NetworkError { #[error(transparent)] #[diagnostic(transparent)] FSEmul(#[from] FSEmulNetworkError), + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// See [`reqwest::Error`] for details. #[error("Underlying HTTP client error: {0}")] @@ -231,11 +232,13 @@ pub enum NetworkError { #[error("Error talking to the network could not send/receive data: {0}")] #[diagnostic(code(cat_dev::net::io_error))] IO(#[from] IoError), + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// See [`network_interface::Error::GetIfAddrsError`] for details. #[error("Failed to list the network interfaces on your device: {0:?}.")] #[diagnostic(code(cat_dev::net::list_interfaces_error))] ListInterfacesFailure(NetworkInterfaceError), + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// See [`local_ip_address::Error`] for details. #[error("Failure fetching local ip address: {0}")] @@ -254,6 +257,7 @@ pub enum NetworkError { #[diagnostic(code(cat_dev::net::set_broadcast_failure))] SetBroadcastFailure, /// Error adding a packet to a queue to send. + #[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] #[error("Error queueing up packet to be sent out over a conenction: {0:?}")] #[diagnostic(code(cat_dev::net::send_queue_failure))] @@ -269,6 +273,7 @@ pub enum NetworkError { Timeout(Duration), } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for CatBridgeError { fn from(value: CommonNetClientNetworkError) -> Self { @@ -276,6 +281,7 @@ impl From for CatBridgeError { } } +#[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] impl From for CatBridgeError { fn from(value: CommonNetNetworkError) -> Self { @@ -283,6 +289,7 @@ impl From for CatBridgeError { } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for CatBridgeError { fn from(value: ReqwestError) -> Self { diff --git a/pkg/cat-dev/src/fsemul/atapi/mod.rs b/pkg/cat-dev/src/fsemul/atapi/mod.rs index 972e321..b4f6f6c 100644 --- a/pkg/cat-dev/src/fsemul/atapi/mod.rs +++ b/pkg/cat-dev/src/fsemul/atapi/mod.rs @@ -8,5 +8,6 @@ //! you got a search request for ATAPI, and were looking for actual real ATAPI //! code. Not this weird nintendo variant. +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] pub mod server; diff --git a/pkg/cat-dev/src/fsemul/atapi/server.rs b/pkg/cat-dev/src/fsemul/atapi/server.rs index 7c32df1..d23941a 100644 --- a/pkg/cat-dev/src/fsemul/atapi/server.rs +++ b/pkg/cat-dev/src/fsemul/atapi/server.rs @@ -184,22 +184,31 @@ async fn handle_read_dlf( let dlf = DiskLayoutFile::try_from(Bytes::from(bytes_of_dlf))?; if let Some((path, offset)) = dlf.get_path_and_offset_for_file(read_address).await { - let metadata = path.metadata().map_err(FSError::from)?; - let file_size_bytes = usize::try_from(metadata.len() - offset).unwrap_or(usize::MAX); // Read the file contents... - let mut handle = File::open(&path).await.map_err(FSError::from)?; - handle - .seek(SeekFrom::Start(offset)) - .await - .map_err(FSError::from)?; - let mut buff = BytesMut::zeroed(std::cmp::min(file_size_bytes, rl_as_usize)); - handle.read_exact(&mut buff).await.map_err(FSError::from)?; - std::mem::drop(handle); - // Pad if necessary... - if file_size_bytes < rl_as_usize { - buff.reserve(rl_as_usize - file_size_bytes); - buff.extend(BytesMut::zeroed(rl_as_usize - file_size_bytes)); - } + let buff = { + let mut handle = File::open(&path).await.map_err(FSError::from)?; + handle + .seek(SeekFrom::Start(offset)) + .await + .map_err(FSError::from)?; + + let mut file_buff = BytesMut::zeroed(rl_as_usize); + let mut bytes_read = 0; + while bytes_read < rl_as_usize { + let read_this_go = handle + .read(&mut file_buff[bytes_read..]) + .await + .map_err(FSError::IO)?; + // EOF, rest of the buff is already 0's, so no need to pad. + if read_this_go == 0 { + break; + } + bytes_read += read_this_go; + } + + file_buff + }; + // Send! Ok(Some(buff.freeze())) } else { diff --git a/pkg/cat-dev/src/fsemul/errors.rs b/pkg/cat-dev/src/fsemul/errors.rs index 27069e7..1d284a9 100644 --- a/pkg/cat-dev/src/fsemul/errors.rs +++ b/pkg/cat-dev/src/fsemul/errors.rs @@ -15,6 +15,9 @@ use thiserror::Error; /// Errors related to API errors for `FSEmul`. #[derive(Diagnostic, Error, Debug, PartialEq, Eq)] pub enum FSEmulAPIError { + #[error("File descriptors have already been opened, we cannot change our strategy now!")] + #[diagnostic(code(cat_dev::api::fsemul::cannot_swap_fd_strategy))] + CannotSwapFdStrategy, /// This DLF address is too large to be inserted. #[error("DLF Address is too large ({0:016X}) to be inserted ({1:016X})")] #[diagnostic(code(cat_dev::api::fsemul::dlf_address_too_large))] diff --git a/pkg/cat-dev/src/fsemul/host_filesystem.rs b/pkg/cat-dev/src/fsemul/host_filesystem.rs index a04b4b5..c7c2d79 100644 --- a/pkg/cat-dev/src/fsemul/host_filesystem.rs +++ b/pkg/cat-dev/src/fsemul/host_filesystem.rs @@ -5,7 +5,10 @@ use crate::{ TitleID, errors::{CatBridgeError, FSError}, fsemul::{ - bsf::BootSystemFile, dlf::DiskLayoutFile, errors::FSEmulFSError, pcfs::errors::PCFSApiError, + bsf::BootSystemFile, + dlf::DiskLayoutFile, + errors::{FSEmulAPIError, FSEmulFSError}, + pcfs::errors::PCFSApiError, }, }; use bytes::{Bytes, BytesMut}; @@ -14,24 +17,33 @@ use scc::{ }; use std::{ collections::HashMap, + ffi::OsString, + fs::{ + DirEntry, copy as copy_file_sync, create_dir_all as create_dir_all_sync, + read_dir as read_dir_sync, read_link as read_link_sync, + remove_dir_all as remove_dir_all_sync, remove_file as remove_file_sync, + rename as rename_sync, + }, hash::RandomState, io::{Error as IOError, SeekFrom}, path::{Path, PathBuf}, sync::{ Arc, - atomic::{AtomicI32, Ordering as AtomicOrdering}, + atomic::{AtomicBool, AtomicI32, Ordering as AtomicOrdering}, }, }; use tokio::{ - fs::{ - File, OpenOptions, ReadDir, create_dir_all, read_dir, remove_file, rename, - write as fs_write, - }, + fs::{File, OpenOptions, read as fs_read, write as fs_write}, io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, + sync::Mutex, }; +use tracing::{info, warn}; use valuable::{Fields, NamedField, NamedValues, StructDef, Structable, Valuable, Value, Visit}; +use walkdir::WalkDir; use whoami::username; +/// A way to create truly unique file fd's. Just a counter going up. +static UNIQUE_FILE_FD: AtomicI32 = AtomicI32::new(1); /// Current "FD" for directories. Just a counter going up. static FOLDER_FD: AtomicI32 = AtomicI32::new(1); @@ -50,14 +62,22 @@ static FOLDER_FD: AtomicI32 = AtomicI32::new(1); pub struct HostFilesystem { /// The path to the base data directory to serve a filesystem out of. cafe_sdk_path: PathBuf, + /// The actively mounted "disc". + /// + /// This is a tuple of (isSLC, isSystem, [`TitleID`]). + /// + /// When a disc is mounted we will copy the title from SLC/MLC + /// directory, into `disc/` recursively. + disc_mounted: Arc>>, /// List of open file handles. /// /// This contains a value of (file, file size, path, stream owner). open_file_handles: Arc)>>, /// List of open folder "handles". /// - /// This contains a value of (read directory, is end, path, stream owner) - open_folder_handles: Arc)>>, + /// This contains a value of (directory items, index, is end, path, stream owner) + open_folder_handles: + Arc, usize, bool, PathBuf, Option)>>, /// A set of folders that we've "marked" as read-only. /// /// We don't actually synchronize this to the filesystem because the original @@ -68,6 +88,12 @@ pub struct HostFilesystem { /// This is not the case on older windows distributions, unix based distros, /// or similar. folders_marked_read_only: Arc>, + /// If we are forcing unique file fd's. This should only be changed + /// if we have not opened a file yet. + is_using_unique_fds: bool, + /// If we've opened a file, used to safely ensure we don't switch from + /// unique fdf's to not. + has_opened_file: Arc, } impl HostFilesystem { @@ -100,48 +126,48 @@ impl HostFilesystem { return Err(FSEmulFSError::CantFindCafeSdkPath.into()); }; - Self::patch_case_sensitive_title_ids(&cafe_sdk_path).await?; + Self::patch_case_sensitivity(&cafe_sdk_path).await?; - if !Self::join_many( - &cafe_sdk_path, - [ + for path in [ + &[ "data", "mlc", "sys", "title", "00050030", "1001000a", "code", "app.xml", - ], - ) - .exists() || !Self::join_many( - &cafe_sdk_path, - [ + ] as &[&str], + &[ "data", "mlc", "sys", "title", "00050030", "1001010a", "code", "app.xml", ], - ) - .exists() || !Self::join_many( - &cafe_sdk_path, - [ + &[ "data", "mlc", "sys", "title", "00050030", "1001020a", "code", "app.xml", ], - ) - .exists() - { - return Err(FSEmulFSError::CafeSdkPathCorrupt.into()); - } - - // Can't generate a `fw.img` file for now :( - if !Self::join_many( - &cafe_sdk_path, - [ + &[ + "data", "mlc", "sys", "title", "00050010", "1f700500", "code", + ], + &[ + "data", "mlc", "sys", "title", "00050010", "1f700500", "content", + ], + &[ + "data", "mlc", "sys", "title", "00050010", "1f700500", "meta", + ], + // Can't generate a `fw.img` for now.... :( + &[ "data", "slc", "sys", "title", "00050010", "1000400a", "code", "fw.img", ], - ) - .exists() - { - return Err(FSEmulFSError::CafeSdkPathCorrupt.into()); + ] { + if !Self::join_many(&cafe_sdk_path, path).exists() { + return Err(FSEmulFSError::CafeSdkPathCorrupt.into()); + } } + Self::prepare_for_serving(&cafe_sdk_path).await?; + let ro_folders = Self::get_default_read_only_folders(&cafe_sdk_path); + Ok(Self { cafe_sdk_path, - folders_marked_read_only: Arc::new(ConcurrentSet::new()), + disc_mounted: Arc::new(Mutex::new(None)), + folders_marked_read_only: Arc::new(ro_folders), open_file_handles: Arc::new(ConcurrentMap::new()), open_folder_handles: Arc::new(ConcurrentMap::new()), + is_using_unique_fds: false, + has_opened_file: Arc::new(AtomicBool::new(false)), }) } @@ -155,6 +181,41 @@ impl HostFilesystem { &self.cafe_sdk_path } + /// The root path to the Cafe SDK. + /// + /// *note: although we do expose this for logging, and other info... we do + /// not recommend manually interacting with the SDK path. There are much + /// better alternatives.* + #[must_use] + pub fn disc_emu_path(&self) -> PathBuf { + Self::join_many(&self.cafe_sdk_path, ["data", "disc"]) + } + + /// Force unique file descriptors for open files. + /// + /// Certain OS's _can_ return duplicate fd's especially when opening, + /// and closing files. This can make deciphering logs harder because the + /// same FD may appear multiple times, when you're trying to just find + /// the logs related to one file descriptor. + /// + /// When unique fd's is turned on, similar to folders we just use a global + /// wrapping counter so that way every file descriptor is guaranteed to be + /// unique. + /// + /// ## Errors + /// + /// This will error if any file has ever been opened. This is because once + /// a client has already connected, and done some stuff with file stuff it + /// expects one set of behaviors, we cannot change another one. + pub fn force_unique_fds(&mut self) -> Result<(), FSEmulAPIError> { + if self.has_opened_file.load(AtomicOrdering::Relaxed) { + Err(FSEmulAPIError::CannotSwapFdStrategy) + } else { + self.is_using_unique_fds = true; + Ok(()) + } + } + /// Open a file, and return it's file descriptor number. /// /// ## Errors @@ -166,6 +227,7 @@ impl HostFilesystem { path: &PathBuf, stream_owner: Option, ) -> Result { + self.has_opened_file.store(true, AtomicOrdering::Relaxed); let fd = open_options.open(path).await?; let raw_fd; #[cfg(unix)] @@ -180,11 +242,16 @@ impl HostFilesystem { } let md = fd.metadata().await?; + let final_fd = if self.is_using_unique_fds { + UNIQUE_FILE_FD.fetch_add(1, AtomicOrdering::SeqCst) + } else { + raw_fd + }; self.open_file_handles - .insert(raw_fd, (fd, md.len(), path.clone(), stream_owner)) - .map_err(|_| IOError::other("OS returned duplicate fd?"))?; - Ok(raw_fd) + .insert(final_fd, (fd, md.len(), path.clone(), stream_owner)) + .map_err(|_| IOError::other("somehow got duplicate fd?"))?; + Ok(final_fd) } /// Get a file from a file descriptor number. @@ -236,7 +303,7 @@ impl HostFilesystem { pub async fn read_file( &self, fd: i32, - total_data_to_read: usize, + mut total_data_to_read: usize, for_stream: Option, ) -> Result, FSError> { let Some(mut real_entry) = self.open_file_handles.get_async(&fd).await else { @@ -246,10 +313,19 @@ impl HostFilesystem { return Ok(None); } let file_reader = &mut real_entry.0; + let mut file_buff = BytesMut::zeroed(total_data_to_read); - let bytes_read = file_reader.read(&mut file_buff).await?; - if bytes_read < total_data_to_read { - file_buff[bytes_read..].fill(0xCD); + let mut total_bytes_read = 0_usize; + while total_data_to_read > 0 { + let bytes_read = file_reader.read(&mut file_buff[total_bytes_read..]).await?; + if bytes_read == 0 { + break; + } + total_data_to_read -= bytes_read; + total_bytes_read += bytes_read; + } + if file_buff.len() > total_bytes_read { + file_buff.truncate(total_bytes_read); } Ok(Some(file_buff.freeze())) @@ -342,16 +418,16 @@ impl HostFilesystem { /// ## Errors /// /// If the path doesn't exist, then we can't open the folder. - pub async fn open_folder( - &self, - path: &PathBuf, - for_stream: Option, - ) -> Result { - let dhandle = read_dir(path).await?; + pub fn open_folder(&self, path: &PathBuf, for_stream: Option) -> Result { + let mut dhandle = read_dir_sync(path)? + .filter_map(Result::ok) + .collect::>(); + dhandle.sort_by_key(DirEntry::path); + let fake_fd = FOLDER_FD.fetch_add(1, AtomicOrdering::SeqCst); self.open_folder_handles - .insert(fake_fd, (dhandle, false, path.clone(), for_stream)) + .insert(fake_fd, (dhandle, 0, false, path.clone(), for_stream)) .map_err(|_| IOError::other("OS returned duplicate fd?"))?; Ok(fake_fd) } @@ -361,12 +437,8 @@ impl HostFilesystem { /// ## Errors /// /// If we could not actually insert the folder into the read only map. - pub async fn mark_folder_read_only(&self, path: PathBuf) -> Result<(), FSError> { - self.folders_marked_read_only - .insert_async(path) - .await - .map_err(|_| IOError::other("Folder could not be marked read-only?")) - .map_err(FSError::IO) + pub async fn mark_folder_read_only(&self, path: PathBuf) { + _ = self.folders_marked_read_only.insert_async(path).await; } /// Mark a folder as being 'read-write' for this session. @@ -400,21 +472,25 @@ impl HostFilesystem { return Ok(None); } - let component_count = entry.2.components().count(); + let component_count = entry.3.components().count(); let mut value: Option = None; - if !entry.1 { - let iter = &mut entry.0; + if !entry.2 { loop { - value = iter.next_entry().await?.map(|de| de.path()); - if let Some(ref_value) = value.as_ref() { + if entry.1 < entry.0.len() { + let ref_value = entry.0[entry.1].path(); + entry.1 += 1; + if (!ref_value.is_file() && !ref_value.is_dir()) || ref_value.is_symlink() { continue; } + + value = Some(ref_value); } + break; } if value.is_none() { - entry.1 = true; + entry.2 = true; } } @@ -437,9 +513,12 @@ impl HostFilesystem { if !Self::allow_folder_access(&real_entry, for_stream) { return Ok(()); } + if real_entry.1 == 0 { + return Ok(()); + } - real_entry.0 = read_dir(&real_entry.2).await?; - real_entry.1 = false; + real_entry.1 -= 1; + real_entry.2 = false; Ok(()) } @@ -472,10 +551,10 @@ impl HostFilesystem { /// - If the temp directory does not exist, and we can't create it. /// - If the boot system file does not exist, and we can't write it to disk. pub async fn boot1_sytstem_path(&self) -> Result { - let mut path = self.temp_path().await?; + let mut path = self.temp_path()?; path.push("caferun"); if !path.exists() { - create_dir_all(&path).await?; + create_dir_all_sync(&path)?; } path.push("ppc.bsf"); @@ -496,10 +575,10 @@ impl HostFilesystem { /// - If the temporary directory does not exist, and we can't create it. /// - If the disk ID path does not exist, and we can't write it to disk. pub async fn disk_id_path(&self) -> Result { - let mut path = self.temp_path().await?; + let mut path = self.temp_path()?; path.push("caferun"); if !path.exists() { - create_dir_all(&path).await?; + create_dir_all_sync(&path)?; } path.push("diskid.bin"); @@ -510,6 +589,47 @@ impl HostFilesystem { Ok(path) } + #[doc( + // This is not yet finished and the signature may change.... + hidden, + )] + /// Mount a particular title as if it were a disc. + /// + /// ## Errors + /// + /// - If we cannot remove any existing disc that may be present. + /// - If we cannot copy the title to the disc id path. + pub async fn mount_disk_title( + &mut self, + is_slc: bool, + is_sys: bool, + title_id: TitleID, + ) -> Result<(), FSError> { + let source_path = Self::join_many( + &self.cafe_sdk_path, + [ + "data".to_owned(), + if is_slc { "slc" } else { "mlc" }.to_owned(), + if is_sys { "sys" } else { "usr" }.to_owned(), + "title".to_owned(), + format!("{:08x}", title_id.0), + format!("{:08x}", title_id.1), + ], + ); + let dest_path = Self::join_many(&self.cafe_sdk_path, ["data", "disc"]); + if dest_path.exists() { + remove_dir_all_sync(&dest_path).map_err(FSError::IO)?; + } + + Self::copy_dir(&source_path, &dest_path)?; + // Mount was successful! + { + let mut guard = self.disc_mounted.lock().await; + guard.replace((is_slc, is_sys, title_id)); + } + todo!("figure out how to mount diskid.bin") + } + /// Get the path to the current firmware file to boot on the MION. /// /// This is guaranteed to always exist, as it's part of our check for a @@ -535,10 +655,10 @@ impl HostFilesystem { /// - If the firmware image file does not exist. /// - If the dlf file does not exist, and we can't create it. pub async fn ppc_boot_dlf_path(&self) -> Result { - let mut path = self.temp_path().await?; + let mut path = self.temp_path()?; path.push("caferun"); if !path.exists() { - create_dir_all(&path).await.map_err(FSError::from)?; + create_dir_all_sync(&path).map_err(FSError::from)?; } path.push("ppc_boot.dlf"); @@ -561,8 +681,21 @@ impl HostFilesystem { #[must_use] pub fn path_allows_writes(&self, path: &Path) -> bool { // TODO(mythra): check FSEmulAttributeRules - !path.to_string_lossy().contains("%DISC_EMU_DIR") - && !path.starts_with(Self::join_many(&self.cafe_sdk_path, ["data", "disc"])) + let lossy_path = path.to_string_lossy(); + let trimmed_lossy_path = lossy_path + .trim_start_matches("/vol/pc") + .trim_start_matches('/'); + if trimmed_lossy_path.starts_with("%DISC_EMU_DIR") { + return trimmed_lossy_path.starts_with("%DISC_EMU_DIR/save"); + } + if path.starts_with(Self::join_many(&self.cafe_sdk_path, ["data", "disc"])) { + return path.starts_with(Self::join_many( + &self.cafe_sdk_path, + ["data", "disc", "save"], + )); + } + + true } /// Given a UTF-8 string path, get a pathbuf reference. @@ -646,6 +779,45 @@ impl HostFilesystem { ))) } + /// Create a directory within a particular path. + /// + /// ## Errors + /// + /// If we cannot end up creating this directory due to a filesystem error. + pub fn create_directory(&self, at: &Path) -> Result<(), FSError> { + create_dir_all_sync(at).map_err(FSError::IO) + } + + /// Copy a file, symlink, or directory. + /// + /// ## Errors + /// + /// If we run into any filesystem error renaming a source, or directory. + pub fn copy(&self, from: &Path, to: &Path) -> Result<(), FSError> { + if from.is_dir() { + Self::copy_dir(from, to) + } else { + copy_file_sync(from, to).map_err(FSError::IO).map(|_| ()) + } + } + + /// Rename a file, symlink, or directory. + /// + /// This is implemented so we can rename directories, and files without + /// having to worry about the logic. Especially given the fact the built in + /// rename doesn't support directories. + /// + /// ## Errors + /// + /// - If we run into any filesystem error renaming a source, or directory. + pub fn rename(&self, from: &Path, to: &Path) -> Result<(), FSError> { + if from.is_dir() { + Self::rename_dir(from, to) + } else { + rename_sync(from, to).map_err(FSError::IO) + } + } + /// Get a file from the SLC. /// /// The SLC always serves "sys" files, and are relative to a title id, almost @@ -667,19 +839,48 @@ impl HostFilesystem { ) } + /// Get the current OS's default directory path. + /// + /// For Windows this is: `C:\cafe_sdk`. + /// For Unix/BSD likes this is: `/opt/cafe_sdk` + #[allow( + // Not actually unreachable unless on unsupported OS. + unreachable_code, + )] + #[must_use] + pub fn default_cafe_folder() -> Option { + #[cfg(target_os = "windows")] + { + return Some(PathBuf::from(r"C:\cafe_sdk")); + } + + #[cfg(any( + target_os = "linux", + target_os = "freebsd", + target_os = "openbsd", + target_os = "netbsd", + target_os = "macos" + ))] + { + return Some(PathBuf::from("/opt/cafe_sdk")); + } + + None + } + /// Get the current path to the temporary directory for this Cafe SDK /// install. /// /// ## Errors /// /// - If the temporary path does not exist and could not be created. - async fn temp_path(&self) -> Result { + fn temp_path(&self) -> Result { let temp_path = Self::join_many( &self.cafe_sdk_path, ["temp".to_owned(), username().to_lowercase()], ); if !temp_path.exists() { - create_dir_all(&temp_path).await?; + create_dir_all_sync(&temp_path)?; } Ok(temp_path) } @@ -712,36 +913,7 @@ impl HostFilesystem { ) } - /// Get the current OS's default directory path. - /// - /// For Windows this is: `C:\cafe_sdk`. - /// For Unix/BSD likes this is: `/opt/cafe_sdk` - #[allow( - // Not actually unreachable unless on unsupported OS. - unreachable_code, - )] - #[must_use] - pub fn default_cafe_folder() -> Option { - #[cfg(target_os = "windows")] - { - return Some(PathBuf::from(r"C:\cafe_sdk")); - } - - #[cfg(any( - target_os = "linux", - target_os = "freebsd", - target_os = "openbsd", - target_os = "netbsd", - target_os = "macos" - ))] - { - return Some(PathBuf::from("/opt/cafe_sdk")); - } - - None - } - - async fn patch_case_sensitive_title_ids(cafe_sdk_path: &Path) -> Result<(), FSError> { + async fn patch_case_sensitivity(cafe_sdk_path: &Path) -> Result<(), FSError> { // First we need to check if we're even on a temporary filesystem/path. if !cafe_sdk_path.exists() { return Ok(()); @@ -751,74 +923,67 @@ impl HostFilesystem { let is_insensitive = File::open(Self::join_many(cafe_sdk_path, ["insensitivecheck.txt"])) .await .is_ok(); - remove_file(capital_path).await?; + remove_file_sync(capital_path)?; if is_insensitive { return Ok(()); } - for directory in [ - Self::join_many(cafe_sdk_path, ["data", "slc", "sys", "title"]), - Self::join_many(cafe_sdk_path, ["data", "slc", "usr", "title"]), - Self::join_many(cafe_sdk_path, ["data", "mlc", "sys", "title"]), - Self::join_many(cafe_sdk_path, ["data", "mlc", "usr", "title"]), - ] { - if !directory.exists() { - // Don't need to patch directories that don't exist. - continue; - } - - // Now we need to scan, and lowercase all title ids. So those are the - // next two sub dirs as they're split into `title/{upper}/{lower}`. - let mut iter = read_dir(&directory).await?; - let lossy_cafe_dir = cafe_sdk_path.as_os_str().to_string_lossy().to_string(); - while let Ok(Some(entry)) = iter.next_entry().await { - let p = entry.path(); - if !p.is_dir() || !p.exists() { + info!( + "Your Host OS is not case-insensitive for file-paths... ensuring CafeSDK is all lowercase, this may take awhile..." + ); + let cafe_sdk_components = cafe_sdk_path.components().count(); + let mut had_rename = true; + while had_rename { + had_rename = false; + for directory in [ + Self::join_many(cafe_sdk_path, ["data", "slc", "sys", "title"]), + Self::join_many(cafe_sdk_path, ["data", "slc", "usr", "title"]), + Self::join_many(cafe_sdk_path, ["data", "mlc", "sys", "title"]), + Self::join_many(cafe_sdk_path, ["data", "mlc", "usr", "title"]), + ] { + if !directory.exists() { + // Don't need to patch directories that don't exist. continue; } - let mut inner_iter = read_dir(&p).await?; - while let Ok(Some(inner_entry)) = inner_iter.next_entry().await { - let ip = inner_entry.path(); - if !ip.is_dir() || !ip.exists() { + let mut iter = WalkDir::new(&directory) + .contents_first(false) + .follow_links(false) + .follow_root_links(false) + .into_iter(); + while let Some(Ok(entry)) = iter.next() { + let p = entry.path(); + if !p.exists() { continue; } - // Doing a lossy conversion is safe here cause we know all title ids are valid ascii + utf-8. - let new_path = ip - .as_os_str() - .to_string_lossy() - .trim_start_matches(&lossy_cafe_dir) - .to_ascii_lowercase(); - if ip - .as_os_str() - .to_string_lossy() - .trim_start_matches(&lossy_cafe_dir) - != new_path - { + let path_minus_cafe = p + .components() + .skip(cafe_sdk_components) + .collect::(); + let Some(path_as_utf8) = path_minus_cafe.as_os_str().to_str() else { + warn!(problematic_path = %p.display(), "Path in Cafe SDK directory is not UTF-8! This may cause errors fetching!"); + continue; + }; + let new_path = path_as_utf8.to_ascii_lowercase(); + if path_as_utf8 != new_path { let mut final_new_path = cafe_sdk_path.as_os_str().to_owned(); + final_new_path.push("/"); final_new_path.push(&new_path); let new = PathBuf::from(final_new_path); - rename(ip, new).await?; - } - } - let new_path = p - .as_os_str() - .to_string_lossy() - .trim_start_matches(&lossy_cafe_dir) - .to_ascii_lowercase(); - if p.as_os_str() - .to_string_lossy() - .trim_start_matches(&lossy_cafe_dir) - != new_path - { - let mut final_new_path = cafe_sdk_path.as_os_str().to_owned(); - final_new_path.push(&new_path); - rename(p, final_new_path).await?; + if p.is_dir() { + Self::rename_dir(p, &new)?; + had_rename = true; + } else { + rename_sync(p, new)?; + had_rename = true; + } + } } } } + info!("ensure CafeSDK path is now case-insensitive by renaming to all lowercase..."); Ok(()) } @@ -837,19 +1002,359 @@ impl HostFilesystem { requesting_stream_id == owned_stream_id } + #[allow( + // TODO(mythra): fix + clippy::type_complexity + )] fn allow_folder_access( - entry: &CMOccupiedEntry), RandomState>, + entry: &CMOccupiedEntry< + i32, + (Vec, usize, bool, PathBuf, Option), + RandomState, + >, requester: Option, ) -> bool { let Some(requesting_stream_id) = requester else { return true; }; - let Some(owned_stream_id) = entry.3 else { + let Some(owned_stream_id) = entry.4 else { return true; }; requesting_stream_id == owned_stream_id } + + /// Enusre an SDK path is ready for serving this means: + /// + /// - Create some configuration files that SDKs don't come with, but will + /// help the OS boot up. + /// - Mount the `DISC` directory if one is not present. + async fn prepare_for_serving(cafe_sdk_path: &Path) -> Result<(), FSError> { + if !Self::join_many(cafe_sdk_path, ["data", "slc", "sys", "config", "eco.xml"]).exists() { + Self::generate_eco_xml(cafe_sdk_path).await?; + } + if !Self::join_many( + cafe_sdk_path, + ["data", "slc", "sys", "proc", "prefs", "wii_acct.xml"], + ) + .exists() + { + Self::generate_wii_acct_xml(cafe_sdk_path).await?; + } + + // Unmount any leftover discs.... + if Self::join_many(cafe_sdk_path, ["data", "disc"]).exists() { + remove_dir_all_sync(Self::join_many(cafe_sdk_path, ["data", "disc"])) + .map_err(FSError::IO)?; + } + // Manually mount in SysConfigTool..... + // + // This doesn't actually create a discid.bin, but the files do exist. + let disc_dir = Self::join_many(cafe_sdk_path, ["data", "disc"]); + let sctt_dir = Self::join_many( + cafe_sdk_path, + ["data", "mlc", "sys", "title", "00050010", "1f700500"], + ); + for subpath in ["code", "content", "meta"] { + Self::copy_dir( + &Self::join_many(&sctt_dir, [subpath]), + &Self::join_many(&disc_dir, [subpath]), + )?; + } + // Manually capitilize the title id in app.xml, the normal PCFS + // tooling does this, even though it is case-insensitive, but for matching. + let app_xml_path = Self::join_many(cafe_sdk_path, ["data", "disc", "code", "app.xml"]); + // app.xml must be utf-8 to be read by the OS completely, so if we end up + // writing a corrupt app.xml, would be the exact same as the OS + // interpreting that. + let base_app_xml = String::from_utf8_lossy(&fs_read(&app_xml_path).await?).to_string(); + fs_write(&app_xml_path, Self::capitilize_title_id(base_app_xml)).await?; + + Ok(()) + } + + fn copy_dir(source_path: &Path, dest_path: &Path) -> Result<(), FSError> { + if !dest_path.exists() { + create_dir_all_sync(dest_path)?; + } + let new_path_as_str_bytes = dest_path.as_os_str().as_encoded_bytes(); + let old_path_bytes = source_path.as_os_str().as_encoded_bytes(); + + for result in WalkDir::new(source_path) + .follow_links(false) + .follow_root_links(false) + { + let rpb = result?.into_path(); + let os_str_for_entry = rpb.as_os_str().as_encoded_bytes(); + let mut new_bytes = Vec::with_capacity(os_str_for_entry.len() + 3); + new_bytes.extend_from_slice(new_path_as_str_bytes); + new_bytes.extend_from_slice(&os_str_for_entry[old_path_bytes.len()..]); + let as_new_path = + PathBuf::from(unsafe { OsString::from_encoded_bytes_unchecked(new_bytes) }); + + if rpb.is_symlink() { + let mut resolved_path = read_link_sync(&rpb)?; + { + // If this symlink is a symlink to another path within the same + // directory, then rewrite it as well to start under our new directory. + let os_str_for_resolved = resolved_path.as_os_str().as_encoded_bytes(); + if os_str_for_resolved.starts_with(old_path_bytes) { + let mut new_bytes = Vec::with_capacity(os_str_for_resolved.len() + 3); + new_bytes.extend_from_slice(new_path_as_str_bytes); + new_bytes.extend_from_slice(&os_str_for_entry[old_path_bytes.len()..]); + resolved_path = PathBuf::from(unsafe { + OsString::from_encoded_bytes_unchecked(new_bytes) + }); + } + } + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + symlink(resolved_path, &as_new_path)?; + } + + #[cfg(target_os = "windows")] + { + use std::os::windows::fs::{symlink_dir, symlink_file}; + + if resolved_path.is_dir() { + symlink_dir(resolved_path, &as_new_path)?; + } else { + symlink_file(resolved_path, &as_new_path)?; + } + } + } else if rpb.is_file() { + copy_file_sync(&rpb, &as_new_path)?; + } else if rpb.is_dir() { + create_dir_all_sync(&as_new_path)?; + } + } + + Ok(()) + } + + /// Rename an entire directory. + /// + /// We have to implement this ourselves, because [`tokio::fs::rename`], and + /// [`std::fs::rename`] don't support renaming a directory at all on windows, + /// which is one of the critical OS's that we need to support. + /// + /// This 'rename' works by actually creating a new directory. Then + /// moving all the files over with rename. This is slow, but + /// works. + fn rename_dir(source_path: &Path, dest_path: &Path) -> Result<(), FSError> { + if !dest_path.exists() { + create_dir_all_sync(dest_path)?; + } + let new_path_as_str_bytes = dest_path.as_os_str().as_encoded_bytes(); + let old_path_bytes = source_path.as_os_str().as_encoded_bytes(); + + for result in WalkDir::new(source_path) + .follow_links(false) + .follow_root_links(false) + { + let rpb = result?.into_path(); + let os_str_for_entry = rpb.as_os_str().as_encoded_bytes(); + let mut new_bytes = Vec::with_capacity(os_str_for_entry.len() + 3); + new_bytes.extend_from_slice(new_path_as_str_bytes); + new_bytes.extend_from_slice(&os_str_for_entry[old_path_bytes.len()..]); + let as_new_path = + PathBuf::from(unsafe { OsString::from_encoded_bytes_unchecked(new_bytes) }); + + if rpb.is_symlink() { + let mut resolved_path = read_link_sync(&rpb)?; + { + // If this symlink is a symlink to another path within the same + // directory, then rewrite it as well to start under our new directory. + let os_str_for_resolved = resolved_path.as_os_str().as_encoded_bytes(); + if os_str_for_resolved.starts_with(old_path_bytes) { + let mut new_bytes = Vec::with_capacity(os_str_for_resolved.len() + 3); + new_bytes.extend_from_slice(new_path_as_str_bytes); + new_bytes.extend_from_slice(&os_str_for_entry[old_path_bytes.len()..]); + resolved_path = PathBuf::from(unsafe { + OsString::from_encoded_bytes_unchecked(new_bytes) + }); + } + } + + // Symlinks to directories on Windows run into + // edge cases, and will frequently get permission denied when + // attempting to remove them. + // + // They will instead be cleaned up by the final folder cleanup which + // will not run into any such errors. + let should_remove: bool; + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + symlink(resolved_path, &as_new_path)?; + should_remove = true; + } + + #[cfg(target_os = "windows")] + { + use std::os::windows::fs::{symlink_dir, symlink_file}; + + if resolved_path.is_dir() { + symlink_dir(resolved_path, &as_new_path)?; + should_remove = false; + } else { + symlink_file(resolved_path, &as_new_path)?; + should_remove = true; + } + } + + // Remove the original link, we renamed this.... + if should_remove { + remove_file_sync(&rpb)?; + } + } else if rpb.is_file() { + rename_sync(&rpb, &as_new_path)?; + } else if rpb.is_dir() { + create_dir_all_sync(&as_new_path)?; + } + } + // Clean up after ourselves... + remove_dir_all_sync(source_path)?; + + Ok(()) + } + + /// Generate an `eco.xml` if one is not present. + /// + /// This is _required_ in order to provide an actual functional PCFS install, + /// and not actually normally created on the host filesystem with the + /// official tools. It just generates it in memory. + /// + /// ## Errors + /// + /// If we cannot create the config directory, or write the eco config + /// file to disk. + async fn generate_eco_xml(cafe_os_path: &Path) -> Result<(), FSError> { + let mut eco_path = Self::join_many(cafe_os_path, ["data", "slc", "sys", "config"]); + if !eco_path.exists() { + create_dir_all_sync(&eco_path).map_err(FSError::IO)?; + } + eco_path.push("eco.xml"); + + let mut eco_file = File::create(eco_path).await.map_err(FSError::IO)?; + eco_file + .write_all( + br#" + + 0 + 3601 + 15 + 1 +"#, + ) + .await + .map_err(FSError::IO)?; + + #[cfg(unix)] + { + use std::{fs::Permissions, os::unix::prelude::*}; + eco_file + .set_permissions(Permissions::from_mode(0o770)) + .await?; + } + + Ok(()) + } + + /// Generate a `wii_acct.xml` if one is not present. + /// + /// This is _required_ in order to provide an actual functional PCFS install, + /// and not actually normally created on the host filesystem with the + /// official tools. It just generates it in memory. + /// + /// ## Errors + /// + /// If we cannot create the config directory, or write the wii acct config + /// file to disk. + async fn generate_wii_acct_xml(cafe_os_path: &Path) -> Result<(), FSError> { + let mut wii_path = Self::join_many(cafe_os_path, ["data", "slc", "sys", "proc", "prefs"]); + if !wii_path.exists() { + create_dir_all_sync(&wii_path).map_err(FSError::IO)?; + } + wii_path.push("wii_acct.xml"); + + let mut wii_file = File::create(wii_path).await.map_err(FSError::IO)?; + wii_file + .write_all( + br#" + + + 00570069006900000000000000000000000000000000 + + 0 + 1 + + + 18 + 0 + 0 + 0 + 0 + +"#, + ) + .await + .map_err(FSError::IO)?; + + #[cfg(unix)] + { + use std::{fs::Permissions, os::unix::prelude::*}; + wii_file + .set_permissions(Permissions::from_mode(0o770)) + .await?; + } + + Ok(()) + } + + /// Take an app.xml, and capitilize the title id. Used for byte-matching + /// perfectly with the official SDK. + #[must_use] + fn capitilize_title_id(app_xml: String) -> String { + let Some(title_id_xml_tag_start) = app_xml.find("') else { + return app_xml; + }; + + let tid_start = title_id_xml_tag_start + title_id_tag_end; + let Some(title_slash_location) = app_xml[tid_start..].find("") else { + return app_xml; + }; + let tid_end = tid_start + title_slash_location; + let title_id = &app_xml[tid_start..tid_end]; + let mut final_xml = String::with_capacity(app_xml.len()); + final_xml += &app_xml[..tid_start]; + final_xml += &title_id.to_uppercase(); + final_xml += &app_xml[tid_end..]; + + final_xml + } + + fn get_default_read_only_folders(cafe_dir: &Path) -> ConcurrentSet { + let set = ConcurrentSet::new(); + + for cafe_sub_paths in [ + &["data", "slc", "sys", "config"] as &[&str], + &["data", "slc", "sys", "proc"], + &["data", "slc", "sys", "logs"], + &["data", "mlc", "usr"], + &["data", "mlc", "usr", "import"], + &["data", "mlc", "usr", "title"], + ] { + _ = set.insert(Self::join_many(cafe_dir, cafe_sub_paths)); + } + + set + } } const HOST_FILESYSTEM_FIELDS: &[NamedField<'static>] = &[ @@ -876,7 +1381,7 @@ impl Valuable for HostFilesystem { }); let mut folder_values = HashMap::with_capacity(self.open_folder_handles.len()); self.open_folder_handles.scan(|k, v| { - folder_values.insert(*k, format!("{}", v.2.display())); + folder_values.insert(*k, format!("{}", v.3.display())); }); visitor.visit_named_fields(&NamedValues::new( @@ -978,6 +1483,7 @@ impl Valuable for FilesystemLocation { } } +#[cfg_attr(docsrs, doc(cfg(test)))] #[cfg(test)] pub mod test_helpers { use super::*; @@ -1002,6 +1508,15 @@ pub mod test_helpers { vec![ "data", "mlc", "sys", "title", "00050030", "1001000a", "code", ], + vec![ + "data", "mlc", "sys", "title", "00050010", "1f700500", "code", + ], + vec![ + "data", "mlc", "sys", "title", "00050010", "1f700500", "content", + ], + vec![ + "data", "mlc", "sys", "title", "00050010", "1f700500", "meta", + ], // Purposefully create capital so we can validate renaming works! vec![ "data", "mlc", "sys", "title", "00050030", "1001010A", "code", @@ -1012,6 +1527,15 @@ pub mod test_helpers { vec![ "data", "slc", "sys", "title", "00050010", "1000400a", "code", ], + vec!["data", "mlc", "sys", "update", "nand", "os_v10_ndebug"], + vec!["data", "mlc", "sys", "update", "nand", "os_v10_debug"], + vec!["data", "slc", "sys", "proc", "prefs"], + vec![ + "data", "slc", "sys", "title", "00050010", "1000800a", "code", + ], + vec![ + "data", "slc", "sys", "title", "00050010", "1000400a", "code", + ], ] { create_dir_all(HostFilesystem::join_many(dir.path(), directory_to_create)) .expect("Failed to create directories necessary for host filesystem to work."); @@ -1048,6 +1572,13 @@ pub mod test_helpers { ], )) .expect("Failed to create needed fw.img!"); + File::create(HostFilesystem::join_many( + dir.path(), + [ + "data", "mlc", "sys", "title", "00050010", "1f700500", "code", "app.xml", + ], + )) + .expect("Failed to create needed app.xml for disc!"); let fs = HostFilesystem::from_cafe_dir(Some(PathBuf::from(dir.path()))) .await @@ -1375,7 +1906,6 @@ mod unit_tests { let fd = fs .open_folder(&path, None) - .await .expect("Failed to open existing folder!"); assert!( fs.open_folder_handles.len() == 1, @@ -1430,10 +1960,7 @@ mod unit_tests { .await .expect("Failed to create file to use!"); - let dfd = fs - .open_folder(&path, None) - .await - .expect("Failed to open file!"); + let dfd = fs.open_folder(&path, None).expect("Failed to open folder!"); assert!( fs.next_in_folder(dfd, None) .await @@ -1480,13 +2007,124 @@ mod unit_tests { fs.next_in_folder(dfd, None) .await .expect("Failed to query for next in folder! 2.2!") - .is_some() + .is_none() ); - assert!( - fs.next_in_folder(dfd, None) - .await - .expect("Failed to query for next in folder! 2.3!") - .is_some() + } + + #[test] + pub fn can_capitilize_ids() { + assert_eq!( + HostFilesystem::capitilize_title_id( + r#" + + 16 + 000500101000400A + 000500101f700500 + 090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"# + .to_owned() + ), + r#" + + 16 + 000500101000400A + 000500101F700500 + 090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"# + .to_owned(), + ); + + assert_eq!( + HostFilesystem::capitilize_title_id( + r#" + + 16 + 000500101000400A + 000500101F700500 + 090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"# + .to_owned() + ), + r#" + + 16 + 000500101000400A + 000500101F700500 + 090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"# + .to_owned(), + ); + + assert_eq!( + HostFilesystem::capitilize_title_id( + r#" + + 16 + 000500101000400A + 090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"# + .to_owned() + ), + r#" + + 16 + 000500101000400A + 090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"# + .to_owned(), + ); + + assert_eq!( + HostFilesystem::capitilize_title_id(r#" + + 16 + 000500101000400A000500101f700500090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"#.to_owned()), + r#" + + 16 + 000500101000400A000500101F700500090D + 21213 + 90000001 + 00000400 + 0 + 0000000000000000 +"#.to_owned(), ); } } diff --git a/pkg/cat-dev/src/fsemul/pcfs/errors.rs b/pkg/cat-dev/src/fsemul/pcfs/errors.rs index 9721f32..7e123f6 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/errors.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/errors.rs @@ -3,6 +3,7 @@ use miette::Diagnostic; use thiserror::Error; +#[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] use crate::fsemul::pcfs::sata::proto::SataQueryResponse; @@ -26,6 +27,7 @@ pub enum PCFSApiError { #[error("The requested path: [{0}] was not inside of a mapped directory, cannot serve.")] #[diagnostic(code(cat_dev::api::fsemul::pcfs::path_not_mapped))] PathNotMapped(String), + #[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] #[error( "The server was not configured correctly (programmer error), please report that extension: {0} did not load properly!" @@ -66,6 +68,7 @@ pub enum SataProtocolError { #[error("Unknown file location to move too: {0}")] #[diagnostic(code(cat_dev::net::parse::pcfs::sata::unknown_file_location))] UnknownFileLocation(u32), + #[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] #[error("Sata query response returned the wrong type of response: {0:?}")] #[diagnostic(code(cat_dev::net::parse::pcfs::sata::wrong_query_response_type))] diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/client.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/client.rs index a73c6ac..3c3de61 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/client.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/client.rs @@ -11,8 +11,8 @@ use crate::{ SataFileDescriptorResult, SataGetInfoByQueryPacketBody, SataOpenFilePacketBody, SataPacketHeader, SataPingPacketBody, SataPongBody, SataQueryResponse, SataQueryType, SataReadFilePacketBody, SataReadFolderPacketBody, SataRemovePacketBody, SataRequest, - SataResponse, SataResultCode, SataRewindFolderPacketBody, SataStatFilePacketBody, - SataWriteFilePacketBody, + SataResponse, SataResultCode, SataRewindFolderPacketBody, + SataSetFilePositionPacketBody, SataStatFilePacketBody, SataWriteFilePacketBody, }, }, net::{ @@ -24,7 +24,7 @@ use bytes::{Buf, Bytes, BytesMut}; use std::{ sync::{ Arc, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU32, Ordering}, }, time::Duration, }; @@ -41,6 +41,12 @@ pub struct SataClient { supports_csr: Arc, /// If we're acctively supporting "FFIO", or "Fast File I/O". supports_ffio: Arc, + /// Means something different to the official server, but for us means + /// when padding stops being added. + first_read_size: Arc, + /// Means something different to the official server, but for us means + /// when padding stops being added. + first_write_size: Arc, /// The TCP Client we're warpping. underlying_client: TCPClient, } @@ -55,6 +61,8 @@ impl SataClient { address: AddrTy, supports_csr: bool, supports_ffio: bool, + first_read_size: u32, + first_write_size: u32, trace_io_during_debug: bool, ) -> Result { let client = TCPClient::new( @@ -69,6 +77,8 @@ impl SataClient { supports_csr: Arc::new(AtomicBool::new(supports_csr)), supports_ffio: Arc::new(AtomicBool::new(supports_ffio)), underlying_client: client, + first_read_size: Arc::new(AtomicU32::new(first_read_size)), + first_write_size: Arc::new(AtomicU32::new(first_write_size)), }; this.ping(Some(DEFAULT_CLIENT_TIMEOUT)).await?; @@ -134,6 +144,10 @@ impl SataClient { } let mut req = Self::construct(0x14, SataPingPacketBody::new()); + req.command_info_mut().set_user(( + self.first_read_size.load(Ordering::Acquire), + self.first_write_size.load(Ordering::Acquire), + )); req.command_info_mut().set_capabilities((u32::MAX, 0)); req.header_mut().set_flags(flags.0); let (_stream_id, _req_id, opt_response) = self @@ -502,6 +516,7 @@ impl SataClient { /// - If we cannot receive a response from the PCFS Sata server. /// - If we cannot parse the response from the PCFS Sata server. /// - If the return code of the response was not successful. + /// - If you're requesting to read too much at once. async fn do_file_read( &self, block_count: u32, @@ -511,35 +526,55 @@ impl SataClient { timeout: Option, ) -> Result<(usize, Bytes), CatBridgeError> { if self.supports_ffio.load(Ordering::Acquire) { - // For FFIO our normal nagle split doesn't really work well. - let (_stream_id, _req_id, opt_response) = self - .underlying_client - .send_with_read_amount( - Self::construct( - 0x6, - SataReadFilePacketBody::new( - block_count, - block_size, - file_descriptor, - move_to, + let mut left_to_read = block_size * block_count; + + let mut file_size = 0_usize; + let mut final_body = BytesMut::with_capacity( + usize::try_from(left_to_read) + .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)?, + ); + while left_to_read > 0 { + // This keeps us within 'padding' range, which is easier to parse + // responses out of. + let read_in_this_go = std::cmp::min( + left_to_read, + self.first_read_size.load(Ordering::Acquire) - 0x25, + ); + let read_in_this_go_size = usize::try_from(read_in_this_go) + .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)?; + // For FFIO our normal nagle split doesn't really work well. + let (_stream_id, _req_id, opt_response) = self + .underlying_client + .send_with_read_amount( + Self::construct( + 0x6, + SataReadFilePacketBody::new( + 1, + read_in_this_go, + file_descriptor, + move_to, + ), ), - ), - Some(timeout.unwrap_or(DEFAULT_CLIENT_TIMEOUT)), - // 0x20 emptied PCFS header, 0x4 final file length + (block_size * block_count) - 0x20_usize - + 0x4_usize + (usize::try_from(block_size).unwrap_or(usize::MAX) - * usize::try_from(block_count).unwrap_or(usize::MAX)), - ) - .await?; - - let mut full_body = opt_response - .ok_or(NetworkError::ExpectedData)? - .take_body() - .ok_or(NetworkError::ExpectedData)?; - // Remove 'blank' header. - full_body.advance(0x20); - let file_size = usize::try_from(full_body.get_u32()).unwrap_or(usize::MAX); - Ok((file_size, full_body)) + Some(timeout.unwrap_or(DEFAULT_CLIENT_TIMEOUT)), + // 0x20 emptied PCFS header, 0x4 final file length + (block_size * block_count) + 0x20_usize + 0x4_usize + read_in_this_go_size, + ) + .await?; + + let mut full_body = opt_response + .ok_or(NetworkError::ExpectedData)? + .take_body() + .ok_or(NetworkError::ExpectedData)?; + // Remove 'blank' header. + full_body.advance(0x20); + if file_size == 0 { + file_size = usize::try_from(full_body.get_u32()).unwrap_or(usize::MAX); + } + final_body.extend(full_body); + left_to_read -= read_in_this_go; + } + + Ok((file_size, final_body.freeze())) } else { todo!("Implement non-FFIO file support.") } @@ -611,6 +646,34 @@ impl SataClient { } } + async fn do_file_move( + &self, + file_descriptor: i32, + move_to: MoveToFileLocation, + timeout: Option, + ) -> Result<(), CatBridgeError> { + let resp = self + .underlying_client + .send( + Self::construct( + 0x9, + SataSetFilePositionPacketBody::new(file_descriptor, move_to), + ), + Some(timeout.unwrap_or(DEFAULT_CLIENT_TIMEOUT)), + ) + .await? + .2 + .ok_or(NetworkError::ExpectedData)? + .take_body() + .ok_or(NetworkError::ExpectedData)?; + + let sata_resp = SataResponse::::try_from(resp)?; + if sata_resp.body().0 != 0 { + return Err(NetworkParseError::ErrorCode(sata_resp.body().0).into()); + } + Ok(()) + } + async fn close_file( &self, file_descriptor: i32, @@ -792,6 +855,24 @@ impl SataClientFileHandle<'_> { .await } + /// Move to a new location within this file. + /// + /// ## Errors + /// + /// - If we cannot send the request to the PCFS Sata server. + /// - If we cannot receive a response from the PCFS Sata server. + /// - If we cannot parse the response from the PCFS Sata server. + /// - If the return code of the response was not successful. + pub async fn move_to( + &self, + move_to: MoveToFileLocation, + timeout: Option, + ) -> Result<(), CatBridgeError> { + self.underlying_client + .do_file_move(self.file_descriptor, move_to, timeout) + .await + } + /// Write N amounts of bytes from a file. /// /// You can optionally move the file pointer around before doing the diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/mod.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/mod.rs index 7dcd7bd..ce33d26 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/mod.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/mod.rs @@ -3,9 +3,12 @@ //! The custom sata protocol is the default for PCFS, and implements a real //! filesystem such as "Create File"/"Open File"/etc. +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub mod client; +#[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] pub mod proto; +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] pub mod server; diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/proto/get_info_by_query.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/get_info_by_query.rs index bed6218..351485c 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/proto/get_info_by_query.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/get_info_by_query.rs @@ -229,7 +229,7 @@ impl TryFrom for SataQueryType { /// just return a very basic "size" (e.g. file count, or file length, etc.). /// However the file stat query type actually returns all the information about /// a particular path. -#[derive(Debug, Valuable, PartialEq, Eq)] +#[derive(Clone, Debug, Valuable, PartialEq, Eq)] pub enum SataQueryResponse { /// An error has occured, and we are returning an error code. ErrorCode(u32), @@ -418,6 +418,24 @@ impl SataFDInfo { } } + /// Create a fake fd info from totally controlled values. + #[must_use] + pub fn create_fake_info( + file_or_folder_flags: u32, + perms: u32, + file_length: u32, + created_timestamp: u64, + updated_timestamp: u64, + ) -> Self { + Self { + file_or_folder_flags, + perms, + file_length, + created_timestamp, + last_updated_timestamp: updated_timestamp, + } + } + /// Get the raw file or folder type flags for a particular path. #[must_use] pub const fn flags(&self) -> u32 { diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/proto/mod.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/mod.rs index dafc927..fc6d03f 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/proto/mod.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/mod.rs @@ -16,13 +16,18 @@ mod ping; mod read_file; mod read_folder; mod remove; +mod rename; mod rewind_folder; +mod set_file_position; mod stat_file; mod write_file; use crate::{ - errors::{CatBridgeError, NetworkError, NetworkParseError}, - fsemul::pcfs::errors::{PCFSApiError, SataProtocolError}, + errors::{CatBridgeError, FSError, NetworkError, NetworkParseError}, + fsemul::{ + HostFilesystem, + pcfs::errors::{PCFSApiError, SataProtocolError}, + }, net::models::{IntoResponse, Response}, }; use bytes::{BufMut, Bytes, BytesMut}; @@ -35,7 +40,7 @@ use valuable::{Fields, NamedField, NamedValues, StructDef, Structable, Valuable, pub use crate::fsemul::pcfs::sata::proto::{ change_mode::*, change_owner::*, close_file::*, close_folder::*, create_folder::*, get_info_by_query::*, open_file::*, open_folder::*, ping::*, read_file::*, read_folder::*, - remove::*, rewind_folder::*, stat_file::*, write_file::*, + remove::*, rename::*, rewind_folder::*, set_file_position::*, stat_file::*, write_file::*, }; /// The Default PCFS Version we claim to be. @@ -439,6 +444,35 @@ pub enum MoveToFileLocation { End, } +impl MoveToFileLocation { + /// Actually seek around a particular file descriptor on a host filesystem. + /// + /// ## Errors + /// + /// If we do end up calling seek file, and the host filesystem seek files + /// throws an error. + pub async fn do_move( + &self, + host_fs: &HostFilesystem, + handle: i32, + stream_id: Option, + ) -> Result<(), FSError> { + match *self { + MoveToFileLocation::Begin => { + host_fs.seek_file(handle, true, stream_id).await?; + } + MoveToFileLocation::Current => { + // Luckily to move to current, we don't need to move at all. + } + MoveToFileLocation::End => { + host_fs.seek_file(handle, false, stream_id).await?; + } + } + + Ok(()) + } +} + impl From<&MoveToFileLocation> for u32 { fn from(value: &MoveToFileLocation) -> u32 { match *value { @@ -468,6 +502,16 @@ impl TryFrom for MoveToFileLocation { } } +impl Display for MoveToFileLocation { + fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult { + match self { + Self::Begin => write!(fmt, "Begin"), + Self::Current => write!(fmt, "Nowhere"), + Self::End => write!(fmt, "End"), + } + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Valuable)] #[repr(transparent)] pub struct SataCapabilitiesFlags(pub u32); @@ -640,6 +684,8 @@ pub struct SataResponse { flags: u32, /// The process id to report. pid: u32, + /// Whether or not to force a '0' as our version. + force_zero_version: bool, /// The body of the rsponse. body: InnerTy, } @@ -653,6 +699,18 @@ impl SataResponse { flags: 0, pid, body, + force_zero_version: false, + } + } + + #[must_use] + pub const fn new_force_zero_version(pid: u32, header: SataPacketHeader, body: InnerTy) -> Self { + Self { + header, + flags: 0, + pid, + body, + force_zero_version: true, } } @@ -668,6 +726,7 @@ impl SataResponse { flags, pid, body, + force_zero_version: false, } } @@ -681,6 +740,7 @@ impl SataResponse { flags, pid: host_pid, body, + force_zero_version: false, } } @@ -752,6 +812,7 @@ where flags, pid, body, + force_zero_version: false, }) } } @@ -772,11 +833,15 @@ where ); new_buff.put_u32(value.header.id()); new_buff.put_u32(value.flags); - new_buff.put_u32(if value.header.version() != 0 { - value.header.version() + if value.force_zero_version { + new_buff.put_u32(0); } else { - DEFAULT_PCFS_VERSION - }); + new_buff.put_u32(if value.header.version() != 0 { + value.header.version() + } else { + DEFAULT_PCFS_VERSION + }); + } // Calculate epoch, and wrap around... new_buff.put_u32( u32::try_from( @@ -896,6 +961,16 @@ impl TryFrom for SataResultCode { } } +impl Display for SataResultCode { + fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult { + if self.0 == 0 { + write!(fmt, "Success") + } else { + write!(fmt, "Failure ({:02x})", self.0) + } + } +} + /// A file descriptor from a sata endpoint that optionally includes a single /// result code in back. #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Valuable)] @@ -990,6 +1065,15 @@ impl TryFrom for SataFileDescriptorResult { } } +impl Display for SataFileDescriptorResult { + fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult { + match self.file_descriptor { + Ok(fd) => write!(fmt, "Success ({fd})"), + Err(rc) => write!(fmt, "Failure ({rc:02x})"), + } + } +} + #[cfg(test)] mod unit_tests { use super::*; diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/proto/rename.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/rename.rs new file mode 100644 index 0000000..ba93e0f --- /dev/null +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/rename.rs @@ -0,0 +1,171 @@ +//! Definitions for the `Rename` packet type, and it's response types. +//! +//! This moves a file from one place to another as necessary. + +use crate::{errors::NetworkParseError, fsemul::pcfs::errors::PCFSApiError}; +use bytes::{Bytes, BytesMut}; +use std::ffi::CStr; +use valuable::{Fields, NamedField, NamedValues, StructDef, Structable, Valuable, Value, Visit}; + +/// A packet to rename an arbitrary path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SataRenamePacketBody { + /// The source path which will be renamed. + source_path: String, + /// The path, aka where the final path will be. + dest_path: String, +} + +impl SataRenamePacketBody { + /// Attempt to construct a new rename packet body. + /// + /// ## Errors + /// + /// If any path is longer than 511 bytes. Normally the max path is 512 bytes, + /// but because we need to encode our data as a C-String with a NUL + /// terminator we cannot be longer than 511 bytes. + /// + /// Consider using relative/mapped paths if possible when dealing with long + /// paths. + pub fn new(source_path: String, dest_path: String) -> Result { + if source_path.len() > 511 { + return Err(PCFSApiError::PathTooLong(source_path)); + } + if dest_path.len() > 511 { + return Err(PCFSApiError::PathTooLong(dest_path)); + } + + Ok(Self { + source_path, + dest_path, + }) + } + + #[must_use] + pub fn source_path(&self) -> &str { + self.source_path.as_str() + } + + #[must_use] + pub fn dest_path(&self) -> &str { + self.dest_path.as_str() + } + + /// Update the source path to send in this rename packet directory. + /// + /// ## Errors + /// + /// If the path is longer than 511 bytes. Normally the max path is 512 bytes, + /// but because we need to encode our data as a C-String with a NUL + /// terminator we cannot be longer than 511 bytes. + /// + /// Consider using relative/mapped paths if possible when dealing with long + /// paths. + pub fn set_source_path(&mut self, new_path: String) -> Result<(), PCFSApiError> { + if new_path.len() > 511 { + return Err(PCFSApiError::PathTooLong(new_path)); + } + + self.source_path = new_path; + Ok(()) + } + + /// Update the destination path to send in this rename packet directory. + /// + /// ## Errors + /// + /// If the path is longer than 511 bytes. Normally the max path is 512 bytes, + /// but because we need to encode our data as a C-String with a NUL + /// terminator we cannot be longer than 511 bytes. + /// + /// Consider using relative/mapped paths if possible when dealing with long + /// paths. + pub fn set_dest_path(&mut self, new_path: String) -> Result<(), PCFSApiError> { + if new_path.len() > 511 { + return Err(PCFSApiError::PathTooLong(new_path)); + } + + self.dest_path = new_path; + Ok(()) + } +} + +impl TryFrom for SataRenamePacketBody { + type Error = NetworkParseError; + + fn try_from(value: Bytes) -> Result { + if value.len() < 0x400 { + return Err(NetworkParseError::FieldNotLongEnough( + "SataRenamePacketBody", + "Body", + 0x400, + value.len(), + value, + )); + } + if value.len() > 0x400 { + return Err(NetworkParseError::UnexpectedTrailer( + "SataRenamePacketBody", + value.slice(0x400..), + )); + } + + let source_path_c_str = + CStr::from_bytes_until_nul(&value[..0x200]).map_err(NetworkParseError::BadCString)?; + let dest_path_c_str = + CStr::from_bytes_until_nul(&value[0x200..]).map_err(NetworkParseError::BadCString)?; + + Ok(Self { + source_path: source_path_c_str.to_str()?.to_owned(), + dest_path: dest_path_c_str.to_str()?.to_owned(), + }) + } +} + +impl From<&SataRenamePacketBody> for Bytes { + fn from(value: &SataRenamePacketBody) -> Self { + let mut result = BytesMut::with_capacity(0x400); + result.extend_from_slice(value.source_path.as_bytes()); + // These are C Strings so we need a NUL terminator. + // Pad with `0`, til we get a full path with a nul terminator. + result.extend(BytesMut::zeroed(0x200 - result.len())); + result.extend_from_slice(value.dest_path.as_bytes()); + // Pad again.... + result.extend(BytesMut::zeroed(0x400 - result.len())); + result.freeze() + } +} + +impl From for Bytes { + fn from(value: SataRenamePacketBody) -> Self { + Self::from(&value) + } +} + +const SATA_RENAME_PACKET_BODY_FIELDS: &[NamedField<'static>] = + &[NamedField::new("source_path"), NamedField::new("dest_path")]; + +impl Structable for SataRenamePacketBody { + fn definition(&self) -> StructDef<'_> { + StructDef::new_static( + "SataRenamePacketBody", + Fields::Named(SATA_RENAME_PACKET_BODY_FIELDS), + ) + } +} + +impl Valuable for SataRenamePacketBody { + fn as_value(&self) -> Value<'_> { + Value::Structable(self) + } + + fn visit(&self, visitor: &mut dyn Visit) { + visitor.visit_named_fields(&NamedValues::new( + SATA_RENAME_PACKET_BODY_FIELDS, + &[ + Valuable::as_value(&self.source_path), + Valuable::as_value(&self.dest_path), + ], + )); + } +} diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/proto/set_file_position.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/set_file_position.rs new file mode 100644 index 0000000..cb6b036 --- /dev/null +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/proto/set_file_position.rs @@ -0,0 +1,122 @@ +//! Definitions for the `SetFilePosition` packet type, and it's response types. +//! +//! This can change the location of an already open file, similar to what you'd +//! do before reading a file or writing a file. + +use crate::{errors::NetworkParseError, fsemul::pcfs::sata::proto::MoveToFileLocation}; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use valuable::{Fields, NamedField, NamedValues, StructDef, Structable, Valuable, Value, Visit}; + +/// A packet to set the position of an already open file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SataSetFilePositionPacketBody { + handle: i32, + move_to_pointer: MoveToFileLocation, +} + +impl SataSetFilePositionPacketBody { + /// Create a new set file position packet. + #[must_use] + pub const fn new(file_descriptor: i32, move_to: MoveToFileLocation) -> Self { + Self { + handle: file_descriptor, + move_to_pointer: move_to, + } + } + + #[must_use] + pub const fn file_descriptor(&self) -> i32 { + self.handle + } + + pub const fn set_file_descriptor(&mut self, new_fd: i32) { + self.handle = new_fd; + } + + #[must_use] + pub const fn move_to_pointer(&self) -> MoveToFileLocation { + self.move_to_pointer + } + + pub const fn set_move_to_pointer(&mut self, new_move: MoveToFileLocation) { + self.move_to_pointer = new_move; + } +} + +impl From<&SataSetFilePositionPacketBody> for Bytes { + fn from(value: &SataSetFilePositionPacketBody) -> Self { + let mut buff = BytesMut::with_capacity(8); + + buff.put_i32(value.handle); + buff.put_u32(u32::from(value.move_to_pointer)); + + buff.freeze() + } +} + +impl From for Bytes { + fn from(value: SataSetFilePositionPacketBody) -> Self { + Self::from(&value) + } +} + +impl TryFrom for SataSetFilePositionPacketBody { + type Error = NetworkParseError; + + fn try_from(mut value: Bytes) -> Result { + if value.len() < 8 { + return Err(NetworkParseError::FieldNotLongEnough( + "SataSetFilePosition", + "Body", + 8, + value.len(), + value, + )); + } + // Packets _can_ come with an extra padded 4 bytes... + if value.len() > 12 { + return Err(NetworkParseError::UnexpectedTrailer( + "SataSetFilePosition", + value.slice(12..), + )); + } + + let handle = value.get_i32(); + let move_to_ptr = value.get_u32(); + + Ok(Self { + handle, + move_to_pointer: MoveToFileLocation::try_from(move_to_ptr)?, + }) + } +} + +const SATA_SET_FILE_POSITION_PACKET_BODY_FIELDS: &[NamedField<'static>] = &[ + NamedField::new("handle"), + NamedField::new("move_to_pointer"), +]; + +impl Structable for SataSetFilePositionPacketBody { + fn definition(&self) -> StructDef<'_> { + StructDef::new_static( + "SataSetFilePositionPacketBody", + Fields::Named(SATA_SET_FILE_POSITION_PACKET_BODY_FIELDS), + ) + } +} + +impl Valuable for SataSetFilePositionPacketBody { + fn as_value(&self) -> Value<'_> { + Value::Structable(self) + } + + fn visit(&self, visitor: &mut dyn Visit) { + visitor.visit_named_fields(&NamedValues::new( + SATA_SET_FILE_POSITION_PACKET_BODY_FIELDS, + &[ + Valuable::as_value(&self.handle), + Valuable::as_value(&self.move_to_pointer), + ], + )); + } +} diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/change_mode.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/change_mode.rs index 1a5fd8a..7e80c3c 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/change_mode.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/change_mode.rs @@ -103,13 +103,14 @@ pub async fn handle_change_mode( .host_filesystem() .ensure_folder_not_read_only(fs_location.closest_resolved_path()) .await; - Ok(()) } else { state .host_filesystem() .mark_folder_read_only(fs_location.closest_resolved_path().clone()) - .await + .await; } + + Ok(()) } else { set_permissions(fs_location.closest_resolved_path(), perms).map_err(FSError::IO) }; diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/connection_flags.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/connection_flags.rs index 46a524a..10958f1 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/connection_flags.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/connection_flags.rs @@ -41,6 +41,9 @@ pub struct SataConnectionFlags { fast_file_io_enabled: Arc, combined_send_recv_enabled: Arc, version: Arc, + first_read_size: Arc, + first_write_size: Arc, + ffio_buffer_should_have_grown: Arc, } impl SataConnectionFlags { @@ -50,6 +53,9 @@ impl SataConnectionFlags { fast_file_io_enabled: Arc::new(AtomicBool::new(true)), combined_send_recv_enabled: Arc::new(AtomicBool::new(true)), version: Arc::new(AtomicU32::new(DEFAULT_PCFS_VERSION)), + first_read_size: Arc::new(AtomicU32::new(196_672)), + first_write_size: Arc::new(AtomicU32::new(196_640)), + ffio_buffer_should_have_grown: Arc::new(AtomicBool::new(false)), } } @@ -59,6 +65,9 @@ impl SataConnectionFlags { fast_file_io_enabled: Arc::new(AtomicBool::new(ffio_enabled)), combined_send_recv_enabled: Arc::new(AtomicBool::new(csr_enabled)), version: Arc::new(AtomicU32::new(DEFAULT_PCFS_VERSION)), + first_read_size: Arc::new(AtomicU32::new(196_672)), + first_write_size: Arc::new(AtomicU32::new(196_640)), + ffio_buffer_should_have_grown: Arc::new(AtomicBool::new(false)), } } @@ -81,15 +90,42 @@ impl SataConnectionFlags { .store(enabled, Ordering::Release); } - #[allow(unused)] + #[must_use] pub fn version(&self) -> u32 { self.version.load(Ordering::Acquire) } - #[allow(unused)] pub fn set_version(&self, version_num: u32) { self.version.store(version_num, Ordering::Release); } + + #[must_use] + pub fn first_read_size(&self) -> u32 { + self.first_read_size.load(Ordering::Acquire) + } + + pub fn set_first_read_size(&self, new_size: u32) { + self.first_read_size.store(new_size, Ordering::Release); + } + + #[must_use] + pub fn first_write_size(&self) -> u32 { + self.first_write_size.load(Ordering::Acquire) + } + + pub fn set_first_write_size(&self, new_size: u32) { + self.first_write_size.store(new_size, Ordering::Release); + } + + #[must_use] + pub fn ffio_buffer_should_have_grown(&self) -> bool { + self.ffio_buffer_should_have_grown.load(Ordering::Acquire) + } + + pub fn set_ffio_buffer_should_have_grown(&self, did_grow: bool) { + self.ffio_buffer_should_have_grown + .store(did_grow, Ordering::Release); + } } impl Default for SataConnectionFlags { diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/create_folder.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/create_folder.rs index eb53e31..bb82a05 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/create_folder.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/create_folder.rs @@ -77,18 +77,11 @@ pub async fn handle_create_folder( .host_filesystem() .ensure_folder_not_read_only(fs_location.resolved_path()) .await; - } else if let Err(cause) = state - .host_filesystem() - .mark_folder_read_only(fs_location.resolved_path().clone()) - .await - { - error!( - ?cause, - path = %fs_location.resolved_path().display(), - "Failed to mark directory as read-only for PCFS.", - ); - - return SataResponse::new(state.pid(), request_header, SataResultCode::error(FS_ERROR)); + } else { + state + .host_filesystem() + .mark_folder_read_only(fs_location.resolved_path().clone()) + .await; } SataResponse::new(state.pid(), request_header, SataResultCode::success()) diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/info_by_query.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/info_by_query.rs index 7d75825..5d7fdf4 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/info_by_query.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/info_by_query.rs @@ -47,20 +47,38 @@ pub async fn handle_get_info_by_query( let info_request = request.body(); let header = request.header().clone(); - let Ok(final_location) = state.host_filesystem().resolve_path(info_request.path()) else { + let fs = state.host_filesystem(); + let Ok(final_location) = fs.resolve_path(info_request.path()) else { return SataResponse::new( state.pid(), header, SataQueryResponse::ErrorCode(PATH_NOT_EXIST_ERROR), ); }; + if request.command_info().user().0 == 0x1000_00FC + && request.command_info().user().1 == 0x1000_00FF + { + let cloned_location = final_location.clone(); + let ResolvedLocation::Filesystem(fs_location) = cloned_location else { + todo!("network shares not yet implemented!") + }; + let resolved_path = fs_location.resolved_path(); + + if fs.path_allows_writes(resolved_path) + && resolved_path.extension().is_none() + && !resolved_path.exists() + { + // Ignore any errors, file details or otherwise will properly error out. + _ = fs.create_directory(resolved_path); + } + } match info_request.query_type() { SataQueryType::FreeDiskSpace => handle_disk_space(state.pid(), header, final_location), SataQueryType::SizeOfFolder => handle_folder_size(state.pid(), header, final_location), SataQueryType::FileCount => handle_file_count(state.pid(), header, final_location), SataQueryType::FileDetails => { - handle_file_info(state.pid(), header, state.host_filesystem(), final_location).await + handle_file_info(state.pid(), header, fs, final_location).await } } } @@ -120,6 +138,7 @@ fn handle_disk_space( // needs to not be on the network.... let ResolvedLocation::Filesystem(fs_location) = location else { debug!( + packet.location = valuable(&location), packet.typ = "PCFSSrvGetInfo", packet.sub_type = "handle_disk_space", "Failed to resolve path!", @@ -198,6 +217,7 @@ fn handle_folder_size( // because it doesn't exist. if !fs_location.canonicalized_is_exact() { debug!( + packet.location = valuable(&fs_location), packet.typ = "PCFSSrvGetInfo", packet.sub_type = "handle_folder_size", "Failed to resolve path!", @@ -272,6 +292,7 @@ fn handle_file_count( // because it doesn't exist. if !fs_location.canonicalized_is_exact() { debug!( + packet.location = valuable(&fs_location), packet.typ = "PCFSSrvGetInfo", packet.sub_type = "handle_file_count", "Failed to resolve path!", @@ -352,6 +373,7 @@ async fn handle_file_info( ResolvedLocation::Filesystem(ref filesystem) => { let Ok(metadata) = filesystem.resolved_path().metadata() else { debug!( + packet.location = valuable(&location), packet.typ = "PCFSSrvGetInfo", packet.sub_type = "handle_file_info", "Failed to resolve path!", @@ -363,9 +385,21 @@ async fn handle_file_info( SataQueryResponse::ErrorCode(PATH_NOT_EXIST_ERROR), ); }; + if &fs.disc_emu_path() == filesystem.resolved_path() { + return SataResponse::new( + pid, + request_header, + SataQueryResponse::FDInfo( + // Yes, i know this claims to be an empty file, but this is + // legitimately how the official software responds. + SataFDInfo::create_fake_info(0x8000_0000, 0x666, 0, 0, 0), + ), + ); + } let info = SataFDInfo::get_info(fs, &metadata, filesystem.resolved_path()).await; debug!( + packet.location = valuable(&location), packet.typ = "PCFSSrvGetInfo", packet.sub_type = "handle_file_info", packet.result = valuable(&info), diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/mod.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/mod.rs index 17e6e11..df3e4da 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/mod.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/mod.rs @@ -4,7 +4,7 @@ mod change_mode; mod change_owner; mod close_file; mod close_folder; -mod connection_flags; +pub mod connection_flags; mod create_folder; mod info_by_query; mod open_file; @@ -13,7 +13,10 @@ mod ping; mod read_file; mod read_folder; mod remove; +mod rename; mod rewind_folder; +mod set_file_position; +pub mod wal; mod write_file; use crate::{ @@ -22,8 +25,14 @@ use crate::{ HostFilesystem, pcfs::sata::{ proto::SataRequest, - server::connection_flags::{ - SATA_CONNECTION_FLAGS, SataConnectionFlags, SataConnectionFlagsLayer, + server::{ + connection_flags::{ + SATA_CONNECTION_FLAGS, SataConnectionFlags, SataConnectionFlagsLayer, + }, + wal::{ + WriteAheadLog, + layer::{WALBeginStreamLayer, WALEndStreamLayer, WALMessageLayer}, + }, }, }, }, @@ -38,6 +47,7 @@ use bytes::Bytes; use local_ip_address::local_ip; use std::{ net::{IpAddr, Ipv4Addr, SocketAddrV4}, + path::PathBuf, time::Duration, }; use tower::ServiceBuilder; @@ -118,6 +128,7 @@ pub async fn pcfs_sata_server( disable_ffio: bool, disable_csr: bool, disable_real_removal: bool, + sata_wal_location: Option, cat_dev_sleep_override: Option, fully_disable_cat_dev_sleep: bool, chunk_override: Option, @@ -145,9 +156,14 @@ pub async fn pcfs_sata_server( router.add_route(&0x5_u32.to_be_bytes(), open_file::handle_open_file)?; router.add_route(&0x6_u32.to_be_bytes(), read_file::handle_read_file)?; router.add_route(&0x7_u32.to_be_bytes(), write_file::handle_write_file)?; + router.add_route( + &0x9_u32.to_be_bytes(), + set_file_position::handle_set_file_position, + )?; router.add_route(&0xB_u32.to_be_bytes(), info_by_query::stat_fd)?; router.add_route(&0xD_u32.to_be_bytes(), close_file::handle_close_file)?; router.add_route(&0xE_u32.to_be_bytes(), remove::handle_removal)?; + router.add_route(&0xF_u32.to_be_bytes(), rename::handle_rename)?; router.add_route( &0x10_u32.to_be_bytes(), info_by_query::handle_get_info_by_query, @@ -168,6 +184,11 @@ pub async fn pcfs_sata_server( ) .await?; + let mut wal = match sata_wal_location { + Some(p) => WriteAheadLog::new(p).ok(), + None => None, + }; + server.set_on_stream_begin(async move |event: ResponseStreamEvent| { let sid = event.stream_id(); @@ -180,13 +201,30 @@ pub async fn pcfs_sata_server( Ok(true) })?; + if let Some(w) = wal.as_ref() { + server.layer_on_stream_begin(WALBeginStreamLayer(w.clone()))?; + } server.set_on_stream_end(on_sata_stream_end)?; - server.layer_initial_service( - ServiceBuilder::new() - .layer(RequestIDLayer) - .layer(StreamIDLayer) - .layer(SataConnectionFlagsLayer), - ); + if let Some(w) = wal.as_ref() { + server.layer_on_stream_end(WALEndStreamLayer(w.clone()))?; + } + + if let Some(w) = wal.take() { + server.layer_initial_service( + ServiceBuilder::new() + .layer(RequestIDLayer) + .layer(StreamIDLayer) + .layer(SataConnectionFlagsLayer) + .layer(WALMessageLayer(w)), + ); + } else { + server.layer_initial_service( + ServiceBuilder::new() + .layer(RequestIDLayer) + .layer(StreamIDLayer) + .layer(SataConnectionFlagsLayer), + ); + } server.set_chunk_output_at_size(if fully_disable_chunk_override { None @@ -205,13 +243,18 @@ pub async fn pcfs_sata_server( } async fn unknown_packet_handler(Body(request): Body) -> Response { - if let Ok(req) = SataRequest::::parse_opaque(request) { + if let Ok(req) = SataRequest::::parse_opaque(request.clone()) { warn!( header = valuable(req.header()), command_info = valuable(req.command_info()), body = format!("{:02X?}", req.body()), "Unknown PCFS Sata packet!", ); + } else { + warn!( + packet = format!("{:02X?}", request), + "Unknown Unparsable PCFS Sata Packet!", + ); } Response::empty_close() diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_file.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_file.rs index 21d7723..2323e5e 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_file.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_file.rs @@ -73,17 +73,21 @@ pub async fn handle_open_file( // Okay time to open! let mut options = OpenOptions::new(); + let mut will_create = false; if mode.contains('r') { options.read(true); } if mode.contains('w') { options.write(true).truncate(true).create(true); + will_create = true; } if mode.contains('a') { options.write(true).truncate(false).create(true); + will_create = true; } if mode.contains('+') { options.create(true); + will_create = true; } let fd = match state @@ -100,10 +104,18 @@ pub async fn handle_open_file( "Failed to open file!", ); + if fs_location.resolved_path().exists() || will_create { + return SataResponse::new( + state.pid(), + request_header, + SataFileDescriptorResult::error(FS_ERROR), + ); + } + return SataResponse::new( state.pid(), request_header, - SataFileDescriptorResult::error(FS_ERROR), + SataFileDescriptorResult::error(PATH_NOT_EXIST_ERROR), ); } }; diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_folder.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_folder.rs index 33ce9c3..baf69d3 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_folder.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/open_folder.rs @@ -59,7 +59,6 @@ pub async fn handle_open_folder( let Ok(fd) = state .host_filesystem() .open_folder(fs_location.resolved_path(), Some(stream.to_raw())) - .await else { debug!( packet.path = packet.path(), diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/ping.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/ping.rs index 0e86afd..2b1002a 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/ping.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/ping.rs @@ -43,6 +43,8 @@ pub async fn handle_ping( flags.set_csr_enabled(false); flags.set_ffio_enabled(false); } + flags.set_first_read_size(command_info.user().0); + flags.set_first_write_size(command_info.user().1); SataResponse::new( pid, diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_file.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_file.rs index 9ba321e..fd4b9e7 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_file.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_file.rs @@ -5,7 +5,7 @@ use crate::{ fsemul::{ HostFilesystem, pcfs::sata::{ - proto::{MoveToFileLocation, SataReadFilePacketBody, SataRequest}, + proto::{SataReadFilePacketBody, SataRequest}, server::SataConnectionFlags, }, }, @@ -36,49 +36,26 @@ pub async fn handle_read_file( let packet = request.body(); let handle = packet.file_descriptor(); let ffio_enabled = flags.ffio_enabled(); + let mut buffer_grew = flags.ffio_buffer_should_have_grown(); if packet.should_move() { - match packet.move_to_pointer() { - MoveToFileLocation::Begin => { - if fs - .seek_file(handle, true, Some(stream.to_raw())) - .await - .is_err() - { - debug!( - packet.fd = handle, - packet.typ = "PCFSSrvReadFile", - "Failed to seek to beginning of file!", - ); - - if ffio_enabled { - return Ok(construct_ffio_error(FS_ERROR)); - } - - todo!("Implement non-FFIO support."); - } - } - MoveToFileLocation::Current => { - // Luckily to move to current, we don't need to move at all. - } - MoveToFileLocation::End => { - if fs - .seek_file(handle, false, Some(stream.to_raw())) - .await - .is_err() - { - debug!( - packet.fd = handle, - packet.typ = "PCFSSrvReadFile", - "Failed to seek to end of file of file!", - ); - - if ffio_enabled { - return Ok(construct_ffio_error(FS_ERROR)); - } - todo!("Implement non-FFIO support."); - } + if let Err(cause) = packet + .move_to_pointer() + .do_move(&fs, handle, Some(stream.to_raw())) + .await + { + debug!( + ?cause, + packet.fd = handle, + packet.typ = "PCFSSrvReadFile", + "Failed to move file to a specific pointer!", + ); + + if ffio_enabled { + return Ok(construct_ffio_error(FS_ERROR)); } + + todo!("Implement non-FFIO support."); } } @@ -94,15 +71,15 @@ pub async fn handle_read_file( } todo!("Implement non-ffio support."); }; + + let first_read_size = usize::try_from(flags.first_read_size()) + .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)?; + let total_read_amount = usize::try_from(packet.block_size()) + .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)? + * usize::try_from(packet.block_count()) + .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)?; let Ok(Some(read_file)) = fs - .read_file( - handle, - usize::try_from(packet.block_size()) - .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)? - * usize::try_from(packet.block_count()) - .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)?, - Some(stream.to_raw()), - ) + .read_file(handle, total_read_amount, Some(stream.to_raw())) .await else { debug!( @@ -118,15 +95,30 @@ pub async fn handle_read_file( }; if ffio_enabled { - let mut buff = BytesMut::with_capacity(read_file.len() + 0x24); - // The header is normally just 'malloc'd and not cleared between - // buffers. Luckily for us we can just zero it out, and it's easier than - // actually dealing with whatever random bytes PCFSServer would normally - // send. - buff.extend_from_slice(&[0; 0x20]); - buff.put_u32(u32::try_from(file_size).unwrap_or(u32::MAX)); + let mut buff = BytesMut::with_capacity(0x24 + read_file.len()); + + if !buffer_grew && read_file.len() > first_read_size { + flags.set_ffio_buffer_should_have_grown(true); + buffer_grew = true; + } + if (read_file.len() < total_read_amount && read_file.len() > first_read_size) || buffer_grew + { + buff.extend_from_slice(&[0; 0x20]); + buff.put_u32(u32::try_from(read_file.len()).unwrap_or(u32::MAX)); + } else { + buff.extend_from_slice(&[0xC4, 0x00, 0x24, 0x02, 0xE8, 0xEF, 0x24, 0x02]); + buff.extend_from_slice(&[0; 0x18]); + buff.put_u32(u32::try_from(file_size).unwrap_or(u32::MAX)); + } + + let rf_len = read_file.len(); buff.extend(read_file); + if rf_len < total_read_amount && rf_len < first_read_size { + let pad_amount = std::cmp::min(total_read_amount, first_read_size) - rf_len; + buff.extend(vec![0xCD; pad_amount]); + } + Ok(buff.freeze()) } else { todo!("Implement non-FFIO support.") @@ -136,7 +128,7 @@ pub async fn handle_read_file( fn construct_ffio_error(error_code: u32) -> Bytes { let mut buff = BytesMut::with_capacity(36); buff.extend(&[0xC4, 0x00, 0xFE, 0x00, 0x20, 0xEF, 0xFE, 0x00]); - buff.extend([0; 24]); + buff.extend([0; 0x18]); buff.put_u32(error_code); buff.freeze() } @@ -170,10 +162,13 @@ mod unit_tests { .expect("Failed to open file!"); let read_request = SataReadFilePacketBody::new(4, 1, fd, None); + let conn_flags = SataConnectionFlags::new_with_flags(true, true); + conn_flags.set_first_read_size(1000); + conn_flags.set_first_write_size(1000); let response = handle_read_file( StreamID::from_existing(1), - SataConnectionFlags::new_with_flags(true, true), + conn_flags, State(fs), Body(SataRequest::new( SataPacketHeader::new(0), @@ -186,7 +181,8 @@ mod unit_tests { let mut expected_response = BytesMut::new(); // Header - expected_response.extend_from_slice(&[0; 0x20]); + expected_response.extend_from_slice(&[0xC4, 0x00, 0x24, 0x02, 0xE8, 0xEF, 0x24, 0x02]); + expected_response.extend_from_slice(&[0; 0x18]); // File length. expected_response.extend_from_slice(&2_u32.to_be_bytes()); // File data, and padding. diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_folder.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_folder.rs index e4dbfc6..78d3631 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_folder.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/read_folder.rs @@ -133,7 +133,6 @@ mod unit_tests { let dfd = fs .open_folder(&base_dir, Some(1)) - .await .expect("Failed to open existing directory!"); let request = SataReadFolderPacketBody::new(dfd); diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/remove.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/remove.rs index cfd046f..fe1aef0 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/remove.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/remove.rs @@ -1,7 +1,6 @@ //! Handle remove packets which can remove files or folders. use crate::{ - errors::FSError, fsemul::{ host_filesystem::ResolvedLocation, pcfs::sata::{ @@ -11,13 +10,9 @@ use crate::{ }, net::server::requestable::{Body, State}, }; -use std::{ - ffi::{OsStr, OsString}, - path::PathBuf, -}; -use tokio::fs::{create_dir_all, read_link, remove_dir_all, remove_file, rename}; +use std::ffi::OsStr; +use tokio::fs::{remove_dir_all, remove_file}; use tracing::{debug, error}; -use walkdir::WalkDir; /// A filesystem error occured. const FS_ERROR: u32 = 0xFFF0_FFE0; @@ -53,47 +48,16 @@ pub async fn handle_removal( todo!("network shares not yet implemented!") }; - if fs_location.resolved_path().exists() { - if !state.disable_real_removal() { - if fs_location.resolved_path().is_file() { - if let Err(cause) = remove_file(fs_location.resolved_path()).await { - error!( - ?cause, - path = %fs_location.resolved_path().display(), - "Failed to remove file as requested by PCFS.", - ); - - return SataResponse::new( - state.pid(), - request_header, - SataResultCode::error(FS_ERROR), - ); - } - } else if fs_location.resolved_path().is_dir() { - if let Err(cause) = remove_dir_all(fs_location.resolved_path()).await { - error!( - ?cause, - path = %fs_location.resolved_path().display(), - "Failed to remove directory as requested by PCFS." - ); + if !fs_location.resolved_path().exists() { + return SataResponse::new( + state.pid(), + request_header, + SataResultCode::error(PATH_NOT_EXIST_ERROR), + ); + } - return SataResponse::new( - state.pid(), - request_header, - SataResultCode::error(FS_ERROR), - ); - } - } else { - return SataResponse::new( - state.pid(), - request_header, - SataResultCode::error(FS_ERROR), - ); - } - } else if fs_location.resolved_path().is_file() { - // This should always be fine to do as mount pounts are at most - // specific to a directory, so moving a file within the same directory - // doesn't violate the "can't move across mount points" on windows. + if state.disable_real_removal() { + let new_path = if fs_location.resolved_path().is_file() { let mut new_filename = fs_location .resolved_path() .file_name() @@ -103,112 +67,57 @@ pub async fn handle_removal( let mut new_path = fs_location.resolved_path().clone(); new_path.pop(); new_path.push(new_filename); + new_path + } else { + let mut new_path = fs_location.resolved_path().clone(); + let mut dir_name = new_path + .components() + .next_back() + .map(|c| c.as_os_str().to_os_string()) + .unwrap_or_default(); + dir_name.push(".rm"); + new_path.pop(); + new_path.push(dir_name); + new_path + }; - if let Err(cause) = rename(fs_location.resolved_path(), new_path).await { - error!( - ?cause, - path = %fs_location.resolved_path().display(), - "Failed to rename file (as opposed to remove) as requested by PCFS." - ); - - return SataResponse::new( - state.pid(), - request_header, - SataResultCode::error(FS_ERROR), - ); - } - } else if fs_location.resolved_path().is_dir() { - if let Err(cause) = rename_dir(fs_location.resolved_path()).await { - error!( - ?cause, - path = %fs_location.resolved_path().display(), - "Failed to rename folder (as opposed to remove) as requested by PCFS." - ); + if let Err(cause) = state + .host_filesystem() + .rename(fs_location.resolved_path(), &new_path) + { + error!( + ?cause, + path = %fs_location.resolved_path().display(), + "Failed to remove/rename directory as requested by PCFS." + ); - return SataResponse::new( - state.pid(), - request_header, - SataResultCode::error(FS_ERROR), - ); - } - } else { return SataResponse::new(state.pid(), request_header, SataResultCode::error(FS_ERROR)); } - } + } else if fs_location.resolved_path().is_file() { + if let Err(cause) = remove_file(fs_location.resolved_path()).await { + error!( + ?cause, + path = %fs_location.resolved_path().display(), + "Failed to remove file as requested by PCFS.", + ); - SataResponse::new(state.pid(), request_header, SataResultCode::success()) -} - -/// Rename an entire directory. -/// -/// We have to implement this ourselves, because [`tokio::fs::rename`], and -/// [`std::fs::rename`] don't support renaming a directory at all on windows, -/// which is one of the critical OS's that we need to support. -/// -/// This 'rename' works by actually creating a new directory with the ".rm" -/// added. Then moving all the files over with rename. This is slow, but -/// works. -async fn rename_dir(old_path: &PathBuf) -> Result<(), FSError> { - let mut new_filename = old_path.file_name().unwrap_or_default().to_owned(); - new_filename.push(OsStr::new(".rm")); - let mut new_path = old_path.clone(); - new_path.pop(); - new_path.push(new_filename); - let old_path_bytes = old_path.as_os_str().as_encoded_bytes(); - let new_path_as_str_bytes = new_path.as_os_str().as_encoded_bytes(); - - create_dir_all(&new_path).await?; - for result in WalkDir::new(old_path) - .follow_links(false) - .follow_root_links(false) - { - let rpb = result?.into_path(); - let os_str_for_entry = rpb.as_os_str().as_encoded_bytes(); - let mut new_bytes = Vec::with_capacity(os_str_for_entry.len() + 3); - new_bytes.extend_from_slice(new_path_as_str_bytes); - new_bytes.extend_from_slice(&os_str_for_entry[old_path_bytes.len()..]); - let as_new_path = - PathBuf::from(unsafe { OsString::from_encoded_bytes_unchecked(new_bytes) }); - - if rpb.is_symlink() { - let mut resolved_path = read_link(&rpb).await?; - { - // Rewrite paths within the directory we're removing. - let os_str_for_resolved = resolved_path.as_os_str().as_encoded_bytes(); - if os_str_for_resolved.starts_with(old_path_bytes) { - let mut new_bytes = Vec::with_capacity(os_str_for_resolved.len() + 3); - new_bytes.extend_from_slice(new_path_as_str_bytes); - new_bytes.extend_from_slice(&os_str_for_entry[old_path_bytes.len()..]); - resolved_path = - PathBuf::from(unsafe { OsString::from_encoded_bytes_unchecked(new_bytes) }); - } - } - - #[cfg(unix)] - { - use std::os::unix::fs::symlink; - symlink(resolved_path, &as_new_path)?; - } - - #[cfg(target_os = "windows")] - { - use std::os::windows::fs::{symlink_dir, symlink_file}; + return SataResponse::new(state.pid(), request_header, SataResultCode::error(FS_ERROR)); + } + } else if fs_location.resolved_path().is_dir() { + if let Err(cause) = remove_dir_all(fs_location.resolved_path()).await { + error!( + ?cause, + path = %fs_location.resolved_path().display(), + "Failed to remove directory as requested by PCFS." + ); - if resolved_path.is_dir() { - symlink_dir(resolved_path, &as_new_path)?; - } else { - symlink_file(resolved_path, &as_new_path)?; - } - } - } else if rpb.is_file() { - rename(&rpb, &as_new_path).await?; - } else if rpb.is_dir() { - create_dir_all(as_new_path).await?; + return SataResponse::new(state.pid(), request_header, SataResultCode::error(FS_ERROR)); } + } else { + return SataResponse::new(state.pid(), request_header, SataResultCode::error(FS_ERROR)); } - remove_dir_all(old_path).await?; - Ok(()) + SataResponse::new(state.pid(), request_header, SataResultCode::success()) } #[cfg(test)] @@ -261,34 +170,33 @@ mod unit_tests { pub async fn test_fake_removal() { let (tempdir, fs) = create_temporary_host_filesystem().await; - let base_dir = join_many(tempdir.path(), ["a", "b", "c"]); + // Create folders..... + let base_dir = join_many(tempdir.path(), ["directory-to-test-in"]); + // Created for us by temporary host filesystem... + let data_dir = join_many(tempdir.path(), ["data", "slc"]); + let symlink_folder_path = join_many(&base_dir, ["sub-directory-with-symlink"]); tokio::fs::create_dir_all(&base_dir) .await .expect("Failed to create temporary directory for test!"); + tokio::fs::create_dir_all(&symlink_folder_path) + .await + .expect("Failed to create temporary directory for test!"); + + // Place down files.... let file_path = join_many(&base_dir, ["file.txt"]); tokio::fs::write(&file_path, vec![0; 1307]) .await .expect("Failed to write test file!"); - let inner_path = join_many(tempdir.path(), ["a", "b", "c", "d", "e"]); - tokio::fs::create_dir_all(&inner_path) - .await - .expect("Failed to create temporary directory for test!"); - - let directory_to_symlink = join_many(tempdir.path(), ["data", "slc"]); - let dir_path_to_symlink = join_many(tempdir.path(), ["a", "b", "c", "d", "e", "f"]); - - let file_path_to_symlink = join_many( - tempdir.path(), - ["a", "b", "c", "d", "e", "symlinked-file.txt"], - ); + // Place down symlinks..... + let dir_path_to_symlink = join_many(&symlink_folder_path, ["symlinked-folder"]); + let file_path_to_symlink = join_many(&symlink_folder_path, ["symlinked-file.txt"]); #[cfg(unix)] { use std::os::unix::fs::symlink; - symlink(&directory_to_symlink, &dir_path_to_symlink) - .expect("Failed to symlink directory!"); + symlink(&data_dir, &dir_path_to_symlink).expect("Failed to symlink directory!"); symlink(&file_path, &file_path_to_symlink).expect("Failed to symlink file!"); } @@ -296,8 +204,7 @@ mod unit_tests { { use std::os::windows::fs::{symlink_dir, symlink_file}; - symlink_dir(&directory_to_symlink, &dir_path_to_symlink) - .expect("Failed to symlink directory!"); + symlink_dir(&data_dir, &dir_path_to_symlink).expect("Failed to symlink directory!"); symlink_file(&file_path, &file_path_to_symlink).expect("Failed to symlink file!"); } @@ -318,7 +225,7 @@ mod unit_tests { .await .try_into() .expect("Failed to serialize real removal response!"); - let renamed_dir = join_many(tempdir.path(), ["a", "b", "c.rm"]); + let renamed_dir = join_many(tempdir.path(), ["directory-to-test-in.rm"]); assert!( !base_dir.exists(), diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/rename.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/rename.rs new file mode 100644 index 0000000..e653335 --- /dev/null +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/rename.rs @@ -0,0 +1,188 @@ +//! Handle rename packets which can renamefiles or folders. + +use crate::{ + fsemul::{ + host_filesystem::ResolvedLocation, + pcfs::sata::{ + proto::{SataRenamePacketBody, SataRequest, SataResponse, SataResultCode}, + server::PCFSServerState, + }, + }, + net::server::requestable::{Body, State}, +}; +use tracing::{debug, error}; + +/// A filesystem error occured. +const FS_ERROR: u32 = 0xFFF0_FFE0; +/// An error code to send when a path does not exist. +/// +/// This is also used in some places that are a bit of a stretch like for +/// network shares on disk space. The path doesn't exist on a disk, so this +/// error code is used, even if it's not quite exact. +const PATH_NOT_EXIST_ERROR: u32 = 0xFFF0_FFE9; + +/// Handle renaming a file, or directory upon request. +pub async fn handle_rename( + State(state): State, + Body(request): Body>, +) -> SataResponse { + let request_header = request.header().clone(); + let packet = request.body(); + + let Ok(final_source_location) = state.host_filesystem().resolve_path(packet.source_path()) + else { + debug!( + packet.path = packet.source_path(), + packet.typ = "PCFSSrvRename", + "Failed to resolve path!", + ); + + return SataResponse::new( + state.pid(), + request_header, + SataResultCode::error(PATH_NOT_EXIST_ERROR), + ); + }; + let ResolvedLocation::Filesystem(fs_source_location) = final_source_location else { + todo!("network shares not yet implemented!") + }; + + if !fs_source_location.resolved_path().exists() { + return SataResponse::new( + state.pid(), + request_header, + SataResultCode::error(PATH_NOT_EXIST_ERROR), + ); + } + + let Ok(final_dest_location) = state.host_filesystem().resolve_path(packet.dest_path()) else { + debug!( + packet.path = packet.dest_path(), + packet.typ = "PCFSSrvRename", + "Failed to resolve path!", + ); + + return SataResponse::new( + state.pid(), + request_header, + SataResultCode::error(PATH_NOT_EXIST_ERROR), + ); + }; + let ResolvedLocation::Filesystem(fs_dest_location) = final_dest_location else { + todo!("network shares not yet implemented!") + }; + + let result = if request.command_info().user() == (0x1000_00F5, 0x1000_00FF) { + state.host_filesystem.copy( + fs_source_location.resolved_path(), + fs_dest_location.resolved_path(), + ) + } else { + state.host_filesystem.rename( + fs_source_location.resolved_path(), + fs_dest_location.resolved_path(), + ) + }; + + if let Err(cause) = result { + error!( + ?cause, + packet.source_path = packet.source_path(), + packet.dest_path = packet.dest_path(), + packet.typ = "PCFSSrvRename", + "Failed to rename file or folder!", + ); + + SataResponse::new(state.pid(), request_header, SataResultCode::error(FS_ERROR)) + } else { + SataResponse::new( + state.pid(), + request_header, + SataResultCode::error(0xFFF0_FFEA), + ) + } +} + +#[cfg(test)] +mod unit_tests { + use super::*; + use crate::fsemul::{ + host_filesystem::test_helpers::{create_temporary_host_filesystem, join_many}, + pcfs::sata::proto::{SataCommandInfo, SataPacketHeader}, + }; + use bytes::Bytes; + + #[tokio::test] + pub async fn test_rename() { + let (tempdir, fs) = create_temporary_host_filesystem().await; + + let base_dir = join_many(tempdir.path(), ["a", "b", "c"]); + tokio::fs::create_dir_all(&base_dir) + .await + .expect("Failed to create temporary directory for test!"); + let file_path = join_many(&base_dir, ["file.txt"]); + tokio::fs::write(&file_path, vec![0; 1307]) + .await + .expect("Failed to write test file!"); + + let inner_path = join_many(tempdir.path(), ["a", "b", "c", "d", "e"]); + tokio::fs::create_dir_all(&inner_path) + .await + .expect("Failed to create temporary directory for test!"); + + let directory_to_symlink = join_many(tempdir.path(), ["data", "slc"]); + let dir_path_to_symlink = join_many(tempdir.path(), ["a", "b", "c", "d", "e", "f"]); + + let file_path_to_symlink = join_many( + tempdir.path(), + ["a", "b", "c", "d", "e", "symlinked-file.txt"], + ); + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + + symlink(&directory_to_symlink, &dir_path_to_symlink) + .expect("Failed to symlink directory!"); + symlink(&file_path, &file_path_to_symlink).expect("Failed to symlink file!"); + } + + #[cfg(target_os = "windows")] + { + use std::os::windows::fs::{symlink_dir, symlink_file}; + + symlink_dir(&directory_to_symlink, &dir_path_to_symlink) + .expect("Failed to symlink directory!"); + symlink_file(&file_path, &file_path_to_symlink).expect("Failed to symlink file!"); + } + + let renamed_dir = join_many(tempdir.path(), ["a", "b", "c.rm"]); + let request = SataRenamePacketBody::new( + base_dir + .to_str() + .expect("Test paths must be UTF-8") + .to_owned(), + renamed_dir + .to_str() + .expect("Test paths must be utf-8") + .to_owned(), + ) + .expect("Failed to create sata rename packet body!"); + let mocked_header = SataPacketHeader::new(0); + let mocked_ci = SataCommandInfo::new((0, 0), (0, 0), 0); + + let _bytes: Bytes = handle_rename( + State(PCFSServerState::new(true, fs, 0)), + Body(SataRequest::new(mocked_header, mocked_ci, request)), + ) + .await + .try_into() + .expect("Failed to serialize real removal response!"); + + assert!( + !base_dir.exists(), + "Base directory still exists post 'removal'", + ); + assert!(renamed_dir.exists(), "Renamed directory doesn't exist?"); + } +} diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/rewind_folder.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/rewind_folder.rs index 9e2b7a2..6d1f0a4 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/rewind_folder.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/rewind_folder.rs @@ -69,7 +69,6 @@ mod unit_tests { let dfd = fs .open_folder(&base_dir, Some(1)) - .await .expect("Failed to open existing directory!"); let request = SataRewindFolderPacketBody::new(dfd); diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/set_file_position.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/set_file_position.rs new file mode 100644 index 0000000..1e6e64d --- /dev/null +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/set_file_position.rs @@ -0,0 +1,55 @@ +//! Handle a client requesting us to move the location of an open file. + +use crate::{ + fsemul::pcfs::sata::{ + proto::{SataRequest, SataResponse, SataResultCode, SataSetFilePositionPacketBody}, + server::PCFSServerState, + }, + net::{ + additions::StreamID, + server::requestable::{Body, State}, + }, +}; +use tracing::debug; + +/// A filesystem error occured. +const FS_ERROR: u32 = 0xFFF0_FFE0; + +/// Handle setting the position within an already open file. +pub async fn handle_set_file_position( + stream: StreamID, + State(state): State, + // Validate that the body is actually a change owner packet. + Body(request): Body>, +) -> SataResponse { + let packet = request.body(); + + if let Err(cause) = packet + .move_to_pointer() + .do_move( + state.host_filesystem(), + packet.file_descriptor(), + Some(stream.to_raw()), + ) + .await + { + debug!( + ?cause, + packet.fd = packet.file_descriptor(), + packet.typ = "PCFSSrvSetFilePosition", + "Failed to move file to a specific pointer!", + ); + + SataResponse::new( + state.pid(), + request.header().clone(), + SataResultCode::error(FS_ERROR), + ) + } else { + SataResponse::new( + state.pid(), + request.header().clone(), + SataResultCode::success(), + ) + } +} diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/layer.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/layer.rs new file mode 100644 index 0000000..e81910e --- /dev/null +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/layer.rs @@ -0,0 +1,186 @@ +//! Allow attaching the Write-Ahead log as an arbitrary layer to any server. + +use crate::{ + errors::CatBridgeError, + fsemul::pcfs::sata::server::wal::WriteAheadLog, + net::{ + models::{Request, Response}, + server::models::ResponseStreamEvent, + }, +}; +use std::{ + convert::Infallible, + pin::Pin, + task::{Context, Poll}, +}; +use tower::{Layer, Service}; + +/// A layer that will automatically record begin streams from a +/// TCP Server. +#[derive(Clone, Debug)] +pub struct WALBeginStreamLayer(pub WriteAheadLog); + +impl Layer for WALBeginStreamLayer +where + Layered: Clone, +{ + type Service = LayeredBeginWALStream; + + fn layer(&self, inner: Layered) -> Self::Service { + LayeredBeginWALStream { + inner, + log: self.0.clone(), + } + } +} + +#[derive(Clone)] +pub struct LayeredBeginWALStream { + inner: Layered, + log: WriteAheadLog, +} + +impl Service> + for LayeredBeginWALStream +where + Layered: Service, Response = bool, Error = CatBridgeError> + + Clone + + Send + + 'static, + Layered::Future: Send + 'static, +{ + type Response = Layered::Response; + type Error = Layered::Error; + type Future = Pin> + Send>>; + + #[inline] + fn poll_ready(&mut self, ctx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(ctx) + } + + fn call(&mut self, evt: ResponseStreamEvent) -> Self::Future { + let log = self.log.clone(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + log.record_open_stream(evt.stream_id()).await; + inner.call(evt).await + }) + } +} + +/// A layer that will automatically record end streams from a +/// TCP Server. +#[derive(Clone, Debug)] +pub struct WALEndStreamLayer(pub WriteAheadLog); + +impl Layer for WALEndStreamLayer +where + Layered: Clone, +{ + type Service = LayeredEndWALStream; + + fn layer(&self, inner: Layered) -> Self::Service { + LayeredEndWALStream { + inner, + log: self.0.clone(), + } + } +} + +#[derive(Clone)] +pub struct LayeredEndWALStream { + inner: Layered, + log: WriteAheadLog, +} + +impl Service> + for LayeredEndWALStream +where + Layered: Service, Response = (), Error = CatBridgeError> + + Clone + + Send + + 'static, + Layered::Future: Send + 'static, +{ + type Response = Layered::Response; + type Error = Layered::Error; + type Future = Pin> + Send>>; + + #[inline] + fn poll_ready(&mut self, ctx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(ctx) + } + + fn call(&mut self, evt: ResponseStreamEvent) -> Self::Future { + let log = self.log.clone(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + log.record_close_stream(evt.stream_id()).await; + inner.call(evt).await + }) + } +} + +/// A layer that will automatically record requests/responses, and attach a WAL +/// as an extension. +#[derive(Clone, Debug)] +pub struct WALMessageLayer(pub WriteAheadLog); + +impl Layer for WALMessageLayer +where + Layered: Clone, +{ + type Service = LayeredWALMessage; + + fn layer(&self, inner: Layered) -> Self::Service { + LayeredWALMessage { + inner, + log: self.0.clone(), + } + } +} + +#[derive(Clone)] +pub struct LayeredWALMessage { + inner: Layered, + log: WriteAheadLog, +} + +impl Service> + for LayeredWALMessage +where + Layered: + Service, Response = Response, Error = Infallible> + Clone + Send + 'static, + Layered::Future: Send + 'static, +{ + type Response = Layered::Response; + type Error = Layered::Error; + type Future = Pin> + Send>>; + + #[inline] + fn poll_ready(&mut self, ctx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(ctx) + } + + fn call(&mut self, mut req: Request) -> Self::Future { + let log = self.log.clone(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + let sid = req.stream_id(); + log.record_request(sid, req.body().clone()).await; + req.extensions_mut().insert(log.clone()); + match inner.call(req).await { + Ok(resp) => { + if let Some(bod) = resp.body() { + log.record_response(sid, bod.clone()).await; + } + Ok::(resp) + } + Err(cause) => Err::(cause), + } + }) + } +} diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/mapper.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/mapper.rs new file mode 100644 index 0000000..238cf28 --- /dev/null +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/mapper.rs @@ -0,0 +1,706 @@ +//! The WAL Mapper is responsible for turning a series of bytes into a +//! well-known packet type. + +use crate::{ + errors::NetworkParseError, + fsemul::pcfs::sata::{ + proto::{ + DirectoryItemResponse, MoveToFileLocation, SataCapabilitiesFlags, + SataChangeModePacketBody, SataChangeOwnerPacketBody, SataCloseFilePacketBody, + SataCloseFolderPacketBody, SataCreateFolderPacketBody, SataFDInfo, + SataFileDescriptorResult, SataGetInfoByQueryPacketBody, SataOpenFilePacketBody, + SataOpenFolderPacketBody, SataPingPacketBody, SataPongBody, SataQueryResponse, + SataQueryType, SataReadFilePacketBody, SataReadFolderPacketBody, SataRemovePacketBody, + SataRenamePacketBody, SataRequest, SataResponse, SataResultCode, + SataRewindFolderPacketBody, SataSetFilePositionPacketBody, SataStatFilePacketBody, + SataWriteFilePacketBody, + }, + server::connection_flags::SataConnectionFlags, + }, +}; +use bytes::Bytes; +use fnv::{FnvHashMap, FnvHasher}; +use std::{ + fmt::{Display, Formatter, Result as FmtResult}, + hash::Hasher, +}; + +/// A request that is waiting for a response. +/// +/// Used to keep track of which packets we're waiting on a response from incase +/// multiple queue up at once. This also holds the info a repsonse may need in +/// order to provide future logging. +/// +/// E.g. in an open file request we send the path, but the response just sends +/// back the fd. In order to create a map of fd -> path (so we can log what's +/// happening to specific paths), we need info from the request when we process +/// the repsonse. So we can create that ideal mapping. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum WaitingRequest { + /// A create folder request (Path, Will Set Write Mode). + CreateFolder(String, bool), + /// An open folder request (Path). + OpenFolder(String), + /// A read folder request (Path). + ReadFolder(String), + /// Rewind a folder iterator (Path). + RewindFolder(String), + /// Close a folder (Path). + CloseFolder(i32, String), + /// A request to open a file (Path, Mode) + OpenFile(String, String), + /// A request to read a file (Path, Move to file location before reading, Allow truncation). + ReadFile(String, Option, u32), + /// A request to write to a file (Path, Move to file location before writing). + WriteFile(String, Option), + /// A request to move the location of a particular file. + SetFilePosition(String, MoveToFileLocation), + /// Get the information about an already open file descriptor (Path). + StatFile(String), + /// Close an already open file (Path). + CloseFile(i32, String), + /// Remove a file or folder from the host filesystem (Path). + Remove(String), + /// Rename a file or folder from the host filesystem (Path, Path). + Rename(String, String), + /// Query the information about a particular path on the host fs (Path). + QueryPath(SataQueryType, String), + /// Change ownership of a particular path on the host filesystem (Path, uid, gid). + ChangeOwner(String, u32, u32), + /// Change the mode of a particular path on the host filesystem (Path, unix mode string). + ChangeMode(String, String), + /// A ping request (FFIO enabled, CSR enabled, first read size, first write size). + Ping(bool, bool, u32, u32), + /// If the packet is just some unknown bytes... + Unknown(Bytes), +} + +impl WaitingRequest { + /// Parse out a Sata Request that is going to be waiting for a response. + #[allow( + // Overall each branch is relatively small, we just have a lot of packet types. + clippy::too_many_lines, + )] + #[must_use] + pub fn parse( + fd_map: &FnvHashMap, + folder_map: &FnvHashMap, + data: Bytes, + ) -> Self { + let Ok(req) = SataRequest::::parse_opaque(data.clone()) else { + return Self::Unknown(data); + }; + let (header, ci, body) = req.into_parts(); + + match ci.command() { + 0x00 => { + if let Ok(body) = SataCreateFolderPacketBody::try_from(body) { + Self::CreateFolder(body.path().to_owned(), body.will_set_write_mode()) + } else { + Self::Unknown(data) + } + } + 0x01 => { + if let Ok(body) = SataOpenFolderPacketBody::try_from(body) { + Self::OpenFolder(body.path().to_owned()) + } else { + Self::Unknown(data) + } + } + 0x02 => { + if let Ok(read_folder) = SataReadFolderPacketBody::try_from(body) { + if let Some(path) = folder_map.get(&read_folder.file_descriptor()) { + Self::ReadFolder(path.clone()) + } else { + Self::ReadFolder(format!("", read_folder.file_descriptor())) + } + } else { + Self::Unknown(data) + } + } + 0x03 => { + if let Ok(rewind_folder) = SataRewindFolderPacketBody::try_from(body) { + if let Some(path) = folder_map.get(&rewind_folder.file_descriptor()) { + Self::RewindFolder(path.clone()) + } else { + Self::RewindFolder(format!( + "", + rewind_folder.file_descriptor() + )) + } + } else { + Self::Unknown(data) + } + } + 0x04 => { + if let Ok(close_folder) = SataCloseFolderPacketBody::try_from(body) { + if let Some(path) = folder_map.get(&close_folder.file_descriptor()) { + Self::CloseFolder(close_folder.file_descriptor(), path.clone()) + } else { + Self::CloseFolder( + close_folder.file_descriptor(), + format!("", close_folder.file_descriptor()), + ) + } + } else { + Self::Unknown(data) + } + } + 0x05 => { + if let Ok(open_file) = SataOpenFilePacketBody::try_from(body) { + Self::OpenFile(open_file.path().to_owned(), open_file.mode().to_owned()) + } else { + Self::Unknown(data) + } + } + 0x06 => { + if let Ok(read_file) = SataReadFilePacketBody::try_from(body) { + if let Some(path) = fd_map.get(&read_file.file_descriptor()) { + Self::ReadFile( + path.clone(), + if read_file.should_move() { + Some(read_file.move_to_pointer()) + } else { + None + }, + read_file.block_count() * read_file.block_size(), + ) + } else { + Self::ReadFile( + format!("", read_file.file_descriptor()), + if read_file.should_move() { + Some(read_file.move_to_pointer()) + } else { + None + }, + read_file.block_count() * read_file.block_size(), + ) + } + } else { + Self::Unknown(data) + } + } + 0x07 => { + if let Ok(write_file) = SataWriteFilePacketBody::try_from(body) { + if let Some(path) = fd_map.get(&write_file.file_descriptor()) { + Self::WriteFile( + path.clone(), + if write_file.should_move() { + Some(write_file.move_to_pointer()) + } else { + None + }, + ) + } else { + Self::WriteFile( + format!("", write_file.file_descriptor()), + if write_file.should_move() { + Some(write_file.move_to_pointer()) + } else { + None + }, + ) + } + } else { + Self::Unknown(data) + } + } + 0x09 => { + if let Ok(set_file_pos) = SataSetFilePositionPacketBody::try_from(body) { + if let Some(path) = fd_map.get(&set_file_pos.file_descriptor()) { + Self::SetFilePosition(path.clone(), set_file_pos.move_to_pointer()) + } else { + Self::SetFilePosition( + format!("", set_file_pos.file_descriptor()), + set_file_pos.move_to_pointer(), + ) + } + } else { + Self::Unknown(data) + } + } + 0x0B => { + if let Ok(stat_file) = SataStatFilePacketBody::try_from(body) { + if let Some(path) = fd_map.get(&stat_file.file_descriptor()) { + Self::StatFile(path.clone()) + } else { + Self::StatFile(format!("", stat_file.file_descriptor())) + } + } else { + Self::Unknown(data) + } + } + 0x0D => { + if let Ok(close_file) = SataCloseFilePacketBody::try_from(body) { + if let Some(path) = fd_map.get(&close_file.file_descriptor()) { + Self::CloseFile(close_file.file_descriptor(), path.clone()) + } else { + Self::CloseFile( + close_file.file_descriptor(), + format!("", close_file.file_descriptor()), + ) + } + } else { + Self::Unknown(data) + } + } + 0x0E => { + if let Ok(remove) = SataRemovePacketBody::try_from(body) { + Self::Remove(remove.path().to_owned()) + } else { + Self::Unknown(data) + } + } + 0x0F => { + if let Ok(ren) = SataRenamePacketBody::try_from(body) { + Self::Rename(ren.source_path().to_owned(), ren.dest_path().to_owned()) + } else { + Self::Unknown(data) + } + } + 0x10 => { + if let Ok(query) = SataGetInfoByQueryPacketBody::try_from(body) { + Self::QueryPath(query.query_type(), query.path().to_owned()) + } else { + Self::Unknown(data) + } + } + 0x12 => { + if let Ok(change_owner) = SataChangeOwnerPacketBody::try_from(body) { + Self::ChangeOwner( + change_owner.path().to_owned(), + change_owner.uid(), + change_owner.gid(), + ) + } else { + Self::Unknown(data) + } + } + 0x13 => { + if let Ok(chmod) = SataChangeModePacketBody::try_from(body) { + Self::ChangeMode( + chmod.path().to_owned(), + if chmod.will_set_write_mode() { + "0666".to_owned() + } else { + "0444".to_owned() + }, + ) + } else { + Self::Unknown(data) + } + } + 0x14 => { + if SataPingPacketBody::try_from(body).is_ok() { + let flags = SataCapabilitiesFlags(header.flags()); + Self::Ping( + flags.intersects(SataCapabilitiesFlags::FAST_FILE_IO_SUPPORTED), + flags.intersects(SataCapabilitiesFlags::COMBINED_SEND_RECV_SUPPORTED), + ci.user().0, + ci.user().1, + ) + } else { + Self::Unknown(data) + } + } + _ => Self::Unknown(data), + } + } +} + +impl Display for WaitingRequest { + #[allow( + // Overall each branch is relatively small, we just have a lot of packet types. + clippy::too_many_lines, + )] + fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult { + match self { + Self::CreateFolder(path, will_set_write_mode) => { + write!( + fmt, + "CreateFolder {{path={path},mode={}}}", + if *will_set_write_mode { "0666" } else { "0444" }, + ) + } + Self::OpenFolder(path) => { + write!(fmt, "OpenFolder {{path={path}}}") + } + Self::ReadFolder(path) => { + write!(fmt, "ReadFolder {{path={path}}}") + } + Self::RewindFolder(path) => { + write!(fmt, "RewindFolder {{path={path}}}") + } + Self::CloseFolder(_, path) => { + write!(fmt, "CloseFolder {{path={path}}}") + } + Self::OpenFile(path, mode) => { + write!(fmt, "OpenFile {{path={path},mode={mode}}}") + } + Self::ReadFile(path, move_to, read_size) => { + write!( + fmt, + "ReadFile {{path={path},move_to={},total_size={read_size}}}", + if let Some(to) = move_to { + format!("{to}") + } else { + "Nowhere".to_owned() + } + ) + } + Self::WriteFile(path, move_to) => { + write!( + fmt, + "WriteFile {{path={path},move_to={}}}", + if let Some(to) = move_to { + format!("{to}") + } else { + "Nowhere".to_owned() + } + ) + } + Self::SetFilePosition(path, move_to) => { + write!(fmt, "SetFilePos {{path={path},move_to={move_to}}}") + } + Self::StatFile(path) => { + write!(fmt, "StatFile {{path={path}}}") + } + Self::CloseFile(_, path) => { + write!(fmt, "CloseFile {{path={path}}}") + } + Self::Remove(path) => { + write!(fmt, "Remove {{path={path}}}") + } + Self::Rename(source, dest) => { + write!(fmt, "Rename {{source={source},dest={dest}}}") + } + Self::QueryPath(query_type, path) => { + write!( + fmt, + "QueryPath {{path={path},query_type={}}}", + match query_type { + SataQueryType::FileCount => "Files in Folder", + SataQueryType::FileDetails => "Path Details", + SataQueryType::FreeDiskSpace => "Free Space left on Disk", + SataQueryType::SizeOfFolder => "Size of Folder", + } + ) + } + Self::ChangeOwner(path, uid, gid) => { + write!(fmt, "ChangeOwner {{path={path},uid={uid},gid={gid}}}") + } + Self::ChangeMode(path, mode) => { + write!(fmt, "ChangeMode {{path={path},mode={mode}}}") + } + Self::Ping(ffio, csr, read_size, write_size) => { + write!( + fmt, + "Ping {{supports_ffio={ffio},supports_csr={csr},first_read_size={read_size},first_write_size={write_size}}}", + ) + } + Self::Unknown(data) => { + write!(fmt, "Unknown {{data={data:02x?}}}") + } + } + } +} + +#[derive(Debug, Clone)] +pub enum WaitingResponse { + /// Create a new folder (rc). + CreateFolder(SataResultCode), + /// Open a folder (fd or rc). + OpenFolder(SataFileDescriptorResult), + /// Read the next file in a folder (RC, ). + ReadFolder(SataResultCode, Option), + /// Rewind the iterator for a folder (rc). + RewindFolder(SataResultCode), + /// Close an open folder (rc). + CloseFolder(SataResultCode), + /// Open an existing file (fd or rc). + OpenFile(SataFileDescriptorResult), + /// We read an open file <(file size as u32, bytes read, checksum), error code>. + ReadFile(Result<(u32, usize, Bytes, u64), SataResultCode>), + /// We wrote to a file (rc). + WriteFile(SataResultCode), + /// We moved the position in the file! + SetFilePosition(SataResultCode), + /// Queried info about an existing file (fd info). + StatFile(Result), + /// If we were able to close a file (rc). + CloseFile(SataResultCode), + /// If we were able to remove a file (rc). + Remove(SataResultCode), + /// If we were able to rename a file (rc). + Rename(SataResultCode), + /// The query results for a query (query response). + QueryPath(Result), + /// If we were able to change ownership of a file (rc). + ChangeOwner(SataResultCode), + /// If we were able to change the mode of a file (rc). + ChangeMode(SataResultCode), + /// Response to a ping (ffio supported, csr supported). + Pong(bool, bool), + /// ??? -- Data. + Unknown(Bytes), +} + +impl WaitingResponse { + /// Parse out the response into a printable thing. + #[allow( + // Overall each branch is relatively small, we just have a lot of packet types. + clippy::too_many_lines, + )] + #[must_use] + pub fn parse(flags: &SataConnectionFlags, request: &WaitingRequest, response: Bytes) -> Self { + match request { + WaitingRequest::CreateFolder(_, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::CreateFolder(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::OpenFolder(_) => { + if let Ok(resp) = + SataResponse::::try_from(response.clone()) + { + Self::OpenFolder(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::ReadFolder(_) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) + { + Self::ReadFolder( + SataResultCode(resp.body().return_code()), + resp.body().clone().take_file_info().map(|item| item.1), + ) + } else { + Self::Unknown(response) + } + } + WaitingRequest::RewindFolder(_) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::RewindFolder(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::CloseFolder(_, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::CloseFolder(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::OpenFile(_, _) => { + if let Ok(resp) = + SataResponse::::try_from(response.clone()) + { + Self::OpenFile(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::ReadFile(_, _, _) => { + if flags.ffio_enabled() { + if response.len() < 0x24 { + return Self::Unknown(response); + } + let file_len = u32::from_be_bytes([ + response[0x20], + response[0x21], + response[0x22], + response[0x23], + ]); + + let mut chk = FnvHasher::with_key(69420); + chk.write(&response); + let header = response.slice(..0x20); + Self::ReadFile(Ok((file_len, response.len(), header, chk.finish()))) + } else { + todo!("Non-FFIO support") + } + } + WaitingRequest::WriteFile(_, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::WriteFile(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::SetFilePosition(_, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::SetFilePosition(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::StatFile(_) => { + if let Ok(resp) = SataResponse::::parse_opaque(response.clone()) { + match SataQueryResponse::try_from_fd_info(resp.body().clone()) { + Ok(qresp) => match qresp { + SataQueryResponse::FDInfo(info) => Self::StatFile(Ok(info.clone())), + SataQueryResponse::ErrorCode(ec) => { + Self::StatFile(Err(SataResultCode(ec))) + } + _ => Self::Unknown(response), + }, + Err(cause) => match cause { + NetworkParseError::ErrorCode(rc) => { + Self::StatFile(Err(SataResultCode(rc))) + } + _ => Self::Unknown(response), + }, + } + } else { + Self::Unknown(response) + } + } + WaitingRequest::CloseFile(_, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::CloseFile(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::Remove(_) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::Remove(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::Rename(_, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::Rename(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::QueryPath(query_type, _) => { + if let Ok(resp) = SataResponse::::parse_opaque(response.clone()) { + let inner_res = match query_type { + SataQueryType::FileCount => { + SataQueryResponse::try_from_small(resp.body().clone()) + } + SataQueryType::FileDetails => { + SataQueryResponse::try_from_fd_info(resp.body().clone()) + } + SataQueryType::FreeDiskSpace | SataQueryType::SizeOfFolder => { + SataQueryResponse::try_from_large(resp.body().clone()) + } + }; + + match inner_res { + Ok(qi) => Self::QueryPath(Ok(qi)), + Err(cause) => match cause { + NetworkParseError::ErrorCode(ec) => { + Self::QueryPath(Err(SataResultCode(ec))) + } + _ => Self::Unknown(response), + }, + } + } else { + Self::Unknown(response) + } + } + WaitingRequest::ChangeOwner(_, _, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::ChangeOwner(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::ChangeMode(_, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::ChangeMode(*resp.body()) + } else { + Self::Unknown(response) + } + } + WaitingRequest::Ping(_, _, _, _) => { + if let Ok(resp) = SataResponse::::try_from(response.clone()) { + Self::Pong( + resp.body().ffio_enabled(), + resp.body().combined_send_recv_enabled(), + ) + } else { + Self::Unknown(response) + } + } + WaitingRequest::Unknown(_) => Self::Unknown(response), + } + } +} + +impl Display for WaitingResponse { + fn fmt(&self, fmt: &mut Formatter<'_>) -> FmtResult { + match self { + Self::CreateFolder(rc) + | Self::ChangeOwner(rc) + | Self::ChangeMode(rc) + | Self::RewindFolder(rc) + | Self::CloseFolder(rc) + | Self::WriteFile(rc) + | Self::CloseFile(rc) + | Self::Remove(rc) + | Self::Rename(rc) + | Self::SetFilePosition(rc) => write!(fmt, " RC {{return_code={rc}}}"), + Self::OpenFolder(fdres) | Self::OpenFile(fdres) => { + write!(fmt, " FDRES {{response={fdres}}}") + } + Self::ReadFolder(rc, path) => write!( + fmt, + " Folder Item {{rc={rc},next_item={}}}", + if let Some(item) = path { + item.clone() + } else { + "".to_owned() + } + ), + Self::ReadFile(result) => match result { + Ok((file_size, read, header, chk)) => write!( + fmt, + " READ {{chk={chk:02x},file_size={file_size},packet_len={read},header={header:02x?}}}", + ), + Err(rc) => write!(fmt, " RC {{return_code={rc}}}"), + }, + Self::StatFile(result) => match result { + Ok(fdi) => write!( + fmt, + " STAT {{file_or_folder_flags={:02x},perms={:02x},file_length={}}}", + fdi.flags(), + fdi.permissions(), + fdi.file_size().unwrap_or(0), + ), + Err(rc) => write!(fmt, " RC {{return_code={rc}}}"), + }, + Self::QueryPath(result) => match result { + Ok(qr) => write!( + fmt, + " QUERY {{response={}}}", + match qr { + SataQueryResponse::ErrorCode(ec) => format!("ERROR:{ec}"), + SataQueryResponse::FDInfo(info) => format!( + "INFO:flags={:02x}:perms={:02x}:file_length={}", + info.flags(), + info.permissions(), + info.file_size().unwrap_or(0), + ), + SataQueryResponse::LargeSize(ls) => format!("LG:{ls}"), + SataQueryResponse::SmallSize(ss) => format!("SM:{ss}"), + } + ), + Err(rc) => write!(fmt, " RC {{return_code={rc}}}"), + }, + Self::Pong(ffio, csr) => write!( + fmt, + " PONG {{ffio_enabled={ffio},combined_send_recv_enabled={csr}}}", + ), + Self::Unknown(data) => write!(fmt, " Unknown {{data={data:02x?}}}"), + } + } +} diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/mod.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/mod.rs new file mode 100644 index 0000000..d312a84 --- /dev/null +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/wal/mod.rs @@ -0,0 +1,373 @@ +//! A "WAL" style log for all sata requests, where before processing them we +//! will write the operation that occured, along with the return code. +//! +//! The goal of this is to make it easy to get a full log of what the PCFS Sata +//! server is doing, and where it might differ, without having to fully break +//! into a scientists like API where we're doing full diffing between +//! everything that's happening. +//! +//! THIS DOES HAVE A PERFORMANCE OVERHEAD, specifically we will be parsing +//! every sata request twice (or attempting too). So if you're not in a place +//! where you can spare that much CPU be aware. + +pub mod layer; +mod mapper; + +use crate::{ + errors::CatBridgeError, + fsemul::pcfs::sata::server::connection_flags::SataConnectionFlags, + net::models::{FromRequest, FromRequestParts, Request}, +}; +use bytes::Bytes; +use fnv::FnvHashMap; +use std::{ + collections::VecDeque, + path::PathBuf, + time::{Duration, Instant}, +}; +use tokio::{ + fs::File, + io::{AsyncWriteExt, BufWriter}, + sync::mpsc::{ + Receiver as BoundedReceiver, Sender as BoundedSender, channel as bounded_channel, + }, + task::Builder as TaskBuilder, + time::sleep, +}; +use tracing::error; + +/// A reference to a single unique 'write-ahead log' for SATA. +/// +/// This keeps track of requests, and responses coming in and out of a sata +/// server, to determine what is actually happening. +/// +/// This WAL is expected to be from the 'servers' point of view, NOT the clients. +#[derive(Clone, Debug)] +pub struct WriteAheadLog { + logger: BoundedSender, +} + +impl WriteAheadLog { + /// Create a new WAL for sata requests/responses. + /// + /// This is the 'entrypoint', and as a result will spawn the task + /// to start receiving messages. + /// + /// ## Errors + /// + /// If we cannot spawn the background task. + pub fn new(wal: PathBuf) -> Result { + let (sender, receiver) = bounded_channel(8192); + + TaskBuilder::new() + .name("cat_dev::fsemul::pcfs::sata::server::wal::write_to_log") + .spawn(async move { + process_wal(receiver, wal).await; + error!("WAL SHUTTING DOWN..."); + }) + .map_err(CatBridgeError::SpawnFailure)?; + + Ok(Self { logger: sender }) + } + + /// Communicate that a stream has opened up. + pub async fn record_open_stream(&self, stream_id: u64) { + _ = self + .logger + .send(WriteAheadLogMessage::OpenStream(stream_id)) + .await; + } + + /// Communicate that a stream has closed. + pub async fn record_close_stream(&self, stream_id: u64) { + _ = self + .logger + .send(WriteAheadLogMessage::CloseStream(stream_id)) + .await; + } + + /// Communicate that an out of band (e.g. nor equest/response flow) read for write file. + pub async fn record_oob_file_write_read(&self, stream_id: u64, fd: i32, length: usize) { + _ = self + .logger + .send(WriteAheadLogMessage::WriteFileRead(stream_id, fd, length)) + .await; + } + + /// Communicate that a request has come into the server. + pub async fn record_request(&self, stream_id: u64, request: Bytes) { + _ = self + .logger + .send(WriteAheadLogMessage::StreamEvent(stream_id, false, request)) + .await; + } + + /// Communicate that a response has come into the server. + pub async fn record_response(&self, stream_id: u64, request: Bytes) { + _ = self + .logger + .send(WriteAheadLogMessage::StreamEvent(stream_id, true, request)) + .await; + } +} + +impl FromRequestParts for Option { + async fn from_request_parts(parts: &mut Request) -> Result { + Ok(parts.extensions().get::().cloned()) + } +} + +impl FromRequest for Option { + async fn from_request(req: Request) -> Result { + Ok(req.extensions_owned().remove::()) + } +} + +/// A message sent over the logging channel to be written to the WAL. +#[derive(Clone, Debug)] +enum WriteAheadLogMessage { + /// A new TCP stream has been opened (stream id). + OpenStream(u64), + /// A TCP stream has closed (stream id). + CloseStream(u64), + /// An out of band Write File Response has been read in (stream id, fd, length read). + WriteFileRead(u64, i32, usize), + /// A regular stream request/response has happened (stream id, is response, data). + StreamEvent(u64, bool, Bytes), +} + +#[allow( + // TODO(mythra): clean this up. + clippy::too_many_lines, +)] +async fn process_wal(mut stream: BoundedReceiver, path: PathBuf) { + let mut fd = BufWriter::new(match File::create_new(path).await { + Ok(fd) => fd, + Err(cause) => { + error!(?cause, "Failed to open WAL file, will not generate WAL!"); + return; + } + }); + // A list of open fd's so we just write operations on which files they're happening on + // rather than an explicit fd. + let mut fd_map: FnvHashMap> = FnvHashMap::default(); + let mut folder_map: FnvHashMap> = FnvHashMap::default(); + let mut connection_flags: FnvHashMap = FnvHashMap::default(); + let mut waiting_requests: FnvHashMap> = + FnvHashMap::default(); + + // When the last 'WriteAheadLog' is dropped, and thus the producer is closed, + // we will automatically save. This keeps us from running forever. + let mut last_flush = Instant::now(); + let mut needs_flush = false; + + loop { + let msg_opt; + tokio::select! { + opt = stream.recv() => { + msg_opt = opt; + } + () = sleep(Duration::from_secs(5)) => { + if needs_flush { + if let Err(cause) = fd.flush().await { + error!(?cause, "failed to flush WAL log for SATA!"); + } + needs_flush = false; + } + continue; + } + } + + let Some(msg) = msg_opt else { + break; + }; + let current_time = Instant::now(); + if current_time.duration_since(last_flush) > Duration::from_secs(3) { + if let Err(cause) = fd.flush().await { + error!(?cause, "failed to flush WAL log for SATA!"); + } + + last_flush = current_time; + needs_flush = false; + } else { + needs_flush = true; + } + + let (stream_id, is_response, data) = match msg { + WriteAheadLogMessage::OpenStream(stream_id) => { + waiting_requests.insert(stream_id, VecDeque::with_capacity(1)); + fd_map.insert(stream_id, FnvHashMap::default()); + folder_map.insert(stream_id, FnvHashMap::default()); + connection_flags.insert(stream_id, SataConnectionFlags::new()); + continue; + } + WriteAheadLogMessage::CloseStream(stream_id) => { + waiting_requests.remove(&stream_id); + fd_map.remove(&stream_id); + folder_map.remove(&stream_id); + connection_flags.remove(&stream_id); + continue; + } + WriteAheadLogMessage::WriteFileRead(stream_id, file_desc, size) => { + if let Some(req_waiting) = waiting_requests.get_mut(&stream_id) { + // TODO(mythra): god this is messy, clean this shit up. + if let Some(front) = req_waiting.front() { + match &front { + &mapper::WaitingRequest::WriteFile(path, _) => { + let final_path = if let Some(p) = fd_map + .get(&stream_id) + .expect("impossible: fd_map / waiting_request out of sync?") + .get(&file_desc) + { + p.to_owned() + } else { + "".to_owned() + }; + + if path == &final_path { + if let Err(cause) = + fd.write_all(format!(" ->{size}\n").as_bytes()).await + { + error!( + ?cause, + "Failed to write request to WAL log! MAY BE INCOMPLETE!" + ); + } + } else { + error!( + stream_id, + fd = file_desc, + size, + "Mismatched WriteFileRead????" + ); + } + } + _ => { + error!( + stream_id, + fd = file_desc, + size, + "Got WriteFileRead for non write-file request???" + ); + } + } + } else { + error!( + stream_id, + fd = file_desc, + size, + "Got WriteFileRead when not waiting a request???" + ); + } + } else { + error!( + stream_id, + "Got WriteFileRead for stream that doesn't exist???" + ); + } + + continue; + } + WriteAheadLogMessage::StreamEvent(sid, isresp, data) => (sid, isresp, data), + }; + + if is_response { + let Some(list_mut) = waiting_requests.get_mut(&stream_id) else { + error!( + stream_id, + "got WAL message for stream that doesn't exist???" + ); + return; + }; + let Some(req) = list_mut.pop_front() else { + error!( + stream_id, + response = format!("{data:02x?}"), + "got WAL response for stream that is not waiting on request???" + ); + return; + }; + let fd_map_mut = fd_map + .get_mut(&stream_id) + .expect("impossible fd_map/waiting_requests out of sync!"); + let folder_map_mut = folder_map + .get_mut(&stream_id) + .expect("impossible folder_map/waiting_requests out of sync!"); + let conn_flags = connection_flags + .get(&stream_id) + .expect("impossible connection_flags/waiting_requests out of sync!"); + + let resp = mapper::WaitingResponse::parse(conn_flags, &req, data); + match resp { + mapper::WaitingResponse::OpenFile(fdres) => { + if let Ok(fd) = fdres.result() { + if let mapper::WaitingRequest::OpenFile(path, _) = req { + fd_map_mut.insert(fd, path); + } + } + } + mapper::WaitingResponse::OpenFolder(fdres) => { + if let Ok(fd) = fdres.result() { + if let mapper::WaitingRequest::OpenFolder(path) = req { + folder_map_mut.insert(fd, path); + } + } + } + mapper::WaitingResponse::CloseFile(rc) => { + if rc.0 == 0 { + if let mapper::WaitingRequest::CloseFile(fd, _) = req { + fd_map_mut.remove(&fd); + } + } + } + mapper::WaitingResponse::CloseFolder(rc) => { + if rc.0 == 0 { + if let mapper::WaitingRequest::CloseFolder(fd, _) = req { + folder_map_mut.remove(&fd); + } + } + } + mapper::WaitingResponse::Pong(ffio, csr) => { + conn_flags.set_ffio_enabled(ffio); + conn_flags.set_csr_enabled(csr); + } + _ => {} + } + if let Err(cause) = fd.write_all(format!("<-{resp}\n").as_bytes()).await { + error!( + ?cause, + "Failed to write response to WAL log! MAY BE INCOMPLETE!" + ); + } + } else { + let Some(list_mut) = waiting_requests.get_mut(&stream_id) else { + error!( + stream_id, + "got WAL message for stream that doesn't exist???" + ); + return; + }; + + let req = mapper::WaitingRequest::parse( + fd_map + .get(&stream_id) + .expect("impossible fd_map/folder_map/waiting_requests out of sync!"), + folder_map + .get(&stream_id) + .expect("impossible fd_map/folder_map/waiting_requests out of sync!"), + data, + ); + if let Err(cause) = fd.write_all(format!("->{req}\n").as_bytes()).await { + error!( + ?cause, + "Failed to write request to WAL log! MAY BE INCOMPLETE!" + ); + } + list_mut.push_back(req); + } + } + + if let Err(cause) = fd.flush().await { + error!(?cause, "failed to flush WAL log for SATA!"); + } +} diff --git a/pkg/cat-dev/src/fsemul/pcfs/sata/server/write_file.rs b/pkg/cat-dev/src/fsemul/pcfs/sata/server/write_file.rs index 65a24c7..4ef9786 100644 --- a/pkg/cat-dev/src/fsemul/pcfs/sata/server/write_file.rs +++ b/pkg/cat-dev/src/fsemul/pcfs/sata/server/write_file.rs @@ -5,11 +5,8 @@ use crate::{ fsemul::pcfs::{ errors::PCFSApiError, sata::{ - proto::{ - MoveToFileLocation, SataRequest, SataResponse, SataResultCode, - SataWriteFilePacketBody, - }, - server::{PCFSServerState, SataConnectionFlags}, + proto::{SataRequest, SataResponse, SataResultCode, SataWriteFilePacketBody}, + server::{PCFSServerState, SataConnectionFlags, wal::WriteAheadLog}, }, }, net::models::Request, @@ -34,56 +31,34 @@ pub async fn handle_write_file( .get::() .cloned() .ok_or_else(|| PCFSApiError::MissingCriticalExtension("SataConnectionFlags".to_owned()))?; + let opt_wal = req.extensions().get::(); let state = req.state(); let request = SataRequest::::try_from(req.body().clone())?; let request_header = request.header().clone(); let packet = request.body(); if packet.should_move() { - match packet.move_to_pointer() { - MoveToFileLocation::Begin => { - if state - .host_filesystem() - .seek_file(packet.file_descriptor(), true, Some(req.stream_id())) - .await - .is_err() - { - debug!( - packet.fd = packet.file_descriptor(), - packet.typ = "PCFSSrvWriteFile", - "Failed to seek to beginning of file!", - ); + if let Err(cause) = packet + .move_to_pointer() + .do_move( + state.host_filesystem(), + packet.file_descriptor(), + Some(req.stream_id()), + ) + .await + { + debug!( + ?cause, + packet.fd = packet.file_descriptor(), + packet.typ = "PCFSSrvWriteFile", + "Failed to seek file!", + ); - return Ok(SataResponse::new( - state.pid(), - request_header, - SataResultCode::error(FS_ERROR), - )); - } - } - MoveToFileLocation::Current => { - // Luckily to move to current, we don't need to move at all. - } - MoveToFileLocation::End => { - if state - .host_filesystem() - .seek_file(packet.file_descriptor(), false, Some(req.stream_id())) - .await - .is_err() - { - debug!( - packet.fd = packet.file_descriptor(), - packet.typ = "PCFSSrvWriteFile", - "Failed to seek to end of file!", - ); - - return Ok(SataResponse::new( - state.pid(), - request_header, - SataResultCode::error(FS_ERROR), - )); - } - } + return Ok(SataResponse::new( + state.pid(), + request_header, + SataResultCode::error(FS_ERROR), + )); } } @@ -92,15 +67,36 @@ pub async fn handle_write_file( .map_err(|_| CatBridgeError::UnsupportedBitsPerCore)?; // Bypass header and such checks... let buff = req.unsafe_read_more_bytes_from_stream(len_needed).await?; - state + + if let Some(wal) = opt_wal { + wal.record_oob_file_write_read(req.stream_id(), packet.file_descriptor(), len_needed) + .await; + } + + if let Err(cause) = state .host_filesystem .write_file(packet.file_descriptor(), buff, Some(req.stream_id())) - .await?; + .await + { + debug!( + ?cause, + packet.fd = packet.file_descriptor(), + packet.typ = "PCFSSrvWriteFile", + "Failed to write file!", + ); + + return Ok(SataResponse::new( + state.pid(), + request_header, + SataResultCode::error(FS_ERROR), + )); + } - Ok(SataResponse::new( + Ok(SataResponse::new_force_zero_version( state.pid(), request_header, - SataResultCode::success(), + // Abuse result code, we actually write the total amount of bytes written. + SataResultCode::error(packet.block_size() * packet.block_count()), )) } else { todo!("Implement non-FFIO support.") diff --git a/pkg/cat-dev/src/fsemul/sdio/mod.rs b/pkg/cat-dev/src/fsemul/sdio/mod.rs index f7dfa50..e14aee0 100644 --- a/pkg/cat-dev/src/fsemul/sdio/mod.rs +++ b/pkg/cat-dev/src/fsemul/sdio/mod.rs @@ -9,13 +9,17 @@ //! 7975), and "SDIO Block Data" (by default port 7976), which actually //! interact over two totally independent TCP streams. +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub mod client; +#[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] pub(crate) mod data_stream; pub mod errors; +#[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] pub mod proto; +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] pub mod server; diff --git a/pkg/cat-dev/src/fsemul/sdio/proto/message.rs b/pkg/cat-dev/src/fsemul/sdio/proto/message.rs index 297f8b9..e576e57 100644 --- a/pkg/cat-dev/src/fsemul/sdio/proto/message.rs +++ b/pkg/cat-dev/src/fsemul/sdio/proto/message.rs @@ -3,6 +3,7 @@ use crate::{errors::NetworkParseError, fsemul::sdio::errors::SDIOProtocolError}; use bytes::{BufMut, Bytes, BytesMut}; +use std::ffi::CStr; use valuable::{ EnumDef, Enumerable, Fields, NamedField, NamedValues, StructDef, Structable, Valuable, Value, Variant, VariantDef, Visit, @@ -14,11 +15,14 @@ pub enum SdioControlMessage { Printf(Bytes, String), /// TODO(mythra): Currently unknown, mostly used for scientist. Unknown(Vec), + /// Record the boot mode that was being used, and should be used. + RecordBootMode(String), } static SDIO_CONTROL_MESSAGE_VARIANTS: &[VariantDef<'static>] = &[ VariantDef::new("Printf", Fields::Unnamed(2)), VariantDef::new("Unknown", Fields::Unnamed(1)), + VariantDef::new("RecordBootMode", Fields::Unnamed(1)), ]; impl Enumerable for SdioControlMessage { @@ -30,6 +34,9 @@ impl Enumerable for SdioControlMessage { match self { SdioControlMessage::Printf(_, _) => Variant::Static(&SDIO_CONTROL_MESSAGE_VARIANTS[0]), SdioControlMessage::Unknown(_) => Variant::Static(&SDIO_CONTROL_MESSAGE_VARIANTS[1]), + SdioControlMessage::RecordBootMode(_) => { + Variant::Static(&SDIO_CONTROL_MESSAGE_VARIANTS[2]) + } } } } @@ -50,6 +57,9 @@ impl Valuable for SdioControlMessage { SdioControlMessage::Unknown(buff) => { visitor.visit_unnamed_fields(&[Valuable::as_value(&buff)]); } + SdioControlMessage::RecordBootMode(mode) => { + visitor.visit_unnamed_fields(&[Valuable::as_value(&mode)]); + } } } } @@ -95,6 +105,10 @@ impl TryFrom<&SdioControlMessageRequest> for Bytes { SdioControlMessage::Unknown(un) => { size += un.len(); } + SdioControlMessage::RecordBootMode(vers) => { + size += vers.len(); + size += 1; + } } } size @@ -121,6 +135,12 @@ impl TryFrom<&SdioControlMessageRequest> for Bytes { final_buff.extend(buff); break; } + SdioControlMessage::RecordBootMode(version) => { + final_buff.put_u16_le(8); + final_buff.extend(&[0x0C, 0x00, 0xFF, 0xFF, 0x11, 0x00]); + final_buff.extend(version.as_bytes()); + break; + } } } @@ -157,7 +177,7 @@ impl TryFrom for SdioControlMessageRequest { return Err(SDIOProtocolError::UnknownPrintfPacketType(value[0]).into()); } - let _character_length = u16::from_le_bytes([value[0x2], value[0x3]]); + let character_length = u16::from_le_bytes([value[0x2], value[0x3]]); let mut messages = Vec::with_capacity(1); let mut read_size = 0; loop { @@ -183,6 +203,21 @@ impl TryFrom for SdioControlMessageRequest { read_size += unknown_data.len(); messages.push(SdioControlMessage::Unknown(unknown_data)); break; + } else if message_ty == 8 { + // May be useful in the future.... + let _header = value.slice(base_offset + 2..base_offset + 8).to_vec(); + let mode = value + .slice( + base_offset + 8 + ..base_offset + 8 + (usize::from(character_length) - (base_offset + 8)), + ) + .to_vec(); + let mode_cstr = + CStr::from_bytes_until_nul(&mode).map_err(NetworkParseError::BadCString)?; + messages.push(SdioControlMessage::RecordBootMode( + mode_cstr.to_string_lossy().to_string(), + )); + break; } return Err(SDIOProtocolError::UnknownPrintfMessageType(message_ty).into()); diff --git a/pkg/cat-dev/src/fsemul/sdio/server/message.rs b/pkg/cat-dev/src/fsemul/sdio/server/message.rs index b5d210e..c51978f 100644 --- a/pkg/cat-dev/src/fsemul/sdio/server/message.rs +++ b/pkg/cat-dev/src/fsemul/sdio/server/message.rs @@ -9,21 +9,24 @@ use crate::{ }, net::{additions::StreamID, server::requestable::Body}, }; +use bytes::Bytes; use tracing::{debug, info}; pub(super) async fn handle_message( stream_id: StreamID, Body(request): Body, -) -> Result<(), CatBridgeError> { +) -> Result, CatBridgeError> { let Some(mut buff) = SDIO_PRINTF_BUFFS.get_async(&stream_id.to_raw()).await else { // Should be unreachable, but if a catastrophic error occurs. return Err(SDIONetworkError::PrintfMissingBuffer(stream_id.to_raw()).into()); }; + let mut should_process = false; for message in request.messages_owned() { match message { SdioControlMessage::Printf(_extra, to_print) => { buff.push_str(&to_print); + should_process = true; } SdioControlMessage::Unknown(buff) => { debug!( @@ -31,11 +34,19 @@ pub(super) async fn handle_message( "Unknown message type == 9 for SDIO, Not Sure How to Respond?", ); } + SdioControlMessage::RecordBootMode(version) => { + return Ok(Some(Bytes::try_from(SdioControlMessageRequest::new( + vec![SdioControlMessage::RecordBootMode(version)], + ))?)); + } } } - process_log_messages(&mut buff); - Ok(()) + if should_process { + process_log_messages(&mut buff); + } + + Ok(None) } fn process_log_messages(printf_buff: &mut String) { diff --git a/pkg/cat-dev/src/lib.rs b/pkg/cat-dev/src/lib.rs index 1373a4d..ff2af5b 100644 --- a/pkg/cat-dev/src/lib.rs +++ b/pkg/cat-dev/src/lib.rs @@ -8,8 +8,10 @@ pub mod errors; pub mod fsemul; pub mod mion; +#[cfg_attr(docsrs, doc(cfg(any(feature = "clients", feature = "servers"))))] #[cfg(any(feature = "clients", feature = "servers"))] pub mod net; +#[cfg_attr(docsrs, doc(cfg(feature = "serial")))] #[cfg(feature = "serial")] #[macro_use] pub mod serial; diff --git a/pkg/cat-dev/src/mion/cgis/control.rs b/pkg/cat-dev/src/mion/cgis/control.rs index 4649299..80175e3 100644 --- a/pkg/cat-dev/src/mion/cgis/control.rs +++ b/pkg/cat-dev/src/mion/cgis/control.rs @@ -299,6 +299,7 @@ pub async fn do_raw_control_request( Method::POST, format!("http://{mion_ip}/mion/control.cgi"), Some(encode_url_parameters(url_parameters)), + None, ) .await } diff --git a/pkg/cat-dev/src/mion/cgis/dump_eeprom.rs b/pkg/cat-dev/src/mion/cgis/dump_eeprom.rs index 480ae6d..a79d23f 100644 --- a/pkg/cat-dev/src/mion/cgis/dump_eeprom.rs +++ b/pkg/cat-dev/src/mion/cgis/dump_eeprom.rs @@ -10,7 +10,7 @@ use crate::{ }; use bytes::{BufMut, Bytes, BytesMut}; use reqwest::{Client, Method}; -use std::{fmt::Display, net::Ipv4Addr, ops::Deref}; +use std::{fmt::Display, net::Ipv4Addr, ops::Deref, time::Duration}; use tracing::debug; const EEPROM_MAX_ADDRESS: usize = 0x1E00; @@ -117,6 +117,8 @@ pub async fn do_raw_eeprom_request( Method::POST, format!("http://{mion_ip}/dbg/eeprom_dump.cgi"), Some(encode_url_parameters(url_parameters)), + // Dump operations are sometimes slow... + Some(Duration::from_secs(60)), ) .await } diff --git a/pkg/cat-dev/src/mion/cgis/mod.rs b/pkg/cat-dev/src/mion/cgis/mod.rs index f69bcac..c2d8da3 100644 --- a/pkg/cat-dev/src/mion/cgis/mod.rs +++ b/pkg/cat-dev/src/mion/cgis/mod.rs @@ -39,8 +39,14 @@ use crate::{ use bytes::Bytes; use form_urlencoded::byte_serialize; use reqwest::{Body, Client, Method, Response, Version}; -use std::{fmt::Display, ops::Deref}; -use tracing::{field::valuable, warn}; +use std::{fmt::Display, ops::Deref, time::Duration}; +use tokio::time::timeout; +use tracing::{Instrument, error_span, field::valuable, warn}; + +/// The default timeout for making HTTP requests. +/// +/// This is a relatively low value by default as most HTTP pages are fairly simple, and quick to respond. +pub const DEFAULT_HTTP_TIMEOUT: Duration = Duration::from_secs(5); /// Perform a request that attempts to remove all the logic for the 'simple' /// request cases. @@ -62,23 +68,40 @@ async fn do_simple_request( method: Method, url: String, body: Option, + req_timeout: Option, ) -> Result where BodyTy: Into, { - let mut req = client - .request(method, url) - .version(Version::HTTP_11) - .header("authorization", format!("Basic {AUTHZ_HEADER}")) - .header("content-type", "application/x-www-form-urlencoded") - .header("user-agent", concat!("cat-dev/", env!("CARGO_PKG_VERSION"))); - if let Some(body) = body { - req = req.body(body); - } - let response_body = - assert_status_and_read_body(200, req.send().await.map_err(NetworkError::HTTP)?).await?; + let span = error_span!( + "cat_dev::mion::cgis::do_simple_request", + http.method = %method, + http.url = %url, + ); + + async { + let mut req = client + .request(method, url) + .version(Version::HTTP_11) + .header("authorization", format!("Basic {AUTHZ_HEADER}")) + .header("content-type", "application/x-www-form-urlencoded") + .header("user-agent", concat!("cat-dev/", env!("CARGO_PKG_VERSION"))); + if let Some(body) = body { + req = req.body(body); + } - Ok(String::from_utf8(response_body.into()).map_err(NetworkParseError::Utf8Expected)?) + let timeout_time = req_timeout.unwrap_or(DEFAULT_HTTP_TIMEOUT); + let response_body = timeout( + timeout_time, + assert_status_and_read_body(200, req.send().await.map_err(NetworkError::HTTP)?), + ) + .await + .map_err(|_| NetworkError::Timeout(timeout_time))??; + + Ok(String::from_utf8(response_body.into()).map_err(NetworkParseError::Utf8Expected)?) + } + .instrument(span) + .await } fn encode_url_parameters(parameters: &[(impl Deref, impl Display)]) -> String { diff --git a/pkg/cat-dev/src/mion/cgis/setup.rs b/pkg/cat-dev/src/mion/cgis/setup.rs index a96b399..15b1b93 100644 --- a/pkg/cat-dev/src/mion/cgis/setup.rs +++ b/pkg/cat-dev/src/mion/cgis/setup.rs @@ -38,6 +38,7 @@ pub async fn get_setup_parameters_with_raw_client( Method::GET, format!("http://{mion_ip}/setup.cgi"), None, + None, ) .await?; diff --git a/pkg/cat-dev/src/mion/cgis/signal_get.rs b/pkg/cat-dev/src/mion/cgis/signal_get.rs index e9aec51..4466037 100644 --- a/pkg/cat-dev/src/mion/cgis/signal_get.rs +++ b/pkg/cat-dev/src/mion/cgis/signal_get.rs @@ -75,6 +75,7 @@ pub async fn do_raw_signal_http_request( Method::POST, format!("http://{mion_ip}/signal_get.cgi"), Some(encode_url_parameters(url_parameters)), + None, ) .await } diff --git a/pkg/cat-dev/src/mion/cgis/status.rs b/pkg/cat-dev/src/mion/cgis/status.rs index dd26700..3cd1c84 100644 --- a/pkg/cat-dev/src/mion/cgis/status.rs +++ b/pkg/cat-dev/src/mion/cgis/status.rs @@ -79,6 +79,7 @@ pub async fn do_raw_status_request( Method::POST, format!("http://{mion_ip}/mion/status.cgi"), Some(encode_url_parameters(url_parameters)), + None, ) .await } diff --git a/pkg/cat-dev/src/mion/cgis/update.rs b/pkg/cat-dev/src/mion/cgis/update.rs index b93422f..6bfb0fe 100644 --- a/pkg/cat-dev/src/mion/cgis/update.rs +++ b/pkg/cat-dev/src/mion/cgis/update.rs @@ -58,6 +58,7 @@ pub async fn get_versions_with_raw_client( Method::GET, format!("http://{mion_ip}/update.cgi"), None, + None, ) .await?; diff --git a/pkg/cat-dev/src/mion/errors.rs b/pkg/cat-dev/src/mion/errors.rs index a5c27f6..2af96c1 100644 --- a/pkg/cat-dev/src/mion/errors.rs +++ b/pkg/cat-dev/src/mion/errors.rs @@ -24,6 +24,7 @@ use crate::{ #[derive(Error, Diagnostic, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum MIONAPIError { + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] #[error(transparent)] #[diagnostic(transparent)] @@ -58,18 +59,21 @@ pub enum MIONAPIError { #[error(transparent)] #[diagnostic(transparent)] Firmware(#[from] MIONFirmwareAPIError), + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] #[error(transparent)] #[diagnostic(transparent)] ParameterSpace(#[from] MIONParameterAPIError), } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for APIError { fn from(value: MIONCGIApiError) -> Self { Self::MION(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for CatBridgeError { fn from(value: MIONCGIApiError) -> Self { @@ -88,12 +92,14 @@ impl From for CatBridgeError { } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for APIError { fn from(value: MIONParameterAPIError) -> Self { Self::MION(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for CatBridgeError { fn from(value: MIONParameterAPIError) -> Self { @@ -105,16 +111,19 @@ impl From for CatBridgeError { #[derive(Error, Diagnostic, Debug, PartialEq, Eq)] #[non_exhaustive] pub enum MIONProtocolError { + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// Errors related to CGI, and HTML pages. #[error(transparent)] #[diagnostic(transparent)] CGI(#[from] MIONCGIErrors), + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// Errors related to the CONTROL protocol for MION. #[error(transparent)] #[diagnostic(transparent)] Control(#[from] MIONControlProtocolError), + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// Errors related to the PARAMETER SPACE protocol for MION. #[error(transparent)] @@ -133,18 +142,21 @@ impl From for CatBridgeError { } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for NetworkParseError { fn from(value: MIONCGIErrors) -> Self { Self::MION(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for NetworkError { fn from(value: MIONCGIErrors) -> Self { Self::Parse(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for CatBridgeError { fn from(value: MIONCGIErrors) -> Self { @@ -152,18 +164,21 @@ impl From for CatBridgeError { } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for NetworkParseError { fn from(value: MIONParamProtocolError) -> Self { Self::MION(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for NetworkError { fn from(value: MIONParamProtocolError) -> Self { Self::Parse(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for CatBridgeError { fn from(value: MIONParamProtocolError) -> Self { @@ -171,18 +186,21 @@ impl From for CatBridgeError { } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for NetworkParseError { fn from(value: MIONControlProtocolError) -> Self { Self::MION(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for NetworkError { fn from(value: MIONControlProtocolError) -> Self { Self::Parse(value.into()) } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl From for CatBridgeError { fn from(value: MIONControlProtocolError) -> Self { diff --git a/pkg/cat-dev/src/mion/mod.rs b/pkg/cat-dev/src/mion/mod.rs index 680f0cb..ee25de4 100644 --- a/pkg/cat-dev/src/mion/mod.rs +++ b/pkg/cat-dev/src/mion/mod.rs @@ -7,14 +7,18 @@ //! In general if you're trying to look for things relating to the bridge as a //! whole, you're _probably_ really actually talking to the MION. +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub mod cgis; +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub mod discovery; pub mod errors; pub mod firmware; +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub mod parameter; +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub mod proto; diff --git a/pkg/cat-dev/src/net/client/tcp.rs b/pkg/cat-dev/src/net/client/tcp.rs index d2c4387..fe94311 100644 --- a/pkg/cat-dev/src/net/client/tcp.rs +++ b/pkg/cat-dev/src/net/client/tcp.rs @@ -20,6 +20,20 @@ //! from getting in our way. So our write calls will always lead to //! hopefully one write call on the other side. //! +//! ## Notes about Concurrency +//! +//! This TCP Client unfortunately has to make the sacrifice and process one +//! packet per stream at a time. While you can have as many TCP streams as you +//! want, and we should be able to handle many at the same time! Unfortunately +//! the ordered nature of TCP, along with some protocol designs implemented by +//! nintendo means this server must also force that we process one packet per +//! tcp stream at a time. +//! +//! Most notably this comes from the fact that our file servers will +//! consistently break their normal "NAGLE" protection, and we have to do just +//! raw reads of N bytes from the stream (in both ways), _BEFORE_ processing +//! another request. +//! //! ## API Notes //! //! Most, TCP Clients only expect to connect to a single server, and have just diff --git a/pkg/cat-dev/src/net/handlers/on_stream_begin_handlers.rs b/pkg/cat-dev/src/net/handlers/on_stream_begin_handlers.rs index cde7615..255952e 100644 --- a/pkg/cat-dev/src/net/handlers/on_stream_begin_handlers.rs +++ b/pkg/cat-dev/src/net/handlers/on_stream_begin_handlers.rs @@ -23,6 +23,7 @@ use crate::net::server::models::{FromResponseStreamEvent, ResponseStreamEvent}; /// /// `ParamTy` is kept to prevent generation of conflicting type implementations /// of this trait. It however is not actually needed by any of our code. +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub trait OnRequestStreamBeginHandler { type Future: Future> + Send + 'static; @@ -31,6 +32,7 @@ pub trait OnRequestStreamBeginHandler OnRequestStreamBeginHandler<(), State> for UnderlyingFnType @@ -48,6 +50,7 @@ where } /// Allow any async function with a single consuming argument. +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl OnRequestStreamBeginHandler for UnderlyingFnType @@ -74,6 +77,7 @@ where /// /// `ParamTy` is kept to prevent generation of conflicting type implementations /// of this trait. It however is not actually needed by any of our code. +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] pub trait OnResponseStreamBeginHandler { type Future: Future> + Send + 'static; @@ -81,7 +85,8 @@ pub trait OnResponseStreamBeginHandler) -> Self::Future; } -/// Allow any async function without arguments to be a handler +/// Allow any async function without arguments to be a handler. +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] impl OnResponseStreamBeginHandler<(), State> for UnderlyingFnType @@ -99,6 +104,7 @@ where } /// Allow any async function with a single consuming argument. +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] impl OnResponseStreamBeginHandler for UnderlyingFnType @@ -121,6 +127,7 @@ macro_rules! fn_to_on_connection_handler { [$($ty:ident),*], $last:ident ) => { #[allow(non_snake_case, unused_mut)] + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl OnRequestStreamBeginHandler<($($ty,)* $last,), State> for UnderlyingFnType where @@ -146,6 +153,7 @@ macro_rules! fn_to_on_connection_handler { } #[allow(non_snake_case, unused_mut)] + #[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] impl OnResponseStreamBeginHandler<($($ty,)* $last,), State> for UnderlyingFnType where @@ -228,6 +236,7 @@ where } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl Service> for OnStreamBeginHandlerAsService @@ -250,6 +259,7 @@ where } } +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] impl Service> for OnStreamBeginHandlerAsService diff --git a/pkg/cat-dev/src/net/handlers/on_stream_end_handlers.rs b/pkg/cat-dev/src/net/handlers/on_stream_end_handlers.rs index f48f40f..9d7becf 100644 --- a/pkg/cat-dev/src/net/handlers/on_stream_end_handlers.rs +++ b/pkg/cat-dev/src/net/handlers/on_stream_end_handlers.rs @@ -14,6 +14,7 @@ use crate::net::client::models::{FromRequestStreamEvent, RequestStreamEvent}; #[cfg(feature = "servers")] use crate::net::server::models::{FromResponseStreamEvent, ResponseStreamEvent}; +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// A stream ending/(on disconnect) handler, attempts to be an incredibly /// thin layer between a function, and the actual ending handler. @@ -29,6 +30,7 @@ pub trait OnRequestStreamEndHandler) -> Self::Future; } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] /// Allow any async function without arguments to be a handler impl OnRequestStreamEndHandler<(), State> @@ -47,6 +49,7 @@ where } /// Allow any async function with a single consuming argument. +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl OnRequestStreamEndHandler for UnderlyingFnType @@ -64,6 +67,7 @@ where } } +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] /// A stream ending/(on disconnect) handler, attempts to be an incredibly /// thin layer between a function, and the actual ending handler. @@ -79,6 +83,7 @@ pub trait OnResponseStreamEndHandler) -> Self::Future; } +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] /// Allow any async function without arguments to be a handler impl OnResponseStreamEndHandler<(), State> @@ -97,6 +102,7 @@ where } /// Allow any async function with a single consuming argument. +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] impl OnResponseStreamEndHandler for UnderlyingFnType @@ -119,6 +125,7 @@ macro_rules! fn_to_on_disconnect_handler { [$($ty:ident),*], $last:ident ) => { #[allow(non_snake_case, unused_mut)] + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl OnRequestStreamEndHandler<($($ty,)* $last,), State> for UnderlyingFnType where @@ -144,6 +151,7 @@ macro_rules! fn_to_on_disconnect_handler { } #[allow(non_snake_case, unused_mut)] + #[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] impl OnResponseStreamEndHandler<($($ty,)* $last,), State> for UnderlyingFnType where @@ -226,6 +234,7 @@ where } } +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] impl Service> for OnStreamEndHandlerAsService @@ -248,6 +257,7 @@ where } } +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] impl Service> for OnStreamEndHandlerAsService diff --git a/pkg/cat-dev/src/net/mod.rs b/pkg/cat-dev/src/net/mod.rs index 474f1d3..c1baddf 100644 --- a/pkg/cat-dev/src/net/mod.rs +++ b/pkg/cat-dev/src/net/mod.rs @@ -5,12 +5,14 @@ //! implement the rest of things like a "TCP Server", or "TCP Client". pub mod additions; +#[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub mod client; pub mod errors; mod ext_map; pub mod handlers; pub mod models; +#[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] pub mod server; diff --git a/pkg/cat-dev/src/net/models.rs b/pkg/cat-dev/src/net/models.rs index 6251719..fed64c7 100644 --- a/pkg/cat-dev/src/net/models.rs +++ b/pkg/cat-dev/src/net/models.rs @@ -24,6 +24,8 @@ use crate::{errors::NetworkError, net::errors::CommonNetNetworkError}; use std::sync::Arc; #[cfg(feature = "servers")] use tokio::{io::AsyncReadExt, net::TcpStream, sync::Mutex}; +#[cfg(feature = "servers")] +use tracing::error; /// Used to do reference-to-value conversions thus not consuming the input value. /// @@ -60,11 +62,17 @@ pub struct Request { /// /// This will still call 'post nagle hook', 'trace io', and will still /// obey NAGLE timeouts. It just overrides the _kind_ of NAGLE we do. + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] explicit_read_amount: Option, /// Allow accessing the raw underlying stream while processing the request. + #[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] - stream_access: Option>>>, + #[allow( + // TODO(mythra): refactor to type. + clippy::type_complexity, + )] + stream_access: Option, TcpStream)>>>>, } impl Request @@ -108,6 +116,7 @@ impl Request { } } + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] #[must_use] pub fn new_with_state_and_read_amount( @@ -129,14 +138,19 @@ impl Request { } } + #[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] + #[allow( + // TODO(mythra): refactor to type. + clippy::type_complexity, + )] #[must_use] pub fn new_with_state_and_stream( body: Bytes, source_address: SocketAddr, state: State, stream_id: Option, - stream: Arc>>, + stream_and_nagle_cache: Arc, TcpStream)>>>, ) -> Self { Self { body, @@ -146,7 +160,7 @@ impl Request { stream_id, #[cfg(feature = "clients")] explicit_read_amount: None, - stream_access: Some(stream), + stream_access: Some(stream_and_nagle_cache), } } @@ -181,7 +195,8 @@ impl Request { /// /// This is a utility only available when we are a client, and are receiving /// a packet that changes what our nagle split is for it's specific response - /// while keeping the nalge the same otherwise. + /// while keeping the nagle the same otherwise. + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] #[must_use] pub const fn explicit_read_amount(&self) -> Option { @@ -190,6 +205,7 @@ impl Request { /// Override the current NAGLE algorithm being used by this client for this /// single request/response pair. Do a single non-nagle'd receive. + #[cfg_attr(docsrs, doc(cfg(feature = "clients")))] #[cfg(feature = "clients")] pub const fn set_explicit_read_amount(&mut self, new_read_amount: usize) { self.explicit_read_amount = Some(new_read_amount); @@ -208,6 +224,7 @@ impl Request { /// /// If the request has been moved outside of it's original processing place, /// and it is no longer possible to read from the stream. + #[cfg_attr(docsrs, doc(cfg(feature = "servers")))] #[cfg(feature = "servers")] pub async fn unsafe_read_more_bytes_from_stream( &self, @@ -216,18 +233,30 @@ impl Request { if let Some(strm) = self.stream_access.as_ref() { let mut guard = strm.lock().await; - if let Some(stream) = guard.as_mut() { + if let Some((opt_cache, stream)) = guard.as_mut() { let mut buff = BytesMut::with_capacity(to_read); - stream.readable().await.map_err(NetworkError::IO)?; - let mut needed = to_read; - while needed > 0 { - let read = stream.read_buf(&mut buff).await.map_err(NetworkError::IO)?; - needed -= read; + + if let Some(cache) = opt_cache.as_mut() { + if cache.len() <= to_read { + buff = cache.split(); + } else { + buff = cache.split_to(to_read); + } + } + + if buff.len() < to_read { + stream.readable().await.map_err(NetworkError::IO)?; + let mut needed = to_read - buff.len(); + while needed > 0 { + let read = stream.read_buf(&mut buff).await.map_err(NetworkError::IO)?; + needed -= read; + } } return Ok::(buff.freeze()); } } + error!("called unsafe_read_more_bytes on a stream that is not processing!"); Err(CommonNetNetworkError::StreamNoLongerProcessing.into()) } diff --git a/pkg/cat-dev/src/net/server/tcp.rs b/pkg/cat-dev/src/net/server/tcp.rs index 9dc645c..98e3f5b 100644 --- a/pkg/cat-dev/src/net/server/tcp.rs +++ b/pkg/cat-dev/src/net/server/tcp.rs @@ -27,6 +27,20 @@ //! as NAGLE's algorithim is usually to blame for weird behaviors here. Where //! a device will combine multiple small packets together. //! +//! ## Notes about Concurrency +//! +//! This TCP Server unfortunately has to make the sacrifice and process one +//! packet per stream at a time. While you can have as many TCP streams as you +//! want, and we should be able to handle many at the same time! Unfortunately +//! the ordered nature of TCP, along with some protocol designs implemented by +//! nintendo means this server must also force that we process one packet per +//! tcp stream at a time. +//! +//! Most notably this comes from the fact that our file servers will +//! consistently break their normal "NAGLE" protection, and we have to do just +//! raw reads of N bytes from the stream (in both ways), _BEFORE_ processing +//! another request. +//! //! ## SLOW-loris //! //! As mentioned TCP is built ontop of a stream, there is a chance that someone @@ -138,7 +152,7 @@ use scc::HashMap as ConcurrentMap; use std::{ convert::Infallible, fmt::{Debug, Formatter, Result as FmtResult}, - net::SocketAddr, + net::{IpAddr, SocketAddr}, sync::{Arc, LazyLock, atomic::Ordering}, time::{Duration, SystemTime}, }; @@ -379,6 +393,12 @@ impl TCPServer { }) } + /// The IP address we should connect to bind too. + #[must_use] + pub const fn ip(&self) -> IpAddr { + self.address_to_bind_or_connect_to.ip() + } + /// Get the port that we're either binding too, or connecting too. #[must_use] pub const fn port(&self) -> u16 { @@ -1087,13 +1107,13 @@ impl TCPServer { buff = existing_buff; } - let lockable_stream = Arc::new(Mutex::new(Some(stream))); while let Some((start_of_packet, end_of_packet)) = nagle_guard.split(&buff)? { let remaining_buff = buff.split_off(end_of_packet); let _start_of_buff = buff.split_to(start_of_packet); let req_body = buff.freeze(); buff = remaining_buff; + let lockable_stream = Arc::new(Mutex::new(Some((Some(buff), stream)))); let mut request_object = Request::new_with_state_and_stream( req_body, client_address, @@ -1118,13 +1138,19 @@ impl TCPServer { "internal queue failure will not send disconnect/response." ); } - } - { - let mut done_lock = lockable_stream.lock().await; - if let Some(strm) = done_lock.take() { - stream = strm; - } else { - return Err(CommonNetNetworkError::StreamNoLongerProcessing.into()); + + { + let mut done_lock = lockable_stream.lock().await; + if let Some((newer_buff, strm)) = done_lock.take() { + if let Some(newest_buff) = newer_buff { + buff = newest_buff; + } else { + return Err(CommonNetNetworkError::StreamNoLongerProcessing.into()); + } + stream = strm; + } else { + return Err(CommonNetNetworkError::StreamNoLongerProcessing.into()); + } } } diff --git a/pkg/cat-dev/src/serial/underlying/mod.rs b/pkg/cat-dev/src/serial/underlying/mod.rs index 500c35f..cd556d9 100644 --- a/pkg/cat-dev/src/serial/underlying/mod.rs +++ b/pkg/cat-dev/src/serial/underlying/mod.rs @@ -564,7 +564,7 @@ impl std::os::windows::io::FromRawHandle for SyncSerialPort { unsafe fn from_raw_handle(handle: std::os::windows::io::RawHandle) -> Self { Self { inner: RawSyncSerialPort { - fd: std::fs::File::from_raw_handle(handle), + fd: unsafe { std::fs::File::from_raw_handle(handle) }, }, } } diff --git a/pkg/cat-dev/src/serial/underlying/sys/unix.rs b/pkg/cat-dev/src/serial/underlying/sys/unix.rs index c073910..481339e 100644 --- a/pkg/cat-dev/src/serial/underlying/sys/unix.rs +++ b/pkg/cat-dev/src/serial/underlying/sys/unix.rs @@ -564,7 +564,7 @@ impl RawSyncSerialPort { revents: 0, }; unsafe { - Self::check(libc::poll(&mut poll_fd, 1, timeout_ms as i32))?; + Self::check(libc::poll(&raw mut poll_fd, 1, timeout_ms as i32))?; } Ok(poll_fd.revents != 0) }