From e040428c91a1124766903fc981c395824e28a6b8 Mon Sep 17 00:00:00 2001 From: Rudi Floren Date: Mon, 10 Aug 2026 16:59:43 +0200 Subject: [PATCH 1/2] feat(lance-io): select TLS provider to keep aws-lc-rs out of non-AWS builds opendal defaults to reqwest's rustls TLS, which pins the aws-lc-rs crypto provider and its cmake build dependency into every build. Build opendal with default-features = false and add tls-aws-lc-rs (default) and tls-no-provider features so non-AWS backends can ride on ring or a no-provider rustls instead. These map to opendal 0.58's granular http-transport-reqwest-rustls[-no-provider] features rather than the reqwest-rustls-* aliases, which additionally re-enable the aws-lc-rs transport. tls-no-provider still yields a working reqwest transport; the application installs a rustls CryptoProvider before first use. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.toml | 10 +++++++++- rust/lance-io/Cargo.toml | 5 ++++- rust/lance-namespace-impls/Cargo.toml | 2 +- rust/lance/Cargo.toml | 4 +++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 30128ad8a5b..deced3170c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -175,7 +175,15 @@ moka = { version = "0.12", features = ["future", "sync"] } ndarray = { version = "0.16.1", features = ["matrixmultiply-threading"] } num-traits = "0.2" object_store = { version = "0.13.2" } -opendal = { version = "0.58.1" } +# default-features = false drops opendal's aws-lc-rs TLS default; lance-io's tls-* features pick the transport. +opendal = { version = "0.58.1", default-features = false, features = [ + "auto-register-services", + "executors-tokio", + "layers-concurrent-limit", + "layers-logging", + "layers-retry", + "layers-timeout", +] } object_store_opendal = { version = "0.58" } pin-project = "1.0" path_abs = "0.5" diff --git a/rust/lance-io/Cargo.toml b/rust/lance-io/Cargo.toml index 03c6780263a..113bb9faa8d 100644 --- a/rust/lance-io/Cargo.toml +++ b/rust/lance-io/Cargo.toml @@ -62,7 +62,7 @@ name = "scheduler" harness = false [features] -default = ["aws", "azure", "gcp"] +default = ["aws", "azure", "gcp", "tls-aws-lc-rs"] metrics = ["dep:metrics"] gcs-test = [] goosefs-test = [] @@ -74,6 +74,9 @@ goosefs = ["dep:opendal", "opendal/services-goosefs", "dep:object_store_opendal" tencent = ["dep:opendal", "opendal/services-cos", "dep:object_store_opendal"] huggingface = ["dep:opendal", "opendal/services-huggingface", "dep:object_store_opendal"] tos = ["dep:opendal", "opendal/services-tos", "dep:object_store_opendal"] +# Granular opendal transports (not the reqwest-rustls-* aliases, which also pull aws-lc-rs); tls-no-provider needs an app-installed rustls provider. +tls-aws-lc-rs = ["opendal?/http-transport-reqwest-rustls"] +tls-no-provider = ["opendal?/http-transport-reqwest-rustls-no-provider"] tos-test = ["tos"] test-util = [] diff --git a/rust/lance-namespace-impls/Cargo.toml b/rust/lance-namespace-impls/Cargo.toml index d682d39f02b..2a21f6cf616 100644 --- a/rust/lance-namespace-impls/Cargo.toml +++ b/rust/lance-namespace-impls/Cargo.toml @@ -12,7 +12,7 @@ categories.workspace = true rust-version.workspace = true [features] -default = ["dir-aws", "dir-azure", "dir-gcp", "dir-oss", "dir-huggingface"] +default = ["dir-aws", "dir-azure", "dir-gcp", "dir-oss", "dir-huggingface", "lance-io/tls-aws-lc-rs"] rest = ["dep:reqwest", "dep:serde"] rest-adapter = ["dep:axum", "dep:tower", "dep:tower-http", "dep:serde"] # Cloud storage features for directory implementation - align with lance-io diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml index 64253478000..4afcfa755dc 100644 --- a/rust/lance/Cargo.toml +++ b/rust/lance/Cargo.toml @@ -139,7 +139,7 @@ parquet = { version = "58", default-features = false, features = ["arrow", "asyn reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } [features] -default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo"] +default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo", "tls-aws-lc-rs"] backtrace = ["lance-core/backtrace"] fp16kernels = ["lance-linalg/fp16kernels"] # Prevent dynamic linking of lzma, which comes from datafusion @@ -162,6 +162,8 @@ tencent = ["lance-io/tencent"] goosefs = ["lance-io/goosefs"] tos = ["lance-io/tos"] huggingface = ["lance-io/huggingface"] +tls-aws-lc-rs = ["lance-io/tls-aws-lc-rs"] +tls-no-provider = ["lance-io/tls-no-provider"] # Publish object store metrics via the `metrics` crate. metrics = ["lance-io/metrics"] geo = ["lance-datafusion/geo", "lance-index/geo"] From 8311c44c0bfd3bd889403ae80e8c222caff2190a Mon Sep 17 00:00:00 2001 From: Rudi Floren Date: Mon, 10 Aug 2026 17:21:35 +0200 Subject: [PATCH 2/2] ci(rust): guard non-AWS builds against aws-lc-rs/cmake cargo-deny resolves the dependency graph with all features enabled, so it cannot assert that aws-lc-rs is absent from a specific feature combo. Add a script that resolves each non-AWS backend under tls-no-provider and fails if aws-lc-rs, aws-lc-sys, aws-lc-fips-sys, or cmake appears. It checks the functional config and requires a reqwest HTTP transport so a transport-less build cannot pass vacuously. Run it from the cargo-deny job. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/rust.yml | 6 ++++ ci/check_forbidden_deps.sh | 63 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100755 ci/check_forbidden_deps.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dcf61b83e93..3000166c6cf 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -16,6 +16,7 @@ on: - Cargo.toml - Cargo.lock - deny.toml + - ci/check_forbidden_deps.sh permissions: contents: read @@ -101,6 +102,11 @@ jobs: with: log-level: warn command: check + # cargo-deny resolves with all features, so it cannot assert a crate is + # absent from a specific feature combo. This checks each non-AWS backend. + - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1 + - name: Check non-AWS builds stay free of aws-lc-rs/cmake + run: ci/check_forbidden_deps.sh linux-coverage-build: runs-on: "ubuntu-24.04-8x" diff --git a/ci/check_forbidden_deps.sh b/ci/check_forbidden_deps.sh new file mode 100755 index 00000000000..ca5cc9a571c --- /dev/null +++ b/ci/check_forbidden_deps.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Lance promises that non-AWS cloud builds can be built without aws-lc-rs. +# Consumers building slim images for a single backend (e.g. GCS-only) rely on +# this: aws-lc-rs compiles C/assembly through its cmake build dependency, +# which bloats builds, complicates cross-compilation, and forces a second +# crypto provider on deployments that standardize on ring. This script +# enforces the promise by resolving each non-AWS backend under the +# provider-neutral TLS feature (tls-no-provider) and failing if aws-lc-rs or +# cmake appears. +# +# It checks the FUNCTIONAL configuration (backend + tls-no-provider), not the +# bare backend: opendal's HTTP transport is opt-in, so a bare backend has no +# TLS stack and would pass this check vacuously while failing at runtime. To +# guard against that, each combination must also still resolve a reqwest HTTP +# transport (opendal-http-transport-reqwest); a missing transport is an error. +# +# The usual way this regresses is a dependency bump changing a TLS default, +# not a local change: opendal's default reqwest transport pins aws-lc-rs, and +# its reqwest-rustls-* feature aliases re-enable it, so lance-io wires the +# granular http-transport-reqwest-rustls-no-provider feature instead. +# +# If this check fails, fix the feature wiring rather than exempting the crate. +# Feature combos that include the AWS SDK are not checked: it links aws-lc-rs +# itself via aws-smithy-http-client. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +FORBIDDEN=(aws-lc-rs aws-lc-sys aws-lc-fips-sys cmake) +REQUIRED=(opendal-http-transport-reqwest) + +check() { + local desc="$1" + shift + + local deps + deps=$(cargo tree --locked -e normal,build --prefix none --format '{p}' "$@" | awk '{print $1}' | sort -u) + + local failed=0 + for crate in "${FORBIDDEN[@]}"; do + if grep -qx "$crate" <<<"$deps"; then + echo "error: forbidden dependency '$crate' found in $desc, pulled in via:" + cargo tree --locked -e normal,build "$@" -i "$crate" + failed=1 + fi + done + for crate in "${REQUIRED[@]}"; do + if ! grep -qx "$crate" <<<"$deps"; then + echo "error: $desc is missing HTTP transport '$crate'; the forbidden-dependency check would pass vacuously. Wire a provider-neutral TLS transport." + failed=1 + fi + done + if [[ $failed -ne 0 ]]; then + exit 1 + fi + echo "ok: $desc is free of {${FORBIDDEN[*]}} and has an HTTP transport" +} + +check "lance-io (gcp, tls-no-provider)" --manifest-path rust/lance-io/Cargo.toml --no-default-features --features gcp,tls-no-provider +check "lance-io (azure, tls-no-provider)" --manifest-path rust/lance-io/Cargo.toml --no-default-features --features azure,tls-no-provider +check "lance-io (oss, tls-no-provider)" --manifest-path rust/lance-io/Cargo.toml --no-default-features --features oss,tls-no-provider