feat(lance-io): select TLS provider to keep aws-lc-rs out of non-AWS builds

.github/workflows/rust.yml

@@ -16,6 +16,7 @@ on:
       - Cargo.toml
       - Cargo.lock
       - deny.toml
+      - ci/check_forbidden_deps.sh
 
 permissions:
   contents: read
@@ -101,6 +102,11 @@ jobs:
         with:
           log-level: warn
           command: check
+      # cargo-deny resolves with all features, so it cannot assert a crate is
+      # absent from a specific feature combo. This checks each non-AWS backend.
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
+      - name: Check non-AWS builds stay free of aws-lc-rs/cmake
+        run: ci/check_forbidden_deps.sh
 
   linux-coverage-build:
     runs-on: "ubuntu-24.04-8x"

Cargo.toml

@@ -175,7 +175,15 @@ moka = { version = "0.12", features = ["future", "sync"] }
 ndarray = { version = "0.16.1", features = ["matrixmultiply-threading"] }
 num-traits = "0.2"
 object_store = { version = "0.13.2" }
-opendal = { version = "0.58.1" }
+# default-features = false drops opendal's aws-lc-rs TLS default; lance-io's tls-* features pick the transport.
+opendal = { version = "0.58.1", default-features = false, features = [
+    "auto-register-services",
+    "executors-tokio",
+    "layers-concurrent-limit",
+    "layers-logging",
+    "layers-retry",
+    "layers-timeout",
+] }
 object_store_opendal = { version = "0.58" }
 pin-project = "1.0"
 path_abs = "0.5"

ci/check_forbidden_deps.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Lance promises that non-AWS cloud builds can be built without aws-lc-rs.
+# Consumers building slim images for a single backend (e.g. GCS-only) rely on
+# this: aws-lc-rs compiles C/assembly through its cmake build dependency,
+# which bloats builds, complicates cross-compilation, and forces a second
+# crypto provider on deployments that standardize on ring. This script
+# enforces the promise by resolving each non-AWS backend under the
+# provider-neutral TLS feature (tls-no-provider) and failing if aws-lc-rs or
+# cmake appears.
+#
+# It checks the FUNCTIONAL configuration (backend + tls-no-provider), not the
+# bare backend: opendal's HTTP transport is opt-in, so a bare backend has no
+# TLS stack and would pass this check vacuously while failing at runtime. To
+# guard against that, each combination must also still resolve a reqwest HTTP
+# transport (opendal-http-transport-reqwest); a missing transport is an error.
+#
+# The usual way this regresses is a dependency bump changing a TLS default,
+# not a local change: opendal's default reqwest transport pins aws-lc-rs, and
+# its reqwest-rustls-* feature aliases re-enable it, so lance-io wires the
+# granular http-transport-reqwest-rustls-no-provider feature instead.
+#
+# If this check fails, fix the feature wiring rather than exempting the crate.
+# Feature combos that include the AWS SDK are not checked: it links aws-lc-rs
+# itself via aws-smithy-http-client.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+FORBIDDEN=(aws-lc-rs aws-lc-sys aws-lc-fips-sys cmake)
+REQUIRED=(opendal-http-transport-reqwest)
+
+check() {
+    local desc="$1"
+    shift
+
+    local deps
+    deps=$(cargo tree --locked -e normal,build --prefix none --format '{p}' "$@" | awk '{print $1}' | sort -u)
+
+    local failed=0
+    for crate in "${FORBIDDEN[@]}"; do
+        if grep -qx "$crate" <<<"$deps"; then
+            echo "error: forbidden dependency '$crate' found in $desc, pulled in via:"
+            cargo tree --locked -e normal,build "$@" -i "$crate"
+            failed=1
+        fi
+    done
+    for crate in "${REQUIRED[@]}"; do
+        if ! grep -qx "$crate" <<<"$deps"; then
+            echo "error: $desc is missing HTTP transport '$crate'; the forbidden-dependency check would pass vacuously. Wire a provider-neutral TLS transport."
+            failed=1
+        fi
+    done
+    if [[ $failed -ne 0 ]]; then
+        exit 1
+    fi
+    echo "ok: $desc is free of {${FORBIDDEN[*]}} and has an HTTP transport"
+}
+
+check "lance-io (gcp, tls-no-provider)" --manifest-path rust/lance-io/Cargo.toml --no-default-features --features gcp,tls-no-provider
+check "lance-io (azure, tls-no-provider)" --manifest-path rust/lance-io/Cargo.toml --no-default-features --features azure,tls-no-provider
+check "lance-io (oss, tls-no-provider)" --manifest-path rust/lance-io/Cargo.toml --no-default-features --features oss,tls-no-provider

rust/lance-io/Cargo.toml

@@ -62,7 +62,7 @@ name = "scheduler"
 harness = false
 
 [features]
-default = ["aws", "azure", "gcp"]
+default = ["aws", "azure", "gcp", "tls-aws-lc-rs"]
 metrics = ["dep:metrics"]
 gcs-test = []
 goosefs-test = []
@@ -74,6 +74,9 @@ goosefs = ["dep:opendal", "opendal/services-goosefs", "dep:object_store_opendal"
 tencent = ["dep:opendal", "opendal/services-cos", "dep:object_store_opendal"]
 huggingface = ["dep:opendal", "opendal/services-huggingface", "dep:object_store_opendal"]
 tos = ["dep:opendal", "opendal/services-tos", "dep:object_store_opendal"]
+# Granular opendal transports (not the reqwest-rustls-* aliases, which also pull aws-lc-rs); tls-no-provider needs an app-installed rustls provider.
+tls-aws-lc-rs = ["opendal?/http-transport-reqwest-rustls"]
+tls-no-provider = ["opendal?/http-transport-reqwest-rustls-no-provider"]
 tos-test = ["tos"]
 test-util = []
 

rust/lance-namespace-impls/Cargo.toml

@@ -12,7 +12,7 @@ categories.workspace = true
 rust-version.workspace = true
 
 [features]
-default = ["dir-aws", "dir-azure", "dir-gcp", "dir-oss", "dir-huggingface"]
+default = ["dir-aws", "dir-azure", "dir-gcp", "dir-oss", "dir-huggingface", "lance-io/tls-aws-lc-rs"]
 rest = ["dep:reqwest", "dep:serde"]
 rest-adapter = ["dep:axum", "dep:tower", "dep:tower-http", "dep:serde"]
 # Cloud storage features for directory implementation - align with lance-io

rust/lance/Cargo.toml

@@ -139,7 +139,7 @@ parquet = { version = "58", default-features = false, features = ["arrow", "asyn
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
 
 [features]
-default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo"]
+default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo", "tls-aws-lc-rs"]
 backtrace = ["lance-core/backtrace"]
 fp16kernels = ["lance-linalg/fp16kernels"]
 # Prevent dynamic linking of lzma, which comes from datafusion
@@ -162,6 +162,8 @@ tencent = ["lance-io/tencent"]
 goosefs = ["lance-io/goosefs"]
 tos = ["lance-io/tos"]
 huggingface = ["lance-io/huggingface"]
+tls-aws-lc-rs = ["lance-io/tls-aws-lc-rs"]
+tls-no-provider = ["lance-io/tls-no-provider"]
 # Publish object store metrics via the `metrics` crate.
 metrics = ["lance-io/metrics"]
 geo = ["lance-datafusion/geo", "lance-index/geo"]