From 360a33d6681cc5c2457489a65348c4b0f23fae7a Mon Sep 17 00:00:00 2001
From: highlyavailable <pwbryant1@gmail.com>
Date: Sun, 12 Jul 2026 06:53:38 -0600
Subject: [PATCH 1/4] esti: test lakectl fs upload of files over 5GiB

Covers the multipart path fixed in #9283 for both pre-signed and
regular uploads. Extends the output sanitizers to handle multipart
etags (md5-N) and relative physical addresses with part suffixes.

Closes #9284
---
 esti/golden/lakectl_fs_upload_large.golden |  7 +++++
 esti/lakectl_test.go                       | 33 ++++++++++++++++++++++
 esti/lakectl_util.go                       | 25 ++++++++++------
 3 files changed, 57 insertions(+), 8 deletions(-)
 create mode 100644 esti/golden/lakectl_fs_upload_large.golden

diff --git a/esti/golden/lakectl_fs_upload_large.golden b/esti/golden/lakectl_fs_upload_large.golden
new file mode 100644
index 00000000000..20fa42b193d
--- /dev/null
+++ b/esti/golden/lakectl_fs_upload_large.golden
@@ -0,0 +1,7 @@
+Path: ${FILE_PATH}
+Modified Time: <DATE> <TIME> <TZ>
+Size: 5368709121 bytes
+Human Size: 5.4 GB
+Physical Address: <OBJECT_KEY>
+Checksum: <CHECKSUM>
+Content-Type: application/octet-stream
diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go
index f47d02d8b8f..3bdb94eba0c 100644
--- a/esti/lakectl_test.go
+++ b/esti/lakectl_test.go
@@ -162,6 +162,39 @@ func TestLakectlPreSignUpload(t *testing.T) {
 	})
 }
 
+func TestLakectlFsUploadLarge(t *testing.T) {
+	RequireBlockstoreType(t, block.BlockstoreTypeS3)
+
+	repoName := GenerateUniqueRepositoryName()
+	storage := GenerateUniqueStorageNamespace(repoName)
+	vars := map[string]string{
+		"REPO":    repoName,
+		"STORAGE": storage,
+		"BRANCH":  mainBranch,
+	}
+	RunCmdAndVerifySuccessWithFile(t, Lakectl()+" repo create lakefs://"+repoName+" "+storage, false, "lakectl_repo_create", vars)
+
+	// S3 rejects single-part uploads above 5GiB, so one byte more forces
+	// the multipart path (#9284). A sparse file keeps disk usage low.
+	const largeFileSize = 5<<30 + 1
+	f, err := os.CreateTemp("", "lakectl-large-upload-")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, os.Remove(f.Name()))
+	})
+	require.NoError(t, f.Truncate(largeFileSize))
+	require.NoError(t, f.Close())
+
+	t.Run("pre-sign", func(t *testing.T) {
+		vars["FILE_PATH"] = "large_file_presign"
+		RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" --pre-sign", false, "lakectl_fs_upload_large", vars)
+	})
+	t.Run("no pre-sign", func(t *testing.T) {
+		vars["FILE_PATH"] = "large_file"
+		RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"], false, "lakectl_fs_upload_large", vars)
+	})
+}
+
 func TestLakectlCommit(t *testing.T) {
 	repoName := GenerateUniqueRepositoryName()
 	storage := GenerateUniqueStorageNamespace(repoName)
diff --git a/esti/lakectl_util.go b/esti/lakectl_util.go
index 93a275576c8..f3dcbfb31d1 100644
--- a/esti/lakectl_util.go
+++ b/esti/lakectl_util.go
@@ -28,13 +28,17 @@ var (
 )
 
 var (
-	reTimestamp       = regexp.MustCompile(`timestamp: \d+`)
-	reTime            = regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} \w{1,4}`)
-	reCommitID        = regexp.MustCompile(`[\d|a-f]{64}`)
-	reShortCommitID   = regexp.MustCompile(`[\d|a-f]{16}`)
-	reChecksum        = regexp.MustCompile(`([\d|a-f]{32})|(0x[0-9A-F]{15})`)
-	reEndpoint        = regexp.MustCompile(`https?://\w+(:\d+)?/api/v\d+/`)
-	rePhysicalAddress = regexp.MustCompile(`/data/[0-9a-v]{20}/(?:[0-9a-v]{20}(?:,.+)?)?`)
+	reTimestamp     = regexp.MustCompile(`timestamp: \d+`)
+	reTime          = regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} \w{1,4}`)
+	reCommitID      = regexp.MustCompile(`[\d|a-f]{64}`)
+	reShortCommitID = regexp.MustCompile(`[\d|a-f]{16}`)
+	// multipart uploads report an etag with a part count suffix, e.g. <md5>-1025
+	reChecksum = regexp.MustCompile(`([\d|a-f]{32}(-\d+)?)|(0x[0-9A-F]{15})`)
+	reEndpoint = regexp.MustCompile(`https?://\w+(:\d+)?/api/v\d+/`)
+	// physical addresses may be printed relative to the storage namespace (no
+	// leading slash), and multipart uploads append a part-count suffix, e.g.
+	// data/<prefix>/<key>,<suffix>
+	rePhysicalAddress = regexp.MustCompile(`/?data/[0-9a-v]{20}/(?:[0-9a-v]{20}(?:,.+)?)?`)
 	reVariable        = regexp.MustCompile(`\$\{([^${}]+)}`)
 	rePreSignURL      = regexp.MustCompile(`https?://\S+\?\S+`)
 	reSecretAccessKey = regexp.MustCompile(`secret_access_key: \S{16,128}`)
@@ -260,7 +264,12 @@ func normalizeProgramTimestamp(output string) string {
 func normalizeRandomObjectKey(output string, objectPrefix string) string {
 	objectPrefix = strings.TrimPrefix(objectPrefix, "/")
 	for _, match := range rePhysicalAddress.FindAllString(output, -1) {
-		output = strings.Replace(output, objectPrefix+match, objectPrefix+"/<OBJECT_KEY>", 1)
+		if strings.Contains(output, objectPrefix+match) {
+			output = strings.Replace(output, objectPrefix+match, objectPrefix+"/<OBJECT_KEY>", 1)
+		} else {
+			// relative physical address, not prefixed by the storage namespace
+			output = strings.Replace(output, match, "<OBJECT_KEY>", 1)
+		}
 	}
 	return output
 }

From 92bda4eecc41414435650c8a9bf4f0f954aa9a1d Mon Sep 17 00:00:00 2001
From: highlyavailable <pwbryant1@gmail.com>
Date: Sat, 31 Oct 2026 04:16:30 +0000
Subject: [PATCH 2/4] esti: check size and clean storage in large upload test

Drop the golden file and the sanitizer changes it needed. The test now
checks the uploaded size through lakectl output and the stat API, and
deletes each large object from the backing store once the subtest ends,
since deleting the repository leaves its data in place.
---
 esti/adapter_utils.go                      | 16 +++++++++++++
 esti/golden/lakectl_fs_upload_large.golden |  7 ------
 esti/lakectl_test.go                       | 28 ++++++++++++++++++----
 esti/lakectl_util.go                       | 25 +++++++------------
 4 files changed, 48 insertions(+), 28 deletions(-)
 delete mode 100644 esti/golden/lakectl_fs_upload_large.golden

diff --git a/esti/adapter_utils.go b/esti/adapter_utils.go
index bcfaa4d84ed..1f150ef8646 100644
--- a/esti/adapter_utils.go
+++ b/esti/adapter_utils.go
@@ -35,6 +35,9 @@ type ClientAdapter interface {
 	// Download returns a Reader to download data from physicalAddress.  The Close method
 	// of that Reader can return errors!
 	Download(ctx context.Context, physicalAddress *url.URL) (io.ReadCloser, error)
+
+	// Delete removes the object at physicalAddress from the backing store.
+	Delete(ctx context.Context, physicalAddress *url.URL) error
 }
 
 type AdapterFactory map[string]func(context.Context) (ClientAdapter, error)
@@ -110,3 +113,16 @@ func (s *s3Adapter) Download(ctx context.Context, physicalAddress *url.URL) (io.
 	}
 	return getObjectResponse.Body, nil
 }
+
+func (s *s3Adapter) Delete(ctx context.Context, physicalAddress *url.URL) error {
+	if physicalAddress.Scheme != s3Scheme {
+		return fmt.Errorf("%s: %w", s3Scheme, helpers.ErrUnsupportedProtocol)
+	}
+	bucket := physicalAddress.Hostname()
+	key := strings.TrimPrefix(physicalAddress.Path, "/")
+	_, err := s.svc.DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	return err
+}
diff --git a/esti/golden/lakectl_fs_upload_large.golden b/esti/golden/lakectl_fs_upload_large.golden
deleted file mode 100644
index 20fa42b193d..00000000000
--- a/esti/golden/lakectl_fs_upload_large.golden
+++ /dev/null
@@ -1,7 +0,0 @@
-Path: ${FILE_PATH}
-Modified Time: <DATE> <TIME> <TZ>
-Size: 5368709121 bytes
-Human Size: 5.4 GB
-Physical Address: <OBJECT_KEY>
-Checksum: <CHECKSUM>
-Content-Type: application/octet-stream
diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go
index 3bdb94eba0c..e2253bbdc30 100644
--- a/esti/lakectl_test.go
+++ b/esti/lakectl_test.go
@@ -2,11 +2,13 @@ package esti
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/go-openapi/swag"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 	"github.com/treeverse/lakefs/pkg/api/apigen"
@@ -164,6 +166,7 @@ func TestLakectlPreSignUpload(t *testing.T) {
 
 func TestLakectlFsUploadLarge(t *testing.T) {
 	RequireBlockstoreType(t, block.BlockstoreTypeS3)
+	ctx := t.Context()
 
 	repoName := GenerateUniqueRepositoryName()
 	storage := GenerateUniqueStorageNamespace(repoName)
@@ -185,13 +188,30 @@ func TestLakectlFsUploadLarge(t *testing.T) {
 	require.NoError(t, f.Truncate(largeFileSize))
 	require.NoError(t, f.Close())
 
+	uploadAndVerify := func(t *testing.T, objPath, extraArgs string) {
+		t.Helper()
+		RunCmdAndVerifyContainsText(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+objPath+extraArgs, false, fmt.Sprintf("Size: %d bytes", largeFileSize), vars)
+
+		statResp, err := client.StatObjectWithResponse(ctx, repoName, mainBranch, &apigen.StatObjectParams{Path: objPath})
+		require.NoError(t, err)
+		require.NotNil(t, statResp.JSON200, "stat %s: %s", objPath, statResp.Status())
+		require.Equal(t, int64(largeFileSize), swag.Int64Value(statResp.JSON200.SizeBytes))
+
+		// deleting the repository does not remove its data, so remove the large object from the storage
+		physicalAddress, err := url.Parse(statResp.JSON200.PhysicalAddress)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			adapter, err := NewAdapter(ctx, physicalAddress.Scheme)
+			require.NoError(t, err)
+			require.NoError(t, adapter.Delete(ctx, physicalAddress))
+		})
+	}
+
 	t.Run("pre-sign", func(t *testing.T) {
-		vars["FILE_PATH"] = "large_file_presign"
-		RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"]+" --pre-sign", false, "lakectl_fs_upload_large", vars)
+		uploadAndVerify(t, "large_file_presign", " --pre-sign")
 	})
 	t.Run("no pre-sign", func(t *testing.T) {
-		vars["FILE_PATH"] = "large_file"
-		RunCmdAndVerifySuccessWithFile(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+vars["FILE_PATH"], false, "lakectl_fs_upload_large", vars)
+		uploadAndVerify(t, "large_file", "")
 	})
 }
 
diff --git a/esti/lakectl_util.go b/esti/lakectl_util.go
index f3dcbfb31d1..93a275576c8 100644
--- a/esti/lakectl_util.go
+++ b/esti/lakectl_util.go
@@ -28,17 +28,13 @@ var (
 )
 
 var (
-	reTimestamp     = regexp.MustCompile(`timestamp: \d+`)
-	reTime          = regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} \w{1,4}`)
-	reCommitID      = regexp.MustCompile(`[\d|a-f]{64}`)
-	reShortCommitID = regexp.MustCompile(`[\d|a-f]{16}`)
-	// multipart uploads report an etag with a part count suffix, e.g. <md5>-1025
-	reChecksum = regexp.MustCompile(`([\d|a-f]{32}(-\d+)?)|(0x[0-9A-F]{15})`)
-	reEndpoint = regexp.MustCompile(`https?://\w+(:\d+)?/api/v\d+/`)
-	// physical addresses may be printed relative to the storage namespace (no
-	// leading slash), and multipart uploads append a part-count suffix, e.g.
-	// data/<prefix>/<key>,<suffix>
-	rePhysicalAddress = regexp.MustCompile(`/?data/[0-9a-v]{20}/(?:[0-9a-v]{20}(?:,.+)?)?`)
+	reTimestamp       = regexp.MustCompile(`timestamp: \d+`)
+	reTime            = regexp.MustCompile(`\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} \w{1,4}`)
+	reCommitID        = regexp.MustCompile(`[\d|a-f]{64}`)
+	reShortCommitID   = regexp.MustCompile(`[\d|a-f]{16}`)
+	reChecksum        = regexp.MustCompile(`([\d|a-f]{32})|(0x[0-9A-F]{15})`)
+	reEndpoint        = regexp.MustCompile(`https?://\w+(:\d+)?/api/v\d+/`)
+	rePhysicalAddress = regexp.MustCompile(`/data/[0-9a-v]{20}/(?:[0-9a-v]{20}(?:,.+)?)?`)
 	reVariable        = regexp.MustCompile(`\$\{([^${}]+)}`)
 	rePreSignURL      = regexp.MustCompile(`https?://\S+\?\S+`)
 	reSecretAccessKey = regexp.MustCompile(`secret_access_key: \S{16,128}`)
@@ -264,12 +260,7 @@ func normalizeProgramTimestamp(output string) string {
 func normalizeRandomObjectKey(output string, objectPrefix string) string {
 	objectPrefix = strings.TrimPrefix(objectPrefix, "/")
 	for _, match := range rePhysicalAddress.FindAllString(output, -1) {
-		if strings.Contains(output, objectPrefix+match) {
-			output = strings.Replace(output, objectPrefix+match, objectPrefix+"/<OBJECT_KEY>", 1)
-		} else {
-			// relative physical address, not prefixed by the storage namespace
-			output = strings.Replace(output, match, "<OBJECT_KEY>", 1)
-		}
+		output = strings.Replace(output, objectPrefix+match, objectPrefix+"/<OBJECT_KEY>", 1)
 	}
 	return output
 }

From 8ee46097e251d3011a043c6645d72ab5a2a90c9a Mon Sep 17 00:00:00 2001
From: highlyavailable <pwbryant1@gmail.com>
Date: Mon, 2 Nov 2026 15:56:21 +0000
Subject: [PATCH 3/4] esti: register large upload cleanup before the upload

Look up the object's physical address in the cleanup itself, so the
large object is removed from storage even if the upload or a later
check fails.
---
 esti/lakectl_test.go | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/esti/lakectl_test.go b/esti/lakectl_test.go
index e2253bbdc30..cce15b9e028 100644
--- a/esti/lakectl_test.go
+++ b/esti/lakectl_test.go
@@ -190,21 +190,26 @@ func TestLakectlFsUploadLarge(t *testing.T) {
 
 	uploadAndVerify := func(t *testing.T, objPath, extraArgs string) {
 		t.Helper()
-		RunCmdAndVerifyContainsText(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+objPath+extraArgs, false, fmt.Sprintf("Size: %d bytes", largeFileSize), vars)
-
-		statResp, err := client.StatObjectWithResponse(ctx, repoName, mainBranch, &apigen.StatObjectParams{Path: objPath})
-		require.NoError(t, err)
-		require.NotNil(t, statResp.JSON200, "stat %s: %s", objPath, statResp.Status())
-		require.Equal(t, int64(largeFileSize), swag.Int64Value(statResp.JSON200.SizeBytes))
-
 		// deleting the repository does not remove its data, so remove the large object from the storage
-		physicalAddress, err := url.Parse(statResp.JSON200.PhysicalAddress)
-		require.NoError(t, err)
 		t.Cleanup(func() {
+			statResp, err := client.StatObjectWithResponse(ctx, repoName, mainBranch, &apigen.StatObjectParams{Path: objPath})
+			require.NoError(t, err)
+			if statResp.JSON200 == nil {
+				return
+			}
+			physicalAddress, err := url.Parse(statResp.JSON200.PhysicalAddress)
+			require.NoError(t, err)
 			adapter, err := NewAdapter(ctx, physicalAddress.Scheme)
 			require.NoError(t, err)
 			require.NoError(t, adapter.Delete(ctx, physicalAddress))
 		})
+
+		RunCmdAndVerifyContainsText(t, Lakectl()+" fs upload -s "+f.Name()+" lakefs://"+repoName+"/"+mainBranch+"/"+objPath+extraArgs, false, fmt.Sprintf("Size: %d bytes", largeFileSize), vars)
+
+		statResp, err := client.StatObjectWithResponse(ctx, repoName, mainBranch, &apigen.StatObjectParams{Path: objPath})
+		require.NoError(t, err)
+		require.NotNil(t, statResp.JSON200, "stat %s: %s", objPath, statResp.Status())
+		require.Equal(t, int64(largeFileSize), swag.Int64Value(statResp.JSON200.SizeBytes))
 	}
 
 	t.Run("pre-sign", func(t *testing.T) {

From 221e697108d0d646ccc270c22b9bd7d81083f8dd Mon Sep 17 00:00:00 2001
From: highlyavailable <pwbryant1@gmail.com>
Date: Tue, 3 Nov 2026 15:41:01 +0000
Subject: [PATCH 4/4] ci: set AWS_REGION for lakectl compatibility tests

The large upload test deletes its objects straight from S3, so the
default AWS config needs a region.
---
 .github/workflows/lakectl-compatibility-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/lakectl-compatibility-tests.yaml b/.github/workflows/lakectl-compatibility-tests.yaml
index 3db33363a61..bfe19ffd2a5 100644
--- a/.github/workflows/lakectl-compatibility-tests.yaml
+++ b/.github/workflows/lakectl-compatibility-tests.yaml
@@ -125,6 +125,7 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-east-1
           ESTI_AWS_ACCESS_KEY_ID: ${{ secrets.ESTI_AWS_ACCESS_KEY_ID }}
           ESTI_AWS_SECRET_ACCESS_KEY: ${{ secrets.ESTI_AWS_SECRET_ACCESS_KEY }}
           ESTI_BINARIES_DIR: ${{ github.workspace }}/bin