From d5f83fb78b8e6bfd19655a83c963ddcd30e46ae2 Mon Sep 17 00:00:00 2001 From: Benjamin Ingberg Date: Fri, 29 May 2026 09:49:44 +0200 Subject: [PATCH] Add support for SplitBlob and SpliceBlob methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds support for the SplitBlob and SpliceBlob methods from the Remote Execution v2 (REv2) api. SplitBlob and SpliceBlob can be used to facilitate uploads and downloads of large files but a naïve implementation like this has some major drawbacks as well. The blobs must exist in both their chunked and non chunked form, which may significantly increase storage requirements for large blobs. The protocol gives no guarantee that a large blob stored in the CAS exists in its chunked form which forces you to perform a fairly heavy Split call that loads the entire large blob in order to decomposition it into its chunks. This implementation mostly exists as a stepping stone for a different implementation where Buildbarn internally manages all blobs as chunked blobs. --- MODULE.bazel | 1 + cmd/bb_storage/main.go | 31 +- go.mod | 1 + go.sum | 2 + pkg/blobstore/BUILD.bazel | 1 + pkg/blobstore/chunklistvalidating/BUILD.bazel | 43 ++ .../chunk_concatenating_reader.go | 68 +++ .../chunk_list_validating_blob_access.go | 367 +++++++++++++++ .../chunk_list_validating_blob_access_test.go | 410 ++++++++++++++++ pkg/blobstore/chunklistvalidating/chunker.go | 16 + .../fake_blob_access_test.go | 88 ++++ .../integration/BUILD.bazel | 19 + .../chunk_list_validating_integration_test.go | 445 ++++++++++++++++++ .../chunklistvalidating/reader_chunker.go | 53 +++ .../reader_chunker_test.go | 107 +++++ pkg/blobstore/cls_read_buffer_factory.go | 27 ++ pkg/blobstore/configuration/BUILD.bazel | 2 + .../configuration/blob_access_creator.go | 2 +- .../configuration/cas_blob_access_creator.go | 4 +- .../configuration/cls_blob_access_creator.go | 86 ++++ .../configuration/new_blob_access.go | 19 +- .../proto_blob_access_creator.go | 3 +- pkg/blobstore/grpcclients/BUILD.bazel | 1 + pkg/blobstore/grpcclients/cls_blob_access.go | 101 ++++ .../content_addressable_storage_server.go | 93 +++- ...content_addressable_storage_server_test.go | 134 +++++- .../configuration/bb_storage/bb_storage.pb.go | 54 ++- .../configuration/bb_storage/bb_storage.proto | 4 + pkg/proto/configuration/blobstore/BUILD.bazel | 2 + .../configuration/blobstore/blobstore.pb.go | 273 +++++++---- .../configuration/blobstore/blobstore.proto | 41 ++ 31 files changed, 2363 insertions(+), 135 deletions(-) create mode 100644 pkg/blobstore/chunklistvalidating/BUILD.bazel create mode 100644 pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go create mode 100644 pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go create mode 100644 pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go create mode 100644 pkg/blobstore/chunklistvalidating/chunker.go create mode 100644 pkg/blobstore/chunklistvalidating/fake_blob_access_test.go create mode 100644 pkg/blobstore/chunklistvalidating/integration/BUILD.bazel create mode 100644 pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go create mode 100644 pkg/blobstore/chunklistvalidating/reader_chunker.go create mode 100644 pkg/blobstore/chunklistvalidating/reader_chunker_test.go create mode 100644 pkg/blobstore/cls_read_buffer_factory.go create mode 100644 pkg/blobstore/configuration/cls_blob_access_creator.go create mode 100644 pkg/blobstore/grpcclients/cls_blob_access.go diff --git a/MODULE.bazel b/MODULE.bazel index 3dc48bb5d..8e1d6cfc5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,6 +49,7 @@ use_repo( "com_github_aws_aws_sdk_go_v2_service_sts", "com_github_bazelbuild_buildtools", "com_github_bazelbuild_remote_apis", + "com_github_buildbarn_go_cdc", "com_github_buildbarn_go_sha256tree", "com_github_fxtlabs_primes", "com_github_go_jose_go_jose_v3", diff --git a/cmd/bb_storage/main.go b/cmd/bb_storage/main.go index 192fcb5e4..87e62e3d5 100644 --- a/cmd/bb_storage/main.go +++ b/cmd/bb_storage/main.go @@ -84,6 +84,27 @@ func main() { contentAddressableStorage = authorizedBackend } + // Chunk List Storage (CLS). + var chunkListStorage blobstore.BlobAccess + if configuration.ChunkListStorage != nil { + info, authorizedBackend, allAuthorizers, err := newScannableBlobAccess( + dependenciesGroup, + configuration.ChunkListStorage, + blobstore_configuration.NewCLSBlobAccessCreator( + contentAddressableStorageInfo, + grpcClientFactory, + int(configuration.MaximumMessageSizeBytes), + ), + grpcClientFactory, + ) + if err != nil { + return util.StatusWrap(err, "Failed to create Chunk List Storage") + } + cacheCapabilitiesProviders = append(cacheCapabilitiesProviders, info.BlobAccess) + cacheCapabilitiesAuthorizers = append(cacheCapabilitiesAuthorizers, allAuthorizers...) + chunkListStorage = authorizedBackend + } + // Action Cache (AC). var actionCache blobstore.BlobAccess if configuration.ActionCache != nil { @@ -193,12 +214,14 @@ func main() { configuration.GrpcServers, func(s grpc.ServiceRegistrar) { if contentAddressableStorage != nil { + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer( + contentAddressableStorage, + chunkListStorage, + configuration.MaximumMessageSizeBytes, + ) remoteexecution.RegisterContentAddressableStorageServer( s, - grpcservers.NewContentAddressableStorageServer( - contentAddressableStorage, - configuration.MaximumMessageSizeBytes, - ), + contentAddressableStorageServer, ) bytestream.RegisterByteStreamServer( s, diff --git a/go.mod b/go.mod index 082763b1a..4fd9fa898 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/bazelbuild/buildtools v0.0.0-20260527135131-3b47c424ecf5 github.com/bazelbuild/remote-apis v0.0.0-20260331222004-becdd8f9ff81 github.com/bazelbuild/rules_go v0.62.0 + github.com/buildbarn/go-cdc v0.0.9 github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449 github.com/go-jose/go-jose/v3 v3.0.5 diff --git a/go.sum b/go.sum index d097ebee7..545e90b57 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/bazelbuild/rules_go v0.62.0/go.mod h1:6YghDRf6l3FSiAwncK+Ww9jE1naoQzN github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/buildbarn/go-cdc v0.0.9 h1:bWfgn92ed8Oo2zZKJdMAfB0APGz7Q8zvnqUn3hPuihM= +github.com/buildbarn/go-cdc v0.0.9/go.mod h1:KUMqSMvoRlby3uak9aKIvgz3KgNqwm2CMUoVX1EDr8k= github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b h1:IKUxixGBm9UxobU7c248z0BF0ojG19uoSLz8MFZM/KA= github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b/go.mod h1:e7g3/yWApcg+PpDqd4eQEEV8pexQmfCgK3frP+1Wuvk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/pkg/blobstore/BUILD.bazel b/pkg/blobstore/BUILD.bazel index 45d054538..80a4a60d5 100644 --- a/pkg/blobstore/BUILD.bazel +++ b/pkg/blobstore/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "authorizing_blob_access.go", "blob_access.go", "cas_read_buffer_factory.go", + "cls_read_buffer_factory.go", "deadline_enforcing_blob_access.go", "demultiplexing_blob_access.go", "empty_blob_injecting_blob_access.go", diff --git a/pkg/blobstore/chunklistvalidating/BUILD.bazel b/pkg/blobstore/chunklistvalidating/BUILD.bazel new file mode 100644 index 000000000..1bb49c7f0 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "chunklistvalidating", + srcs = [ + "chunk_concatenating_reader.go", + "chunk_list_validating_blob_access.go", + "chunker.go", + "reader_chunker.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/digest", + "//pkg/util", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_buildbarn_go_cdc//:go-cdc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_golang_x_sync//errgroup", + ], +) + +go_test( + name = "chunklistvalidating_test", + srcs = [ + "chunk_list_validating_blob_access_test.go", + "fake_blob_access_test.go", + "reader_chunker_test.go", + ], + deps = [ + ":chunklistvalidating", + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/digest", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) diff --git a/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go b/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go new file mode 100644 index 000000000..643af130e --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go @@ -0,0 +1,68 @@ +package chunklistvalidating + +import ( + "context" + "io" + + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// chunkConcatenatingReader is a helper utility that implements the +// io.ReadCloser api over a series of digest.Digest objectes fetched +// sequentially from the CAS. +type chunkConcatenatingReader struct { + ctx context.Context + contentAddressableStorage blobstore.BlobAccess + chunkDigests []digest.Digest + currentIndex int + currentReader io.ReadCloser + closed bool +} + +func (r *chunkConcatenatingReader) Read(p []byte) (int, error) { + if r.closed { + return 0, status.Error(codes.Internal, "Reader is already closed") + } + for { + if r.currentReader == nil { + if r.currentIndex >= len(r.chunkDigests) { + return 0, io.EOF + } + chunkDigest := r.chunkDigests[r.currentIndex] + b := r.contentAddressableStorage.Get(r.ctx, chunkDigest) + r.currentReader = b.ToReader() + r.currentIndex++ + } + + n, err := r.currentReader.Read(p) + if n > 0 { + return n, nil + } + if err == io.EOF { + err = r.currentReader.Close() + r.currentReader = nil + if err != nil { + return 0, err + } + continue + } + if err != nil { + _ = r.currentReader.Close() + r.currentReader = nil + return 0, util.StatusWrap(err, "Failed to read chunk") + } + } +} + +func (r *chunkConcatenatingReader) Close() (err error) { + r.closed = true + if r.currentReader != nil { + err = r.currentReader.Close() + r.currentReader = nil + } + return err +} diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go new file mode 100644 index 000000000..441ef181d --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go @@ -0,0 +1,367 @@ +package chunklistvalidating + +import ( + "context" + "io" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type chunkListValidatingBlobAccess struct { + blobstore.BlobAccess + contentAddressableStorage blobstore.BlobAccess + maximumMessageSizeBytes int +} + +// NewChunkListValidatingBlobAccess creates a wrapper around a Chunk +// List Storage (CLS) that ensures only valid chunk lists are stored in +// the CLS. A valid chunk list is a chunk list which follows the +// chunking parameters, has all the chunks present in the Content +// Addressable Storage (CAS) and where the chunks concatenate into the +// appropriate digest. +// +// This validation is fairly expensive and validation should only be +// done at a single layer as close as possible to the CAS where the full +// view of the CAS is available. +func NewChunkListValidatingBlobAccess(chunkListStorage, contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int) blobstore.BlobAccess { + return &chunkListValidatingBlobAccess{ + BlobAccess: chunkListStorage, + contentAddressableStorage: contentAddressableStorage, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +// Fetch the chunking parameters from the GetCapabilities +// implementation. +func (ba *chunkListValidatingBlobAccess) getValidChunkingParameters(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.RepMaxCdcParams, error) { + capabilities, err := ba.BlobAccess.GetCapabilities(ctx, instanceName) + if err != nil { + return nil, util.StatusWrap(err, "Unable to GetCapabilities to determine chunking parameters") + } + + params := capabilities.CacheCapabilities.GetRepMaxCdcParams() + if params == nil { + return nil, status.Error(codes.Unimplemented, "This backend only supports upstream servers with RepMaxCDC support") + } + if params.MinChunkSizeBytes < 64 { + return nil, status.Errorf(codes.Internal, "RepMaxCDC minimum chunk size was %d bytes but a minimum of 64 bytes is required", params.MinChunkSizeBytes) + } + + return params, nil +} + +// Check the downstream blob access if this particular blob has already +// been split. If that's the case and all the chunks are still there we +// can return early. In case of errors we will return nil and continue +// with the regular code path. +func (ba *chunkListValidatingBlobAccess) checkSplitResult(ctx context.Context, d digest.Digest) buffer.Buffer { + b1, b2 := ba.BlobAccess.Get(ctx, d).CloneCopy(ba.maximumMessageSizeBytes) + responseMsg, err := b1.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + b2.Discard() + return nil + } + + splitBlobResponse := responseMsg.(*remoteexecution.SplitBlobResponse) + digestFunction := d.GetDigestFunction() + digestSetBuilder := digest.NewSetBuilder(len(splitBlobResponse.ChunkDigests)) + digestSetBuilder.Add(d) + + for _, chunkDigestProto := range splitBlobResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + b2.Discard() + return nil + } + digestSetBuilder.Add(chunkDigest) + } + + missing, err := ba.contentAddressableStorage.FindMissing(ctx, digestSetBuilder.Build()) + if err == nil && missing.Empty() { + return b2 + } + b2.Discard() + return nil +} + +// Get returns a valid SplitResult for the given digest chunking the +// blob and storing the chunk list if needed. +func (ba *chunkListValidatingBlobAccess) Get(ctx context.Context, d digest.Digest) buffer.Buffer { + params, err := ba.getValidChunkingParameters(ctx, d.GetInstanceName()) + if err != nil { + return buffer.NewBufferFromError(err) + } + + // Check for the trivial case, the blob is small enough that it will + // always decompose to a single chunk of the same size as the + // original blob. We verify the existence of the blob in CAS and + // break out early. + blobSize := d.GetSizeBytes() + if uint64(blobSize) < 2*params.MinChunkSizeBytes { + missing, err := ba.contentAddressableStorage.FindMissing(ctx, d.ToSingletonSet()) + if err != nil { + return buffer.NewBufferFromError(util.StatusWrap(err, "Failed to verify blob existence")) + } + if !missing.Empty() { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found in CAS")) + } + + response := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{d.GetProto()}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + return buffer.NewProtoBufferFromProto(response, buffer.UserProvided) + } + + // Check if we have already computed the result for this blob. + if result := ba.checkSplitResult(ctx, d); result != nil { + return result + } + + // Fallthrough case, compute the chunk list, upload the chunks and + // store the chunk list. + blobReader := ba.contentAddressableStorage.Get(ctx, d).ToReader() + defer blobReader.Close() + chunker := NewReaderChunker(d.GetDigestFunction(), blobReader, int64(params.MinChunkSizeBytes), int64(params.HorizonSizeBytes)) + + chunkDigests := make([]*remoteexecution.Digest, 0, uint64(blobSize)/params.MinChunkSizeBytes+1) + + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + if err != nil { + return buffer.NewBufferFromError(err) + } + + missing, err := ba.contentAddressableStorage.FindMissing(ctx, chunk.Digest.ToSingletonSet()) + if err != nil { + return buffer.NewBufferFromError(err) + } + if !missing.Empty() { + if err := ba.contentAddressableStorage.Put(ctx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { + return buffer.NewBufferFromError(err) + } + } + + chunkDigests = append(chunkDigests, chunk.Digest.GetProto()) + } + + response := &remoteexecution.SplitBlobResponse{ + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + b1, b2 := buffer.NewProtoBufferFromProto(response, buffer.UserProvided).CloneCopy(ba.maximumMessageSizeBytes) + + if err := ba.BlobAccess.Put(ctx, d, b1); err != nil { + b2.Discard() + return buffer.NewBufferFromError(util.StatusWrap(err, "Failed to store the split blob response")) + } + + return b2 +} + +func (ba *chunkListValidatingBlobAccess) matchesStoredChunkList(ctx context.Context, d digest.Digest, userResponse *remoteexecution.SplitBlobResponse) bool { + existingMsg, err := ba.BlobAccess.Get(ctx, d).ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return false + } + + cachedResponse := existingMsg.(*remoteexecution.SplitBlobResponse) + if len(cachedResponse.ChunkDigests) == len(userResponse.ChunkDigests) { + for i, c := range cachedResponse.ChunkDigests { + u := userResponse.ChunkDigests[i] + if u.Hash != c.Hash || u.SizeBytes != c.SizeBytes { + return false + } + } + } + + return true +} + +func (ba *chunkListValidatingBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { + // Parse the buffer as a SplitBlobResponse + msg, err := b.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return util.StatusWrap(err, "Failed to parse input as SplitBlobResponse") + } + userResponse := msg.(*remoteexecution.SplitBlobResponse) + + digestFunction := d.GetDigestFunction() + var userChunks []digest.Digest + digestSetBuilder := digest.NewSetBuilder(len(userResponse.ChunkDigests)) + for _, chunkDigestProto := range userResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + return status.Errorf(codes.InvalidArgument, "Invalid chunk digest: %v", err) + } + digestSetBuilder.Add(chunkDigest) + userChunks = append(userChunks, chunkDigest) + } + + // Check that all referenced chunks are present in storage. + missing, err := ba.contentAddressableStorage.FindMissing(ctx, digestSetBuilder.Build()) + if err != nil { + return util.StatusWrap(err, "Failed to check existence of chunks") + } + if !missing.Empty() { + return status.Error(codes.NotFound, "At least one chunk in the chunk list was not found") + } + + // Check the trivial cases without hitting the downstream blob + // stores. + + // No chunks given, blob must be the empty blob. + if len(userChunks) == 0 { + if d.GetSizeBytes() != 0 { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + if d.GetDigestFunction().NewGenerator(0).Sum() != d { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + return nil + } + // Single chunk given, the blob must be equal to the chunk. At this + // point we have already verified the presence of the chunk so we do + // not have to verify the presence of the blob. + if len(userChunks) == 1 { + if d != userChunks[0] { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + return nil + } + + chunksMatchesStoredLists := ba.matchesStoredChunkList(ctx, d, userResponse) + missing, err = ba.contentAddressableStorage.FindMissing(ctx, d.ToSingletonSet()) + if err != nil { + return util.StatusWrap(err, "Failed to check existence of blob") + } + blobExistsInCAS := missing.Empty() + + // The request is identical to an already existing chunk list with + // content we have verified exists in CAS. + if blobExistsInCAS && chunksMatchesStoredLists { + return nil + } + + // No more shortcuts available go through the heavy path of + // concatenating/verifying and chunking the blobs. + params, err := ba.getValidChunkingParameters(ctx, d.GetInstanceName()) + if err != nil { + return err + } + + reader := &chunkConcatenatingReader{ + ctx: ctx, + contentAddressableStorage: ba.contentAddressableStorage, + chunkDigests: userChunks, + } + + blobBuffer := buffer.NewCASBufferFromReader(d, reader, buffer.UserProvided) + b1, b2 := blobBuffer.CloneStream() + + // Stream 1: Uploads the blob to CAS. + group, gCtx := errgroup.WithContext(ctx) + group.Go(func() error { + if blobExistsInCAS { + // Upload unnecessary, blob already exists in CAS. + b1.Discard() + return nil + } + return ba.contentAddressableStorage.Put(gCtx, d, b1) + }) + + // Stream 2: Chunk the stream to compute the digest and cache the + // canonical chunks. + var canonicalChunkDigests []*remoteexecution.Digest + group.Go(func() error { + b2Reader := b2.ToReader() + defer b2Reader.Close() + chunker := NewReaderChunker(d.GetDigestFunction(), b2Reader, int64(params.MinChunkSizeBytes), int64(params.HorizonSizeBytes)) + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + missing, err := ba.contentAddressableStorage.FindMissing(gCtx, chunk.Digest.ToSingletonSet()) + if err != nil { + return err + } + if !missing.Empty() { + if err := ba.contentAddressableStorage.Put(gCtx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { + return util.StatusWrap(err, "Failed to save chunk") + } + } + canonicalChunkDigests = append(canonicalChunkDigests, chunk.Digest.GetProto()) + } + }) + + // Wait for the full blob validation and upload to complete. + if err := group.Wait(); err != nil { + return util.StatusWrap(err, "Failed to splice the blob") + } + + // Store the canonical response. + canonicalResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: canonicalChunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + canonicalBuffer := buffer.NewProtoBufferFromProto(canonicalResponse, buffer.UserProvided) + if err := ba.BlobAccess.Put(ctx, d, canonicalBuffer); err != nil { + return util.StatusWrap(err, "Failed to save canonical chunk list") + } + return nil +} + +func (ba *chunkListValidatingBlobAccess) findMissingChunks(ctx context.Context, d digest.Digest) (digest.Set, error) { + splitBlobResponseProto, err := ba.BlobAccess.Get(ctx, d).ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return digest.EmptySet, err + } + splitBlobResponse := splitBlobResponseProto.(*remoteexecution.SplitBlobResponse) + digestFunction := d.GetDigestFunction() + builder := digest.NewSetBuilder(len(splitBlobResponse.ChunkDigests)) + for _, chunkDigestProto := range splitBlobResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + return digest.EmptySet, util.StatusWrapf(err, "Invalid chunk digest %#v", chunkDigestProto) + } + builder.Add(chunkDigest) + } + return ba.contentAddressableStorage.FindMissing(ctx, builder.Build()) +} + +func (ba *chunkListValidatingBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + missingBlobs, err := ba.BlobAccess.FindMissing(ctx, digests) + if err != nil { + return digest.EmptySet, err + } + nonMissingBlobs, _, _ := digest.GetDifferenceAndIntersection(digests, missingBlobs) + missings := make([]digest.Set, 1, 1+nonMissingBlobs.Length()) + missings[0] = missingBlobs + for _, d := range nonMissingBlobs.Items() { + missingChunks, err := ba.findMissingChunks(ctx, d) + if err != nil { + return digest.EmptySet, err + } + if !missingChunks.Empty() { + missings = append(missings, d.ToSingletonSet()) + } + } + return digest.GetUnion(missings), nil +} diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go new file mode 100644 index 000000000..3830e720b --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go @@ -0,0 +1,410 @@ +package chunklistvalidating_test + +import ( + "bytes" + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// mustComputeDigest is a test helper to easily generate digests from +// byte slices. +func mustComputeDigest(t *testing.T, digestFunction digest.Function, data []byte) digest.Digest { + t.Helper() + generator := digestFunction.NewGenerator(int64(len(data))) + _, err := generator.Write(data) + require.NoError(t, err) + return generator.Sum() +} + +var testCDCParams = &remoteexecution.RepMaxCdcParams{ + MinChunkSizeBytes: 1024, + HorizonSizeBytes: 8 * 1024, +} +var maximumMessageSizeBytes = 1024 * 1024 + +func TestChunkListValidatingBlobAccessGetTrivialSmallBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobData := []byte("Small trivial blob") + blobDigest := mustComputeDigest(t, digestFunction, blobData) + + require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) + + fakeCAS.ResetTouches() + msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + + splitResponse := msg.(*remoteexecution.SplitBlobResponse) + require.Len(t, splitResponse.ChunkDigests, 1) + require.Equal(t, blobDigest.GetProto().Hash, splitResponse.ChunkDigests[0].Hash) + require.Greater(t, fakeCAS.GetTouches(blobDigest), 0, "Blob did not have its lifetime renewed.") +} + +func TestChunkListValidatingBlobAccessGetLargeBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobData := bytes.Repeat([]byte("test_data_pattern_"), 6000) + blobDigest := mustComputeDigest(t, digestFunction, blobData) + + require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) + + msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + + splitResponse := msg.(*remoteexecution.SplitBlobResponse) + require.Greater(t, len(splitResponse.ChunkDigests), 1, "Blob should have been divided into multiple chunks.") + + for _, chunkProto := range splitResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) + require.NoError(t, err) + + require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk generated by CDC did not have its lifetime renewed.") + missing, err := fakeCAS.FindMissing(ctx, chunkDigest.ToSingletonSet()) + require.NoError(t, err) + require.True(t, missing.Empty(), "Chunk generated by CDC was not saved to the CAS.") + } + + cachedMsg, err := fakeCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + require.Equal(t, len(splitResponse.ChunkDigests), len(cachedMsg.(*remoteexecution.SplitBlobResponse).ChunkDigests)) +} + +func TestChunkListValidatingBlobAccessGetExtendsLifetimes(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobData := bytes.Repeat([]byte("test_data_pattern_"), 6000) // ~108KB + blobDigest := mustComputeDigest(t, digestFunction, blobData) + require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) + + // Split the blob to populate the CAS and CLS for this blob. + msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + splitResponse := msg.(*remoteexecution.SplitBlobResponse) + + // Reset touches. + fakeCAS.ResetTouches() + + // Perform a cached split. + msgCached, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + cachedResponse := msgCached.(*remoteexecution.SplitBlobResponse) + require.Equal(t, len(splitResponse.ChunkDigests), len(cachedResponse.ChunkDigests)) + + // The original blob MUST have had its lifetime extended + require.Greater(t, fakeCAS.GetTouches(blobDigest), 0, "Original blob's lifetime was not extended during call to SplitBlob") + require.Greater(t, fakeCLS.GetTouches(blobDigest), 0, "Original blob's chunk list lifetime was not extended during call to SplitBlob") + + // Every chunk MUST have been touched in the CAS + for _, chunkProto := range cachedResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) + require.NoError(t, err) + require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk's lifetime was not extended during call to SplitBlob") + } +} + +func TestChunkListValidatingBlobAccessPutManualSplice(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunk1Data := []byte("Hello, ") + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Data := []byte("World!") + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + expectedFullData := []byte("Hello, World!") + fullBlobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunk1Digest.GetProto(), + chunk2Digest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, fullBlobDigest).ToByteSlice(len(expectedFullData)) + require.NoError(t, err) + require.Equal(t, expectedFullData, composedData) +} + +func TestChunkListValidatingBlobAccessPutCanonicalization(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + blobData := bytes.Repeat([]byte("canonicalization_test_data"), 4000) // ~104KB + chunk1Data := blobData[:10] + chunk2Data := blobData[10:] + + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + fullBlobDigest := mustComputeDigest(t, digestFunction, blobData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunk1Digest.GetProto(), + chunk2Digest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, fullBlobDigest).ToByteSlice(200000) + require.NoError(t, err) + require.Equal(t, blobData, composedData) + + canonicalBuffer := fakeCLS.Get(ctx, fullBlobDigest) + canonicalProto, err := canonicalBuffer.ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + + canonicalResp := canonicalProto.(*remoteexecution.SplitBlobResponse) + require.Greater(t, len(canonicalResp.ChunkDigests), 0) + require.NotEqual(t, chunk1Digest.GetProto().Hash, canonicalResp.ChunkDigests[0].Hash, "Server should not have echoed back the non-standard chunks") +} + +func TestChunkListValidatingBlobAccessPutMissingChunk(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + chunkDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{chunkDigest.GetProto()}, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, chunkDigest, reqBuffer) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) +} + +func TestChunkListValidatingBlobAccessPutDigestMismatch(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkData := []byte("Valid chunk data") + chunkDigest := mustComputeDigest(t, digestFunction, chunkData) + require.NoError(t, fakeCAS.Put(ctx, chunkDigest, buffer.NewValidatedBufferFromByteSlice(chunkData))) + + wrongBlobDigest := mustComputeDigest(t, digestFunction, []byte("Different data")) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{chunkDigest.GetProto()}, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, wrongBlobDigest, reqBuffer) + require.Error(t, err) + require.Contains(t, err.Error(), "does not compose to blob") +} + +func TestChunkListValidatingBlobAccessPutEmptyBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + emptyDigest := mustComputeDigest(t, digestFunction, nil) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{}, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, emptyDigest, reqBuffer) + require.NoError(t, err) +} + +func TestChunkListValidatingBlobAccessPutRepeatedChunks(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkA := []byte("A") + digestA := mustComputeDigest(t, digestFunction, chunkA) + require.NoError(t, fakeCAS.Put(ctx, digestA, buffer.NewValidatedBufferFromByteSlice(chunkA))) + + chunkB := []byte("B") + digestB := mustComputeDigest(t, digestFunction, chunkB) + require.NoError(t, fakeCAS.Put(ctx, digestB, buffer.NewValidatedBufferFromByteSlice(chunkB))) + + expectedData := []byte("AABA") + expectedDigest := mustComputeDigest(t, digestFunction, expectedData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + digestA.GetProto(), + digestA.GetProto(), + digestB.GetProto(), + digestA.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, expectedDigest, reqBuffer) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, expectedDigest).ToByteSlice(100) + require.NoError(t, err) + require.Equal(t, expectedData, composedData) +} + +func TestChunkListValidatingBlobAccessPutInlineEmptyChunk(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkData := []byte("Valid") + chunkDigest := mustComputeDigest(t, digestFunction, chunkData) + require.NoError(t, fakeCAS.Put(ctx, chunkDigest, buffer.NewValidatedBufferFromByteSlice(chunkData))) + + emptyDigest := mustComputeDigest(t, digestFunction, nil) + require.NoError(t, fakeCAS.Put(ctx, emptyDigest, buffer.NewValidatedBufferFromByteSlice(nil))) + + expectedDigest := mustComputeDigest(t, digestFunction, chunkData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunkDigest.GetProto(), + emptyDigest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, expectedDigest, reqBuffer) + require.NoError(t, err) +} + +func TestChunkListValidatingBlobAccessPutExtendsLifetimes(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunk1Data := []byte("Hello, ") + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Data := []byte("World!") + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + expectedFullData := []byte("Hello, World!") + fullBlobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + + fakeCAS.ResetTouches() + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunk1Digest.GetProto(), + chunk2Digest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + + // From the REAPI, the server may either process the splice and + // return OK, OR it may return ALREADY_EXISTS if the blob is already + // composed and the server chooses not to extend the lifetime of the + // user's specific chunks. + if status.Code(err) == codes.AlreadyExists { + // The server is free not to touch the user's chunks. However, + // it MUST still have verified/touched the original blob. + require.Greater(t, fakeCAS.GetTouches(fullBlobDigest), 0, "Composed blob lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") + } else { + // Because the server accepted the Splice request, it is + // strictly obligated to extend the lifetimes of BOTH the + // provided chunks and the composed blob. + require.NoError(t, err) + + require.Greater(t, fakeCAS.GetTouches(chunk1Digest), 0, "Chunk 1 lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCAS.GetTouches(chunk2Digest), 0, "Chunk 2 lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCAS.GetTouches(fullBlobDigest), 0, "Composed blob lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") + } +} + +func TestChunkListValidatingBlobAccessGetMissingBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, 1024*1024) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + ghostDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) + + _, err := validatingCLS.Get(ctx, ghostDigest).ToProto(&remoteexecution.SplitBlobResponse{}, 1024*1024) + + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) +} diff --git a/pkg/blobstore/chunklistvalidating/chunker.go b/pkg/blobstore/chunklistvalidating/chunker.go new file mode 100644 index 000000000..8badbc7e9 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunker.go @@ -0,0 +1,16 @@ +package chunklistvalidating + +import ( + "github.com/buildbarn/bb-storage/pkg/digest" +) + +// Chunk is a struct of raw binary data with its digest. +type Chunk struct { + Digest digest.Digest + Data []byte +} + +// Chunker is an interface that provides a sequence of chunks. +type Chunker interface { + NextChunk() (Chunk, error) +} diff --git a/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go b/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go new file mode 100644 index 000000000..ffd7922c1 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go @@ -0,0 +1,88 @@ +package chunklistvalidating_test + +import ( + "context" + "sync" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// fakeBlobAccess provides a thread-safe, in-memory BlobAccess for +// testing. +type fakeBlobAccess struct { + blobstore.BlobAccess + lock sync.Mutex + blobs map[digest.Digest][]byte + touches map[digest.Digest]int // Tracks lifetime extensions + chunkingParameters *remoteexecution.RepMaxCdcParams +} + +func newFakeBlobAccess(chunkingParameters *remoteexecution.RepMaxCdcParams) *fakeBlobAccess { + return &fakeBlobAccess{ + blobs: make(map[digest.Digest][]byte), + touches: make(map[digest.Digest]int), + chunkingParameters: chunkingParameters, + } +} + +func (f *fakeBlobAccess) Get(ctx context.Context, d digest.Digest) buffer.Buffer { + f.lock.Lock() + defer f.lock.Unlock() + data, ok := f.blobs[d] + if !ok { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found")) + } + return buffer.NewValidatedBufferFromByteSlice(data) +} + +func (f *fakeBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { + data, err := b.ToByteSlice(100 * 1024 * 1024) + if err != nil { + return err + } + f.lock.Lock() + defer f.lock.Unlock() + f.touches[d]++ + f.blobs[d] = data + return nil +} + +func (f *fakeBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + f.lock.Lock() + defer f.lock.Unlock() + missing := digest.NewSetBuilder(digests.Length()) + for _, d := range digests.Items() { + if _, ok := f.blobs[d]; !ok { + missing.Add(d) + } else { + f.touches[d]++ + } + } + return missing.Build(), nil +} + +func (f *fakeBlobAccess) GetCapabilities(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.ServerCapabilities, error) { + return &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + RepMaxCdcParams: f.chunkingParameters, + }, + }, nil +} + +func (f *fakeBlobAccess) GetTouches(d digest.Digest) int { + f.lock.Lock() + defer f.lock.Unlock() + return f.touches[d] +} + +func (f *fakeBlobAccess) ResetTouches() { + f.lock.Lock() + defer f.lock.Unlock() + f.touches = make(map[digest.Digest]int) +} diff --git a/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel b/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel new file mode 100644 index 000000000..ca423091e --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "integration_test", + srcs = ["chunk_list_validating_integration_test.go"], + data = ["//cmd/bb_storage"], + env = { + "BB_STORAGE_RUNFILE_PATH": "$(rlocationpath //cmd/bb_storage:bb_storage)", + }, + deps = [ + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//credentials/insecure", + "@org_golang_google_grpc//status", + "@rules_go//go/runfiles", + ], +) diff --git a/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go b/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go new file mode 100644 index 000000000..033ec595c --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go @@ -0,0 +1,445 @@ +package integration + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "math/rand" + "os" + "os/exec" + "strings" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/stretchr/testify/require" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +type serverParams struct { + disableCLS bool + socketPath string + upstreamSocketPath string +} + +func escapeJSON(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, "\"", "\\\"") + s = strings.ReplaceAll(s, "'", "\\'") + s = strings.ReplaceAll(s, "\n", "\\n") + return s +} + +func storageConfig(params serverParams) string { + return fmt.Sprintf(` +local cls = %t; +local listenPath = '%s'; +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: 4 * 1024 * 1024, + contentAddressableStorage: { + backend: { + 'local': { + keyLocationMapInMemory: { entries: 1024 }, + keyLocationMapMaximumGetAttempts: 32, + keyLocationMapMaximumPutAttempts: 64, + oldBlocks: 1, + currentBlocks: 1, + newBlocks: 1, + blocksInMemory: { blockSizeBytes: 1024 * 1024 }, + }, + }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, + chunkListStorage: if !cls then null else { + backend: { + 'local': { + keyLocationMapInMemory: { entries: 1024 }, + keyLocationMapMaximumGetAttempts: 32, + keyLocationMapMaximumPutAttempts: 64, + oldBlocks: 1, + currentBlocks: 1, + newBlocks: 1, + blocksInMemory: { blockSizeBytes: 1024 * 1024 }, + chunkingParameters: { + minChunkSizeBytes: 256, + horizonSizeBytes: 8*256, + } + }, + }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, +} +`, !params.disableCLS, escapeJSON(params.socketPath)) +} + +func frontendConfig(params serverParams) string { + return fmt.Sprintf(` +local cls = %t; +local listenPath = '%s'; +// unix:// doesn't work under Windows. +// https://github.com/grpc/grpc-go/issues/8675 +local upstreamAddress = 'unix:%s'; +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: 4 * 1024 * 1024, + contentAddressableStorage: { + backend: { grpc: { client: { address: upstreamAddress } } }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, + chunkListStorage: if !cls then null else { + backend: { chunkListValidating: { backend: { grpc: { client: { address: upstreamAddress } } } } }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, +} +`, !params.disableCLS, escapeJSON(params.socketPath), escapeJSON(params.upstreamSocketPath)) +} + +func writeConfigFile(name, content string) (file *os.File, err error) { + if file, err = os.CreateTemp("", name); err != nil { + return nil, err + } + if _, err = file.WriteString(content); err != nil { + return nil, err + } + if err = file.Close(); err != nil { + return nil, err + } + return file, nil +} + +func setupServer(t *testing.T, name, config string) func() { + rf, err := runfiles.New() + if err != nil { + t.Fatalf("Failed to initialize runfiles: %v", err) + } + runfilePath := os.Getenv("BB_STORAGE_RUNFILE_PATH") + require.NotEmpty(t, runfilePath, "BB_STORAGE_RUNFILE_PATH environment variable is not set") + + bbStoragePath, err := rf.Rlocation(runfilePath) + require.NoError(t, err) + + configFile, err := writeConfigFile(name, config) + require.NoError(t, err) + + cmd := exec.Command(bbStoragePath, configFile.Name()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Start() + require.NoError(t, err) + + return func() { + cmd.Process.Kill() + cmd.Wait() + os.Remove(configFile.Name()) + } +} + +func createSocketPath(t *testing.T) string { + t.Helper() + socketFile, err := os.CreateTemp("", "bb_*.sock") + require.NoError(t, err) + socketPath := socketFile.Name() + socketFile.Close() + err = os.Remove(socketPath) + require.NoError(t, err) + return socketPath +} + +func setupServers(t *testing.T, storageParams, frontendParams serverParams) (func(), remoteexecution.CapabilitiesClient, remoteexecution.ContentAddressableStorageClient) { + storageParams.socketPath = createSocketPath(t) + closeStorage := setupServer(t, "storage", storageConfig(storageParams)) + require.Eventually(t, func() bool { + _, err := os.Stat(storageParams.socketPath) + return err == nil + }, 1*time.Second, 10*time.Millisecond, "Storage server did not start.") + + frontendParams.socketPath = createSocketPath(t) + frontendParams.upstreamSocketPath = storageParams.socketPath + closeFrontend := setupServer(t, "frontend", frontendConfig(frontendParams)) + require.Eventually(t, func() bool { + _, err := os.Stat(frontendParams.socketPath) + return err == nil + }, 1*time.Second, 10*time.Millisecond, "Frontend server did not start.") + + conn, err := grpc.NewClient(fmt.Sprintf("unix:%s", frontendParams.socketPath), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + + return func() { + closeStorage() + closeFrontend() + conn.Close() + os.Remove(storageParams.socketPath) + os.Remove(frontendParams.socketPath) + }, remoteexecution.NewCapabilitiesClient(conn), remoteexecution.NewContentAddressableStorageClient(conn) +} + +func TestChunkListValidatingCapabilities(t *testing.T) { + tests := []struct { + name string + storageParams serverParams + frontendParams serverParams + expectSupport bool + }{ + {"Enabled In Both", serverParams{}, serverParams{}, true}, + {"Disabled in Storage", serverParams{disableCLS: true}, serverParams{}, false}, + {"Disabled in Frontend", serverParams{}, serverParams{disableCLS: true}, false}, + {"Disabled in Both", serverParams{disableCLS: true}, serverParams{disableCLS: true}, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + closer, capabilitiesClient, _ := setupServers(t, tc.storageParams, tc.frontendParams) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + capabilities, err := capabilitiesClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: "", + }) + require.NoError(t, err) + + cacheCaps := capabilities.CacheCapabilities + require.NotNil(t, cacheCaps) + + if tc.expectSupport { + require.True(t, cacheCaps.SpliceBlobSupport) + require.True(t, cacheCaps.SplitBlobSupport) + + chunkingParameters := cacheCaps.GetRepMaxCdcParams() + require.NotNil(t, chunkingParameters) + require.Equal(t, uint64(256), chunkingParameters.GetMinChunkSizeBytes()) + require.Equal(t, uint64(2048), chunkingParameters.GetHorizonSizeBytes()) + } else { + require.False(t, cacheCaps.SpliceBlobSupport) + require.False(t, cacheCaps.SplitBlobSupport) + require.Nil(t, cacheCaps.GetRepMaxCdcParams()) + } + }) + } +} + +func computeDigest(data []byte) *remoteexecution.Digest { + hash := sha256.Sum256(data) + return &remoteexecution.Digest{ + Hash: hex.EncodeToString(hash[:]), + SizeBytes: int64(len(data)), + } +} + +func makeRandomData(t *testing.T, size int, seed int64) []byte { + t.Helper() + data := make([]byte, size) + r := rand.New(rand.NewSource(seed)) + _, err := r.Read(data) + require.NoError(t, err) + return data +} + +func uploadBlob(ctx context.Context, t *testing.T, cas remoteexecution.ContentAddressableStorageClient, data []byte) *remoteexecution.Digest { + t.Helper() + digest := computeDigest(data) + req := &remoteexecution.BatchUpdateBlobsRequest{ + Requests: []*remoteexecution.BatchUpdateBlobsRequest_Request{ + {Digest: digest, Data: data}, + }, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + res, err := cas.BatchUpdateBlobs(ctx, req) + require.NoError(t, err) + require.NotEmpty(t, res.Responses, "server returned empty responses array") + status := res.Responses[0].GetStatus() + require.Equal(t, int32(0), status.GetCode(), status.GetMessage()) + return digest +} + +func findMissingBlobs(ctx context.Context, t *testing.T, cas remoteexecution.ContentAddressableStorageClient, digests []*remoteexecution.Digest) []*remoteexecution.Digest { + t.Helper() + req := &remoteexecution.FindMissingBlobsRequest{ + BlobDigests: digests, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + res, err := cas.FindMissingBlobs(ctx, req) + require.NoError(t, err) + return res.MissingBlobDigests +} + +func TestRepMaxCDCSplitAndSpliceBehaviors(t *testing.T) { + minChunkSize := int64(256) + + t.Run("RoundTripSplitThenSplice", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + dataSize := (minChunkSize * 4) + 128 + data := makeRandomData(t, int(dataSize), 0) + blobDigest := uploadBlob(ctx, t, casClient, data) + + splitReq := &remoteexecution.SplitBlobRequest{ + BlobDigest: blobDigest, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + splitRes, err := casClient.SplitBlob(ctx, splitReq) + require.NoError(t, err) + + spliceReq := &remoteexecution.SpliceBlobRequest{ + BlobDigest: blobDigest, + ChunkDigests: splitRes.ChunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) + require.NoError(t, err) + require.Equal(t, blobDigest.Hash, spliceRes.BlobDigest.Hash) + }) + + t.Run("SpliceNonStandardChunkingThenSplit", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := makeRandomData(t, int(minChunkSize*2), 0) + + chunk1 := blobData[:1] + chunk2 := blobData[1:] + + digest1 := uploadBlob(ctx, t, casClient, chunk1) + digest2 := uploadBlob(ctx, t, casClient, chunk2) + expectedDigest := computeDigest(blobData) + + spliceReq := &remoteexecution.SpliceBlobRequest{ + BlobDigest: expectedDigest, + ChunkDigests: []*remoteexecution.Digest{digest1, digest2}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) + require.NoError(t, err) + require.Equal(t, expectedDigest.Hash, spliceRes.BlobDigest.Hash) + + splitReq := &remoteexecution.SplitBlobRequest{ + BlobDigest: expectedDigest, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + splitRes, err := casClient.SplitBlob(ctx, splitReq) + require.NoError(t, err) + + require.NotEmpty(t, splitRes.ChunkDigests) + + // Check that it didn't just echo our chunks back + isEcho := len(splitRes.ChunkDigests) == 2 && + splitRes.ChunkDigests[0].Hash == digest1.Hash && + splitRes.ChunkDigests[1].Hash == digest2.Hash + require.False(t, isEcho, "Server echoed non-standard chunks") + + var totalSize int64 + for _, c := range splitRes.ChunkDigests { + totalSize += c.SizeBytes + } + require.Equal(t, expectedDigest.SizeBytes, totalSize) + }) + + t.Run("SpliceAlreadyExistsOrNoop", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := append([]byte("This blob will be fully uploaded before we try to splice it."), makeRandomData(t, 16, 0)...) + expectedDigest := uploadBlob(ctx, t, casClient, blobData) + + chunk1 := blobData[:10] + chunk2 := blobData[10:] + digest1 := uploadBlob(ctx, t, casClient, chunk1) + digest2 := uploadBlob(ctx, t, casClient, chunk2) + + spliceReq := &remoteexecution.SpliceBlobRequest{ + BlobDigest: expectedDigest, + ChunkDigests: []*remoteexecution.Digest{digest1, digest2}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) + + if err != nil { + require.Equal(t, codes.AlreadyExists, status.Code(err), "Expected OK or ALREADY_EXISTS") + } else { + require.Equal(t, expectedDigest.Hash, spliceRes.BlobDigest.Hash) + } + }) + + t.Run("ValidationSpliceBlobRejections", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + validData := makeRandomData(t, 512, 0) + validDigest := uploadBlob(ctx, t, casClient, validData) + ghostDigest := computeDigest([]byte("I do not exist")) + + tests := []struct { + name string + req *remoteexecution.SpliceBlobRequest + expectError codes.Code + }{ + { + name: "Missing Chunk", + req: &remoteexecution.SpliceBlobRequest{ + BlobDigest: ghostDigest, + ChunkDigests: []*remoteexecution.Digest{ghostDigest}, + }, + expectError: codes.NotFound, + }, + { + name: "Digest Mismatch", + req: &remoteexecution.SpliceBlobRequest{ + BlobDigest: computeDigest([]byte("Fake target")), + ChunkDigests: []*remoteexecution.Digest{validDigest}, + }, + expectError: codes.InvalidArgument, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.req.ChunkingFunction = remoteexecution.ChunkingFunction_REP_MAX_CDC + tc.req.DigestFunction = remoteexecution.DigestFunction_SHA256 + + _, err := casClient.SpliceBlob(ctx, tc.req) + require.Error(t, err) + require.Equal(t, tc.expectError, status.Code(err)) + }) + } + }) +} diff --git a/pkg/blobstore/chunklistvalidating/reader_chunker.go b/pkg/blobstore/chunklistvalidating/reader_chunker.go new file mode 100644 index 000000000..0d250dabd --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/reader_chunker.go @@ -0,0 +1,53 @@ +package chunklistvalidating + +import ( + "bufio" + "io" + + "github.com/buildbarn/bb-storage/pkg/digest" + cdc "github.com/buildbarn/go-cdc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type readerChunker struct { + cdcChunker cdc.ContentDefinedChunker + reader io.Reader + digestFunction digest.Function +} + +func (c *readerChunker) NextChunk() (Chunk, error) { + chunk, err := c.cdcChunker.ReadNextChunk() + if err != nil { + return Chunk{}, err + } + + digestGenerator := c.digestFunction.NewGenerator(int64(len(chunk))) + if _, err := digestGenerator.Write(chunk); err != nil { + return Chunk{}, status.Error(codes.Internal, "Could not compute digest of chunk") + } + chunkDigest := digestGenerator.Sum() + + return Chunk{ + Data: chunk, + Digest: chunkDigest, + }, nil +} + +// NewReaderChunker creates a chunker that reads from an io.Reader +func NewReaderChunker(digestFunction digest.Function, reader io.Reader, minChunkSizeBytes, horizonSizeBytes int64) Chunker { + // The internal RepMaxContentDefinedChunker may peek up to this many + // bytes. We therefore make sure that the underlying buffer is big + // enough to prevent bufio.ErrBufferFull errors. + bufferSizeBytes := 2*minChunkSizeBytes + horizonSizeBytes + return &readerChunker{ + cdc.NewRepMaxContentDefinedChunker( + bufio.NewReaderSize(reader, int(bufferSizeBytes)), + &cdc.FastContentDefinedChunkerGearTable, + int(minChunkSizeBytes), + int(horizonSizeBytes), + ), + reader, + digestFunction, + } +} diff --git a/pkg/blobstore/chunklistvalidating/reader_chunker_test.go b/pkg/blobstore/chunklistvalidating/reader_chunker_test.go new file mode 100644 index 000000000..c5630d427 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/reader_chunker_test.go @@ -0,0 +1,107 @@ +package chunklistvalidating_test + +import ( + "io" + "math/rand" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" +) + +const ( + minChunkSize = 256 << 10 // 256 KiB + maxChunkSize = 2*minChunkSize - 1 + horizonLookaheadBytes = 8 * minChunkSize +) + +func FuzzReaderChunker(f *testing.F) { + for i := range 20 { + // Fuzz test i+1 MB of data with seed i. + f.Add((i+1)<<20, int64(i)) + } + f.Fuzz(func(t *testing.T, dataSizeBytes int, seed int64) { + require := require.New(t) + rng := rand.New(rand.NewSource(seed)) + originalData := make([]byte, dataSizeBytes) + rng.Read(originalData) + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + composedData := make([]byte, 0, dataSizeBytes) + var numberOfChunks int + for numberOfChunks = 0; ; numberOfChunks++ { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + require.NoError(err, "Failed to generate chunk %d.", numberOfChunks) + + chunkSize := int64(len(chunk.Data)) + chunkHasher := chunk.Digest.NewHasher(chunkSize) + chunkHasher.Write(chunk.Data) + + require.Equal(chunk.Digest.GetHashBytes(), chunkHasher.Sum(nil), "Digest mismatch for %d.", numberOfChunks) + composedData = append(composedData, chunk.Data...) + } + + require.Equal(originalData, composedData) + + originalDigestGen := digestFunc.NewGenerator(int64(dataSizeBytes)) + originalDigestGen.Write(originalData) + + composedDigestGen := digestFunc.NewGenerator(int64(dataSizeBytes)) + composedDigestGen.Write(composedData) + + require.Equal(originalDigestGen.Sum(), composedDigestGen.Sum(), "The digest of the composed data does not match the digest of the original data.") + + minNumberOfChunks := dataSizeBytes / maxChunkSize + require.GreaterOrEqual(numberOfChunks, minNumberOfChunks, "Produced fewer chunks than should be possible.") + + maxNumberOfChunks := dataSizeBytes / minChunkSize + require.LessOrEqual(numberOfChunks, maxNumberOfChunks, "Produced more chunks than should be possible.") + }) +} + +func TestReaderChunkerSmallBlob(t *testing.T) { + // Test with a small blob that should produce a single chunk + originalData := []byte("Hello, World!") + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + chunks := make([][]byte, 0, 1) + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + require.NoError(t, err) + + chunks = append(chunks, chunk.Data) + } + require.Len(t, chunks, 1) + require.Equal(t, originalData, chunks[0]) +} + +func TestReaderChunkerEmptyBlob(t *testing.T) { + // Test with empty blob + originalData := []byte{} + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + chunk, err := chunker.NextChunk() + require.ErrorIs(t, io.EOF, err) + require.Empty(t, chunk.Data) +} diff --git a/pkg/blobstore/cls_read_buffer_factory.go b/pkg/blobstore/cls_read_buffer_factory.go new file mode 100644 index 000000000..7735f064b --- /dev/null +++ b/pkg/blobstore/cls_read_buffer_factory.go @@ -0,0 +1,27 @@ +package blobstore + +import ( + "io" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" +) + +type clsReadBufferFactory struct{} + +func (clsReadBufferFactory) NewBufferFromByteSlice(digest digest.Digest, data []byte, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return buffer.NewProtoBufferFromByteSlice(&remoteexecution.SplitBlobResponse{}, data, buffer.BackendProvided(dataIntegrityCallback)) +} + +func (clsReadBufferFactory) NewBufferFromReader(digest digest.Digest, r io.ReadCloser, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return buffer.NewProtoBufferFromReader(&remoteexecution.SplitBlobResponse{}, r, buffer.BackendProvided(dataIntegrityCallback)) +} + +func (f clsReadBufferFactory) NewBufferFromReaderAt(digest digest.Digest, r buffer.ReadAtCloser, sizeBytes int64, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return f.NewBufferFromReader(digest, newReaderFromReaderAt(r), dataIntegrityCallback) +} + +// CLSReadBufferFactory is capable of creating identifiers and buffers +// for objects stored in the Chunk List Storage (CLS). +var CLSReadBufferFactory ReadBufferFactory = clsReadBufferFactory{} diff --git a/pkg/blobstore/configuration/BUILD.bazel b/pkg/blobstore/configuration/BUILD.bazel index 33ebe095b..8882b2b2e 100644 --- a/pkg/blobstore/configuration/BUILD.bazel +++ b/pkg/blobstore/configuration/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "blob_replicator_creator.go", "cas_blob_access_creator.go", "cas_blob_replicator_creator.go", + "cls_blob_access_creator.go", "fsac_blob_access_creator.go", "icas_blob_access_creator.go", "icas_blob_replicator_creator.go", @@ -21,6 +22,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/blobstore", + "//pkg/blobstore/chunklistvalidating", "//pkg/blobstore/completenesschecking", "//pkg/blobstore/grpcclients", "//pkg/blobstore/local", diff --git a/pkg/blobstore/configuration/blob_access_creator.go b/pkg/blobstore/configuration/blob_access_creator.go index 6b44c1171..50d6032ce 100644 --- a/pkg/blobstore/configuration/blob_access_creator.go +++ b/pkg/blobstore/configuration/blob_access_creator.go @@ -48,7 +48,7 @@ type BlobAccessCreator interface { // NewHierarchicalInstanceNamesLocalBlobAccess() creates a // BlobAccess suitable for storing data on the local system that // uses hierarchical instance names. - NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) + NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) // NewCustomBlobAccess() can be used as a fallback to create // BlobAccess instances that only apply to this storage type. // For example, CompletenessCheckingBlobAccess is only diff --git a/pkg/blobstore/configuration/cas_blob_access_creator.go b/pkg/blobstore/configuration/cas_blob_access_creator.go index 2677982a9..ac215dd68 100644 --- a/pkg/blobstore/configuration/cas_blob_access_creator.go +++ b/pkg/blobstore/configuration/cas_blob_access_creator.go @@ -72,8 +72,8 @@ func (casBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks in return local.NewImmutableBlockListGrowthPolicy(currentBlocks, newBlocks), nil } -func (casBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) { - return local.NewHierarchicalCASBlobAccess(keyLocationMap, locationBlobMap, globalLock, casCapabilitiesProvider), nil +func (casBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { + return local.NewHierarchicalCASBlobAccess(keyLocationMap, locationBlobMap, globalLock, capabilitiesProvider), nil } func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { diff --git a/pkg/blobstore/configuration/cls_blob_access_creator.go b/pkg/blobstore/configuration/cls_blob_access_creator.go new file mode 100644 index 000000000..af2e695c9 --- /dev/null +++ b/pkg/blobstore/configuration/cls_blob_access_creator.go @@ -0,0 +1,86 @@ +package configuration + +import ( + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" + "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/grpc" + "github.com/buildbarn/bb-storage/pkg/program" + pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type clsBlobAccessCreator struct { + protoBlobAccessCreator + protoBlobReplicatorCreator + + contentAddressableStorage *BlobAccessInfo + grpcClientFactory grpc.ClientFactory + maximumMessageSizeBytes int +} + +// NewCLSBlobAccessCreator creates a BlobAccessCreator that can be +// provided to NewBlobAccessFromConfiguration() to construct a +// BlobAccess that is suitable for querying for chunk list. +func NewCLSBlobAccessCreator(contentAddressableStorage *BlobAccessInfo, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { + return &clsBlobAccessCreator{ + contentAddressableStorage: contentAddressableStorage, + grpcClientFactory: grpcClientFactory, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (clsBlobAccessCreator) GetReadBufferFactory() blobstore.ReadBufferFactory { + return blobstore.CLSReadBufferFactory +} + +func (clsBlobAccessCreator) GetStorageTypeName() string { + return "cls" +} + +func (clsBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provider { + return capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{}) +} + +func (bac *clsBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { + switch backend := configuration.Backend.(type) { + case *pb.BlobAccessConfiguration_ChunkListValidating: + if bac.contentAddressableStorage == nil { + return BlobAccessInfo{}, "", status.Error(codes.InvalidArgument, "Chunk list validation can only be enabled if a Content Addressable Storage is configured") + } + + base, err := nestedCreator.NewNestedBlobAccess(backend.ChunkListValidating.Backend, bac) + if err != nil { + return BlobAccessInfo{}, "", err + } + return BlobAccessInfo{ + BlobAccess: chunklistvalidating.NewChunkListValidatingBlobAccess( + base.BlobAccess, + bac.contentAddressableStorage.BlobAccess, + bac.maximumMessageSizeBytes, + ), + DigestKeyFormat: base.DigestKeyFormat.Combine(bac.contentAddressableStorage.DigestKeyFormat), + }, "chunk_list_validating", nil + + case *pb.BlobAccessConfiguration_Grpc: + client, err := bac.grpcClientFactory.NewClientFromConfiguration(backend.Grpc.Client, terminationGroup) + if err != nil { + return BlobAccessInfo{}, "", err + } + return BlobAccessInfo{ + BlobAccess: grpcclients.NewCLSBlobAccess(client, bac.maximumMessageSizeBytes), + DigestKeyFormat: digest.KeyWithInstance, + }, "grpc", nil + + default: + return newProtoCustomBlobAccess(configuration, nestedCreator, bac) + } +} + +func (clsBlobAccessCreator) WrapTopLevelBlobAccess(blobAccess blobstore.BlobAccess) blobstore.BlobAccess { + return blobAccess +} diff --git a/pkg/blobstore/configuration/new_blob_access.go b/pkg/blobstore/configuration/new_blob_access.go index 941494d66..07e52041c 100644 --- a/pkg/blobstore/configuration/new_blob_access.go +++ b/pkg/blobstore/configuration/new_blob_access.go @@ -7,6 +7,7 @@ import ( "sync" "time" + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/local" "github.com/buildbarn/bb-storage/pkg/blobstore/mirrored" @@ -14,6 +15,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/readfallback" "github.com/buildbarn/bb-storage/pkg/blobstore/sharding" "github.com/buildbarn/bb-storage/pkg/blockdevice" + "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/eviction" @@ -352,11 +354,26 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration * ) var localBlobAccess blobstore.BlobAccess + capabilitiesProvider := creator.GetDefaultCapabilitiesProvider() + chunkingParameters := backend.Local.GetChunkingParameters() + if chunkingParameters != nil { + capabilitiesProvider = capabilities.NewMergingProvider([]capabilities.Provider{ + capabilitiesProvider, + capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: true, + SpliceBlobSupport: true, + RepMaxCdcParams: chunkingParameters, + }, + }), + }) + } if backend.Local.HierarchicalInstanceNames { localBlobAccess, err = creator.NewHierarchicalInstanceNamesLocalBlobAccess( keyLocationMap, locationBlobMap, &globalLock, + capabilitiesProvider, ) if err != nil { return BlobAccessInfo{}, "", err @@ -368,7 +385,7 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration * digestKeyFormat, &globalLock, storageTypeName, - creator.GetDefaultCapabilitiesProvider(), + capabilitiesProvider, ) } return BlobAccessInfo{ diff --git a/pkg/blobstore/configuration/proto_blob_access_creator.go b/pkg/blobstore/configuration/proto_blob_access_creator.go index aa7daf857..d7ff593ba 100644 --- a/pkg/blobstore/configuration/proto_blob_access_creator.go +++ b/pkg/blobstore/configuration/proto_blob_access_creator.go @@ -5,6 +5,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/local" + "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/digest" pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" @@ -25,7 +26,7 @@ func (protoBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks return local.NewMutableBlockListGrowthPolicy(currentBlocks), nil } -func (protoBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) { +func (protoBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { return nil, status.Error(codes.InvalidArgument, "The hierarchical instance names option can only be used for the Content Addressable Storage") } diff --git a/pkg/blobstore/grpcclients/BUILD.bazel b/pkg/blobstore/grpcclients/BUILD.bazel index 257d8fd04..bb518ecf6 100644 --- a/pkg/blobstore/grpcclients/BUILD.bazel +++ b/pkg/blobstore/grpcclients/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "ac_blob_access.go", "cas_blob_access.go", + "cls_blob_access.go", "fsac_blob_access.go", "icas_blob_access.go", "iscc_blob_access.go", diff --git a/pkg/blobstore/grpcclients/cls_blob_access.go b/pkg/blobstore/grpcclients/cls_blob_access.go new file mode 100644 index 000000000..a27e5ea84 --- /dev/null +++ b/pkg/blobstore/grpcclients/cls_blob_access.go @@ -0,0 +1,101 @@ +package grpcclients + +import ( + "context" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type clsBlobAccess struct { + contentAddressableStorageClient remoteexecution.ContentAddressableStorageClient + capabilitiesClient remoteexecution.CapabilitiesClient + maximumMessageSizeBytes int +} + +// NewCLSBlobAccess creates a BlobAccess that relays any requests to a +// gRPC server that implements the split and splice api calls of a +// remoteexecution.ContentAddressableStorage service. +func NewCLSBlobAccess(client grpc.ClientConnInterface, maximumMessageSizeBytes int) blobstore.BlobAccess { + return &clsBlobAccess{ + contentAddressableStorageClient: remoteexecution.NewContentAddressableStorageClient(client), + capabilitiesClient: remoteexecution.NewCapabilitiesClient(client), + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (ba *clsBlobAccess) Get(ctx context.Context, digest digest.Digest) buffer.Buffer { + digestFunction := digest.GetDigestFunction() + splitBlobsResponse, err := ba.contentAddressableStorageClient.SplitBlob(ctx, &remoteexecution.SplitBlobRequest{ + InstanceName: digestFunction.GetInstanceName().String(), + BlobDigest: digest.GetProto(), + DigestFunction: digestFunction.GetEnumValue(), + }) + if err != nil { + return buffer.NewBufferFromError(err) + } + return buffer.NewProtoBufferFromProto(splitBlobsResponse, buffer.BackendProvided(buffer.Irreparable(digest))) +} + +func (ba *clsBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { + b, _ := slicer.Slice(ba.Get(ctx, parentDigest), childDigest) + return b +} + +func (ba *clsBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { + splitBlobResponseProto, err := b.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return err + } + splitBlobResponse := splitBlobResponseProto.(*remoteexecution.SplitBlobResponse) + digestFunction := digest.GetDigestFunction() + _, err = ba.contentAddressableStorageClient.SpliceBlob(ctx, &remoteexecution.SpliceBlobRequest{ + InstanceName: digestFunction.GetInstanceName().String(), + DigestFunction: digestFunction.GetEnumValue(), + ChunkDigests: splitBlobResponse.GetChunkDigests(), + ChunkingFunction: splitBlobResponse.GetChunkingFunction(), + BlobDigest: digest.GetProto(), + }) + return err +} + +func (ba *clsBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + missing := digest.NewSetBuilder(digests.Length()) + for _, d := range digests.Items() { + _, err := ba.contentAddressableStorageClient.SplitBlob(ctx, &remoteexecution.SplitBlobRequest{ + InstanceName: d.GetInstanceName().String(), + BlobDigest: d.GetProto(), + DigestFunction: d.GetDigestFunction().GetEnumValue(), + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + }) + if status.Code(err) == codes.NotFound { + missing.Add(d) + } else if err != nil { + return digest.EmptySet, err + } + } + return missing.Build(), nil +} + +func (ba *clsBlobAccess) GetCapabilities(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.ServerCapabilities, error) { + serverCapabilities, err := getServerCapabilitiesWithCacheCapabilities(ctx, ba.capabilitiesClient, instanceName) + if err != nil { + return nil, err + } + cacheCapabilities := serverCapabilities.CacheCapabilities + // Only return fields that pertain to Chunk List Storage. + return &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: cacheCapabilities.SplitBlobSupport, + SpliceBlobSupport: cacheCapabilities.SpliceBlobSupport, + RepMaxCdcParams: cacheCapabilities.RepMaxCdcParams, + }, + }, nil +} diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server.go b/pkg/blobstore/grpcservers/content_addressable_storage_server.go index 18802b959..bd76a5e4c 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server.go @@ -15,14 +15,16 @@ import ( type contentAddressableStorageServer struct { contentAddressableStorage blobstore.BlobAccess + chunkListStorage blobstore.BlobAccess maximumMessageSizeBytes int64 } // NewContentAddressableStorageServer creates a GRPC service for serving // the contents of a Bazel Content Addressable Storage (CAS) to Bazel. -func NewContentAddressableStorageServer(contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { +func NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage blobstore.BlobAccess, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { return &contentAddressableStorageServer{ contentAddressableStorage: contentAddressableStorage, + chunkListStorage: chunkListStorage, maximumMessageSizeBytes: maximumMessageSizeBytes, } } @@ -56,6 +58,36 @@ func (s *contentAddressableStorageServer) FindMissingBlobs(ctx context.Context, for _, outDigest := range outDigests.Items() { partialDigests = append(partialDigests, outDigest.GetProto()) } + + // Server is configured with Chunk List Storage (CLS) so we must + // verify the CLS as well. Note that in this version of bb-storage a + // missing chunk list for a blob does not imply that the blob is + // missing. It is merely required to manage the life time of chunk + // lists. In a future version of bb-storage FMB calls will go to + // either the chunk storage or the chunk list storage. + if s.chunkListStorage != nil { + capabilities, err := s.chunkListStorage.GetCapabilities(ctx, instanceName) + if err != nil { + return nil, err + } + if capabilities.CacheCapabilities.GetRepMaxCdcParams() == nil { + return nil, status.Error(codes.Internal, "This server implementation is only compatible with RepMaxCDC") + } + minChunkSize := capabilities.CacheCapabilities.RepMaxCdcParams.MinChunkSizeBytes + maxChunkSize := 2*minChunkSize - 1 + bigBlobDigests := digest.NewSetBuilder(0) + for _, partialDigest := range in.BlobDigests { + if partialDigest.GetSizeBytes() > int64(maxChunkSize) { + digest, err := digestFunction.NewDigestFromProto(partialDigest) + if err != nil { + return nil, err + } + bigBlobDigests.Add(digest) + } + } + _, _ = s.chunkListStorage.FindMissing(ctx, bigBlobDigests.Build()) + } + return &remoteexecution.FindMissingBlobsResponse{ MissingBlobDigests: partialDigests, }, nil @@ -150,10 +182,61 @@ func (contentAddressableStorageServer) GetTree(in *remoteexecution.GetTreeReques return status.Error(codes.Unimplemented, "This service does not support downloading directory trees") } -func (contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *remoteexecution.SpliceBlobRequest) (*remoteexecution.SpliceBlobResponse, error) { - return nil, status.Error(codes.Unimplemented, "This service does not support splicing blobs") +func (s *contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *remoteexecution.SpliceBlobRequest) (*remoteexecution.SpliceBlobResponse, error) { + // TODO: Require that s.chunkListStorage is non-null once we require chunking. + if s.chunkListStorage == nil { + return nil, status.Error(codes.Unimplemented, "This service does not support SpliceBlob") + } + + instanceName, err := digest.NewInstanceName(in.InstanceName) + if err != nil { + return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) + } + digestFunction, err := instanceName.GetDigestFunction(in.DigestFunction, len(in.BlobDigest.GetHash())) + if err != nil { + return nil, err + } + blobDigest, err := digestFunction.NewDigestFromProto(in.BlobDigest) + if err != nil { + return nil, err + } + + splitBlobResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: in.ChunkDigests, + ChunkingFunction: in.ChunkingFunction, + } + b := buffer.NewProtoBufferFromProto(splitBlobResponse, buffer.UserProvided) + + if err := s.chunkListStorage.Put(ctx, blobDigest, b); err != nil { + return nil, err + } + + return &remoteexecution.SpliceBlobResponse{ + BlobDigest: in.BlobDigest, + }, nil } -func (contentAddressableStorageServer) SplitBlob(ctx context.Context, in *remoteexecution.SplitBlobRequest) (*remoteexecution.SplitBlobResponse, error) { - return nil, status.Error(codes.Unimplemented, "This service does not support splitting blobs") +func (s *contentAddressableStorageServer) SplitBlob(ctx context.Context, in *remoteexecution.SplitBlobRequest) (*remoteexecution.SplitBlobResponse, error) { + // TODO: Require that s.chunkListStorage is non-null once we require chunking. + if s.chunkListStorage == nil { + return nil, status.Error(codes.Unimplemented, "This service does not support SplitBlob") + } + + instanceName, err := digest.NewInstanceName(in.InstanceName) + if err != nil { + return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) + } + digestFunction, err := instanceName.GetDigestFunction(in.DigestFunction, len(in.BlobDigest.GetHash())) + if err != nil { + return nil, err + } + blobDigest, err := digestFunction.NewDigestFromProto(in.BlobDigest) + if err != nil { + return nil, err + } + splitBlobResponse, err := s.chunkListStorage.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, int(s.maximumMessageSizeBytes)) + if err != nil { + return nil, err + } + return splitBlobResponse.(*remoteexecution.SplitBlobResponse), nil } diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go index 89a278297..7c5ed83b4 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go @@ -55,7 +55,7 @@ func TestContentAddressableStorageServerBatchReadBlobsSuccess(t *testing.T) { buf3 := buffer.NewBufferFromError(status.Error(codes.NotFound, "The object you requested could not be found")) contentAddressableStorage.EXPECT().Get(ctx, digest3).Return(buf3) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, 1<<16) + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, nil, 4<<20) response, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) require.NoError(t, err) @@ -108,8 +108,138 @@ func TestContentAddressableStorageServerBatchReadBlobsFailure(t *testing.T) { contentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, 200) + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, nil, 200) _, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to read a total of at least 357 bytes, while a maximum of 200 bytes is permitted"), err) } + +func TestContentAddressableStorageServerFindMissingBlobs(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + digest1 := digest.MustNewDigest("my_instance_name", remoteexecution.DigestFunction_SHA256, "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", 16) + digest2 := digest.MustNewDigest("my_instance_name", remoteexecution.DigestFunction_SHA256, "0479688f99e8cbc70291ce272876ff8e0db71a0889daf2752884b0996056b4a0", 256) + + request := &remoteexecution.FindMissingBlobsRequest{ + InstanceName: "my_instance_name", + BlobDigests: []*remoteexecution.Digest{ + {Hash: digest1.GetHashString(), SizeBytes: digest1.GetSizeBytes()}, + {Hash: digest2.GetHashString(), SizeBytes: digest2.GetSizeBytes()}, + }, + } + + contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + chunkListStorage := mock.NewMockBlobAccess(ctrl) + setBuilder := digest.NewSetBuilder(2) + digestSet := setBuilder.Add(digest1).Add(digest2).Build() + + // Missing chunk lists is not an error, nor does it imply a missing + // blob at this stage. + contentAddressableStorage.EXPECT().FindMissing(ctx, digestSet).Return(digest.EmptySet, nil) + chunkListStorage.EXPECT().GetCapabilities(ctx, digest1.GetInstanceName()).Return( + &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: true, + SpliceBlobSupport: true, + RepMaxCdcParams: &remoteexecution.RepMaxCdcParams{ + MinChunkSizeBytes: 64, + HorizonSizeBytes: 128, + }, + }, + }, + nil, + ) + chunkListStorage.EXPECT().FindMissing(ctx, digest2.ToSingletonSet()).Return(digest2.ToSingletonSet(), nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + + response, err := contentAddressableStorageServer.FindMissingBlobs(ctx, request) + require.NoError(t, err) + require.Empty(t, response.GetMissingBlobDigests()) +} + +func TestContentAddressableStorageServerSplitBlob(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SplitBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + InstanceName: "my_instance_name", + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + chunkListStorage := mock.NewMockBlobAccess(ctrl) + + instanceName, err := digest.NewInstanceName(request.InstanceName) + require.NoError(t, err) + digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) + require.NoError(t, err) + blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + + chunkListStorage.EXPECT().Get(ctx, blobDigest).Return( + buffer.NewProtoBufferFromProto( + &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + }, + }, + buffer.UserProvided, + ), + ) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + _, err = contentAddressableStorageServer.SplitBlob(ctx, request) + require.NoError(t, err) +} + +func TestContentAddressableStorageServerSpliceBlob(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SpliceBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + ChunkDigests: []*remoteexecution.Digest{ + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + }, + InstanceName: "my_instance_name", + } + + contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + chunkListStorage := mock.NewMockBlobAccess(ctrl) + + instanceName, err := digest.NewInstanceName(request.InstanceName) + require.NoError(t, err) + digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) + require.NoError(t, err) + blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + + chunkListStorage.EXPECT().Put(ctx, blobDigest, buffer.NewProtoBufferFromProto(&remoteexecution.SplitBlobResponse{ + ChunkDigests: request.ChunkDigests, + }, buffer.UserProvided)).Return(nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + response, err := contentAddressableStorageServer.SpliceBlob(ctx, request) + require.NoError(t, err) + require.Equal(t, request.BlobDigest, response.BlobDigest) +} diff --git a/pkg/proto/configuration/bb_storage/bb_storage.pb.go b/pkg/proto/configuration/bb_storage/bb_storage.pb.go index fa4fb021e..fb4668e07 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.pb.go +++ b/pkg/proto/configuration/bb_storage/bb_storage.pb.go @@ -35,6 +35,7 @@ type ApplicationConfiguration struct { MaximumMessageSizeBytes int64 `protobuf:"varint,8,opt,name=maximum_message_size_bytes,json=maximumMessageSizeBytes,proto3" json:"maximum_message_size_bytes,omitempty"` Global *global.Configuration `protobuf:"bytes,9,opt,name=global,proto3" json:"global,omitempty"` ContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,17,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + ChunkListStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,22,opt,name=chunk_list_storage,json=chunkListStorage,proto3" json:"chunk_list_storage,omitempty"` ActionCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,18,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` IndirectContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,10,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` InitialSizeClassCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,11,opt,name=initial_size_class_cache,json=initialSizeClassCache,proto3" json:"initial_size_class_cache,omitempty"` @@ -111,6 +112,13 @@ func (x *ApplicationConfiguration) GetContentAddressableStorage() *ScannableBlob return nil } +func (x *ApplicationConfiguration) GetChunkListStorage() *ScannableBlobAccessConfiguration { + if x != nil { + return x.ChunkListStorage + } + return nil +} + func (x *ApplicationConfiguration) GetActionCache() *NonScannableBlobAccessConfiguration { if x != nil { return x.ActionCache @@ -292,7 +300,7 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_s const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc = "" + "\n" + - "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xbd\v\n" + + "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xb1\f\n" + "\x18ApplicationConfiguration\x12T\n" + "\fgrpc_servers\x18\x04 \x03(\v21.buildbarn.configuration.grpc.ServerConfigurationR\vgrpcServers\x12l\n" + "\n" + @@ -300,7 +308,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb "schedulers\x12;\n" + "\x1amaximum_message_size_bytes\x18\b \x01(\x03R\x17maximumMessageSizeBytes\x12E\n" + "\x06global\x18\t \x01(\v2-.buildbarn.configuration.global.ConfigurationR\x06global\x12\x84\x01\n" + - "\x1bcontent_addressable_storage\x18\x11 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x19contentAddressableStorage\x12j\n" + + "\x1bcontent_addressable_storage\x18\x11 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x19contentAddressableStorage\x12r\n" + + "\x12chunk_list_storage\x18\x16 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x10chunkListStorage\x12j\n" + "\faction_cache\x18\x12 \x01(\v2G.buildbarn.configuration.bb_storage.NonScannableBlobAccessConfigurationR\vactionCache\x12\x95\x01\n" + "$indirect_content_addressable_storage\x18\n" + " \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR!indirectContentAddressableStorage\x12\x80\x01\n" + @@ -353,26 +362,27 @@ var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_s 3, // 1: buildbarn.configuration.bb_storage.ApplicationConfiguration.schedulers:type_name -> buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry 5, // 2: buildbarn.configuration.bb_storage.ApplicationConfiguration.global:type_name -> buildbarn.configuration.global.Configuration 2, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 2, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 1, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 6, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 7, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value - 8, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration - 9, // 11: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 12: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 13: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 9, // 14: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 15: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 16: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 17: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 10, // 18: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration - 19, // [19:19] is the sub-list for method output_type - 19, // [19:19] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 2, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.chunk_list_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + 1, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 2, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + 1, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 1, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 6, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value + 8, // 11: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration + 9, // 12: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 6, // 13: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 6, // 14: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 9, // 15: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 6, // 16: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 6, // 17: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 6, // 18: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 10, // 19: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration + 20, // [20:20] is the sub-list for method output_type + 20, // [20:20] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { diff --git a/pkg/proto/configuration/bb_storage/bb_storage.proto b/pkg/proto/configuration/bb_storage/bb_storage.proto index 365c75527..55f431b9b 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.proto +++ b/pkg/proto/configuration/bb_storage/bb_storage.proto @@ -61,6 +61,10 @@ message ApplicationConfiguration { // Storage (CAS). ScannableBlobAccessConfiguration content_addressable_storage = 17; + // Optional: Blobstore configuration for the Content List Storage + // (CLS). + ScannableBlobAccessConfiguration chunk_list_storage = 22; + // Optional: Blobstore configuration for the Action Cache (AC). NonScannableBlobAccessConfiguration action_cache = 18; diff --git a/pkg/proto/configuration/blobstore/BUILD.bazel b/pkg/proto/configuration/blobstore/BUILD.bazel index a64c36145..83d0b07cc 100644 --- a/pkg/proto/configuration/blobstore/BUILD.bazel +++ b/pkg/proto/configuration/blobstore/BUILD.bazel @@ -14,6 +14,7 @@ proto_library( "//pkg/proto/configuration/digest:digest_proto", "//pkg/proto/configuration/grpc:grpc_proto", "//pkg/proto/configuration/http/client:client_proto", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto", "@googleapis//google/rpc:status_proto", "@protobuf//:duration_proto", "@protobuf//:empty_proto", @@ -33,6 +34,7 @@ go_proto_library( "//pkg/proto/configuration/digest", "//pkg/proto/configuration/grpc", "//pkg/proto/configuration/http/client", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", "@org_golang_google_genproto_googleapis_rpc//status", ], ) diff --git a/pkg/proto/configuration/blobstore/blobstore.pb.go b/pkg/proto/configuration/blobstore/blobstore.pb.go index dc3f8a4e0..237805e13 100644 --- a/pkg/proto/configuration/blobstore/blobstore.pb.go +++ b/pkg/proto/configuration/blobstore/blobstore.pb.go @@ -7,6 +7,7 @@ package blobstore import ( + v2 "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" blockdevice "github.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice" aws "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws" gcp "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp" @@ -106,6 +107,7 @@ type BlobAccessConfiguration struct { // *BlobAccessConfiguration_WithLabels // *BlobAccessConfiguration_Label // *BlobAccessConfiguration_DeadlineEnforcing + // *BlobAccessConfiguration_ChunkListValidating Backend isBlobAccessConfiguration_Backend `protobuf_oneof:"backend"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -319,6 +321,15 @@ func (x *BlobAccessConfiguration) GetDeadlineEnforcing() *DeadlineEnforcingBlobA return nil } +func (x *BlobAccessConfiguration) GetChunkListValidating() *ChunkListValidatingBlobAccessConfiguration { + if x != nil { + if x, ok := x.Backend.(*BlobAccessConfiguration_ChunkListValidating); ok { + return x.ChunkListValidating + } + } + return nil +} + type isBlobAccessConfiguration_Backend interface { isBlobAccessConfiguration_Backend() } @@ -399,6 +410,10 @@ type BlobAccessConfiguration_DeadlineEnforcing struct { DeadlineEnforcing *DeadlineEnforcingBlobAccess `protobuf:"bytes,28,opt,name=deadline_enforcing,json=deadlineEnforcing,proto3,oneof"` } +type BlobAccessConfiguration_ChunkListValidating struct { + ChunkListValidating *ChunkListValidatingBlobAccessConfiguration `protobuf:"bytes,29,opt,name=chunk_list_validating,json=chunkListValidating,proto3,oneof"` +} + func (*BlobAccessConfiguration_ReadCaching) isBlobAccessConfiguration_Backend() {} func (*BlobAccessConfiguration_Grpc) isBlobAccessConfiguration_Backend() {} @@ -437,6 +452,8 @@ func (*BlobAccessConfiguration_Label) isBlobAccessConfiguration_Backend() {} func (*BlobAccessConfiguration_DeadlineEnforcing) isBlobAccessConfiguration_Backend() {} +func (*BlobAccessConfiguration_ChunkListValidating) isBlobAccessConfiguration_Backend() {} + type ReadCachingBlobAccessConfiguration struct { state protoimpl.MessageState `protogen:"open.v1"` Slow *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=slow,proto3" json:"slow,omitempty"` @@ -628,6 +645,7 @@ type LocalBlobAccessConfiguration struct { BlocksBackend isLocalBlobAccessConfiguration_BlocksBackend `protobuf_oneof:"blocks_backend"` Persistent *LocalBlobAccessConfiguration_Persistent `protobuf:"bytes,13,opt,name=persistent,proto3" json:"persistent,omitempty"` HierarchicalInstanceNames bool `protobuf:"varint,14,opt,name=hierarchical_instance_names,json=hierarchicalInstanceNames,proto3" json:"hierarchical_instance_names,omitempty"` + ChunkingParameters *v2.RepMaxCdcParams `protobuf:"bytes,15,opt,name=chunking_parameters,json=chunkingParameters,proto3" json:"chunking_parameters,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -761,6 +779,13 @@ func (x *LocalBlobAccessConfiguration) GetHierarchicalInstanceNames() bool { return false } +func (x *LocalBlobAccessConfiguration) GetChunkingParameters() *v2.RepMaxCdcParams { + if x != nil { + return x.ChunkingParameters + } + return nil +} + type isLocalBlobAccessConfiguration_KeyLocationMapBackend interface { isLocalBlobAccessConfiguration_KeyLocationMapBackend() } @@ -1726,6 +1751,50 @@ func (x *GrpcBlobAccessConfiguration) GetEnableCompression() bool { return false } +type ChunkListValidatingBlobAccessConfiguration struct { + state protoimpl.MessageState `protogen:"open.v1"` + Backend *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChunkListValidatingBlobAccessConfiguration) Reset() { + *x = ChunkListValidatingBlobAccessConfiguration{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkListValidatingBlobAccessConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkListValidatingBlobAccessConfiguration) ProtoMessage() {} + +func (x *ChunkListValidatingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkListValidatingBlobAccessConfiguration.ProtoReflect.Descriptor instead. +func (*ChunkListValidatingBlobAccessConfiguration) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{21} +} + +func (x *ChunkListValidatingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { + if x != nil { + return x.Backend + } + return nil +} + type ShardingBlobAccessConfiguration_Shard struct { state protoimpl.MessageState `protogen:"open.v1"` Backend *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` @@ -1736,7 +1805,7 @@ type ShardingBlobAccessConfiguration_Shard struct { func (x *ShardingBlobAccessConfiguration_Shard) Reset() { *x = ShardingBlobAccessConfiguration_Shard{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1748,7 +1817,7 @@ func (x *ShardingBlobAccessConfiguration_Shard) String() string { func (*ShardingBlobAccessConfiguration_Shard) ProtoMessage() {} func (x *ShardingBlobAccessConfiguration_Shard) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1787,7 +1856,7 @@ type LocalBlobAccessConfiguration_KeyLocationMapInMemory struct { func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) Reset() { *x = LocalBlobAccessConfiguration_KeyLocationMapInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1799,7 +1868,7 @@ func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) String() string { func (*LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1831,7 +1900,7 @@ type LocalBlobAccessConfiguration_BlocksInMemory struct { func (x *LocalBlobAccessConfiguration_BlocksInMemory) Reset() { *x = LocalBlobAccessConfiguration_BlocksInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1843,7 +1912,7 @@ func (x *LocalBlobAccessConfiguration_BlocksInMemory) String() string { func (*LocalBlobAccessConfiguration_BlocksInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1877,7 +1946,7 @@ type LocalBlobAccessConfiguration_BlocksOnBlockDevice struct { func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) Reset() { *x = LocalBlobAccessConfiguration_BlocksOnBlockDevice{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1889,7 +1958,7 @@ func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) String() string { func (*LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1936,7 +2005,7 @@ type LocalBlobAccessConfiguration_Persistent struct { func (x *LocalBlobAccessConfiguration_Persistent) Reset() { *x = LocalBlobAccessConfiguration_Persistent{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1948,7 +2017,7 @@ func (x *LocalBlobAccessConfiguration_Persistent) String() string { func (*LocalBlobAccessConfiguration_Persistent) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_Persistent) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1982,10 +2051,10 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc = "" + "\n" + - "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xf3\x01\n" + + "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xf3\x01\n" + "\x16BlobstoreConfiguration\x12z\n" + "\x1bcontent_addressable_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x19contentAddressableStorage\x12]\n" + - "\faction_cache\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\vactionCache\"\xe3\x0f\n" + + "\faction_cache\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\vactionCache\"\xe9\x10\n" + "\x17BlobAccessConfiguration\x12j\n" + "\fread_caching\x18\x04 \x01(\v2E.buildbarn.configuration.blobstore.ReadCachingBlobAccessConfigurationH\x00R\vreadCaching\x12T\n" + "\x04grpc\x18\a \x01(\v2>.buildbarn.configuration.blobstore.GrpcBlobAccessConfigurationH\x00R\x04grpc\x12*\n" + @@ -2008,7 +2077,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\vwith_labels\x18\x1a \x01(\v2D.buildbarn.configuration.blobstore.WithLabelsBlobAccessConfigurationH\x00R\n" + "withLabels\x12\x16\n" + "\x05label\x18\x1b \x01(\tH\x00R\x05label\x12o\n" + - "\x12deadline_enforcing\x18\x1c \x01(\v2>.buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccessH\x00R\x11deadlineEnforcingB\t\n" + + "\x12deadline_enforcing\x18\x1c \x01(\v2>.buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccessH\x00R\x11deadlineEnforcing\x12\x83\x01\n" + + "\x15chunk_list_validating\x18\x1d \x01(\v2M.buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfigurationH\x00R\x13chunkListValidatingB\t\n" + "\abackendJ\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x05\x10\x06J\x04\b\x06\x10\aJ\x04\b\n" + "\x10\v\"\xa4\x02\n" + "\"ReadCachingBlobAccessConfiguration\x12N\n" + @@ -2029,7 +2099,7 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\tbackend_a\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\bbackendA\x12W\n" + "\tbackend_b\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\bbackendB\x12i\n" + "\x11replicator_a_to_b\x18\x03 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorAToB\x12i\n" + - "\x11replicator_b_to_a\x18\x04 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorBToA\"\xb6\f\n" + + "\x11replicator_b_to_a\x18\x04 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorBToA\"\x99\r\n" + "\x1cLocalBlobAccessConfiguration\x12\x94\x01\n" + "\x1akey_location_map_in_memory\x18\v \x01(\v2V.buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemoryH\x00R\x16keyLocationMapInMemory\x12{\n" + " key_location_map_on_block_device\x18\f \x01(\v22.buildbarn.configuration.blockdevice.ConfigurationH\x00R\x1bkeyLocationMapOnBlockDevice\x12O\n" + @@ -2046,7 +2116,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\n" + "persistent\x18\r \x01(\v2J.buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.PersistentR\n" + "persistent\x12>\n" + - "\x1bhierarchical_instance_names\x18\x0e \x01(\bR\x19hierarchicalInstanceNames\x1a2\n" + + "\x1bhierarchical_instance_names\x18\x0e \x01(\bR\x19hierarchicalInstanceNames\x12a\n" + + "\x13chunking_parameters\x18\x0f \x01(\v20.build.bazel.remote.execution.v2.RepMaxCdcParamsR\x12chunkingParameters\x1a2\n" + "\x16KeyLocationMapInMemory\x12\x18\n" + "\aentries\x18\x01 \x01(\x03R\aentries\x1a:\n" + "\x0eBlocksInMemory\x12(\n" + @@ -2127,7 +2198,9 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\abackend\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackend\"\x97\x01\n" + "\x1bGrpcBlobAccessConfiguration\x12I\n" + "\x06client\x18\x01 \x01(\v21.buildbarn.configuration.grpc.ClientConfigurationR\x06client\x12-\n" + - "\x12enable_compression\x18\x02 \x01(\bR\x11enableCompressionBCZAgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstoreb\x06proto3" + "\x12enable_compression\x18\x02 \x01(\bR\x11enableCompression\"\x82\x01\n" + + "*ChunkListValidatingBlobAccessConfiguration\x12T\n" + + "\abackend\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackendBCZAgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstoreb\x06proto3" var ( file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescOnce sync.Once @@ -2141,7 +2214,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescData } -var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 30) var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_goTypes = []any{ (*BlobstoreConfiguration)(nil), // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration (*BlobAccessConfiguration)(nil), // 1: buildbarn.configuration.blobstore.BlobAccessConfiguration @@ -2164,31 +2237,33 @@ var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs (*WithLabelsBlobAccessConfiguration)(nil), // 18: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration (*DeadlineEnforcingBlobAccess)(nil), // 19: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess (*GrpcBlobAccessConfiguration)(nil), // 20: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - (*ShardingBlobAccessConfiguration_Shard)(nil), // 21: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - nil, // 22: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 23: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 24: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - (*LocalBlobAccessConfiguration_Persistent)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - nil, // 27: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - nil, // 28: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - (*status.Status)(nil), // 29: google.rpc.Status - (*blockdevice.Configuration)(nil), // 30: buildbarn.configuration.blockdevice.Configuration - (*digest.ExistenceCacheConfiguration)(nil), // 31: buildbarn.configuration.digest.ExistenceCacheConfiguration - (*aws.SessionConfiguration)(nil), // 32: buildbarn.configuration.cloud.aws.SessionConfiguration - (*client.Configuration)(nil), // 33: buildbarn.configuration.http.client.Configuration - (*gcp.ClientOptionsConfiguration)(nil), // 34: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - (*emptypb.Empty)(nil), // 35: google.protobuf.Empty - (*grpc.ClientConfiguration)(nil), // 36: buildbarn.configuration.grpc.ClientConfiguration - (*durationpb.Duration)(nil), // 37: google.protobuf.Duration - (*timestamppb.Timestamp)(nil), // 38: google.protobuf.Timestamp + (*ChunkListValidatingBlobAccessConfiguration)(nil), // 21: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + (*ShardingBlobAccessConfiguration_Shard)(nil), // 22: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + nil, // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 24: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + (*LocalBlobAccessConfiguration_Persistent)(nil), // 27: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + nil, // 28: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + nil, // 29: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry + (*status.Status)(nil), // 30: google.rpc.Status + (*blockdevice.Configuration)(nil), // 31: buildbarn.configuration.blockdevice.Configuration + (*v2.RepMaxCdcParams)(nil), // 32: build.bazel.remote.execution.v2.RepMaxCdcParams + (*digest.ExistenceCacheConfiguration)(nil), // 33: buildbarn.configuration.digest.ExistenceCacheConfiguration + (*aws.SessionConfiguration)(nil), // 34: buildbarn.configuration.cloud.aws.SessionConfiguration + (*client.Configuration)(nil), // 35: buildbarn.configuration.http.client.Configuration + (*gcp.ClientOptionsConfiguration)(nil), // 36: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + (*emptypb.Empty)(nil), // 37: google.protobuf.Empty + (*grpc.ClientConfiguration)(nil), // 38: buildbarn.configuration.grpc.ClientConfiguration + (*durationpb.Duration)(nil), // 39: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 40: google.protobuf.Timestamp } var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_depIdxs = []int32{ 1, // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration 1, // 1: buildbarn.configuration.blobstore.BlobstoreConfiguration.action_cache:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration 2, // 2: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_caching:type_name -> buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration 20, // 3: buildbarn.configuration.blobstore.BlobAccessConfiguration.grpc:type_name -> buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - 29, // 4: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status + 30, // 4: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status 3, // 5: buildbarn.configuration.blobstore.BlobAccessConfiguration.sharding:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration 4, // 6: buildbarn.configuration.blobstore.BlobAccessConfiguration.mirrored:type_name -> buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration 5, // 7: buildbarn.configuration.blobstore.BlobAccessConfiguration.local:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration @@ -2204,66 +2279,69 @@ var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs 17, // 17: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_writing:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration 18, // 18: buildbarn.configuration.blobstore.BlobAccessConfiguration.with_labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration 19, // 19: buildbarn.configuration.blobstore.BlobAccessConfiguration.deadline_enforcing:type_name -> buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess - 1, // 20: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 21: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 22: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 22, // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - 1, // 24: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 25: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 26: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 10, // 27: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 23, // 28: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - 30, // 29: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration - 24, // 30: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - 25, // 31: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - 26, // 32: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - 1, // 33: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 31, // 34: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 35: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 36: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 37: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 38: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 1, // 39: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 32, // 40: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration - 33, // 41: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration - 34, // 42: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - 1, // 43: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 35, // 44: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty - 36, // 45: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 11, // 46: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration - 35, // 47: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty - 10, // 48: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 12, // 49: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration - 10, // 50: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 31, // 51: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 10, // 52: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 27, // 53: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - 1, // 54: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 55: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 37, // 56: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration - 37, // 57: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration - 38, // 58: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp - 1, // 59: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 60: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 37, // 61: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration - 31, // 62: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 63: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 28, // 64: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - 37, // 65: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration - 1, // 66: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 36, // 67: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 1, // 68: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 21, // 69: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - 30, // 70: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration - 31, // 71: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 37, // 72: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration - 14, // 73: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration - 1, // 74: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 75, // [75:75] is the sub-list for method output_type - 75, // [75:75] is the sub-list for method input_type - 75, // [75:75] is the sub-list for extension type_name - 75, // [75:75] is the sub-list for extension extendee - 0, // [0:75] is the sub-list for field type_name + 21, // 20: buildbarn.configuration.blobstore.BlobAccessConfiguration.chunk_list_validating:type_name -> buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + 1, // 21: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 22: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 10, // 23: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 23, // 24: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + 1, // 25: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 26: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 10, // 27: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 10, // 28: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 24, // 29: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + 31, // 30: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration + 25, // 31: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + 26, // 32: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + 27, // 33: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + 32, // 34: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.chunking_parameters:type_name -> build.bazel.remote.execution.v2.RepMaxCdcParams + 1, // 35: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 33, // 36: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 1, // 37: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 38: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 39: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 10, // 40: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 1, // 41: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 34, // 42: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration + 35, // 43: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration + 36, // 44: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + 1, // 45: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 37, // 46: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty + 38, // 47: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 11, // 48: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration + 37, // 49: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty + 10, // 50: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 12, // 51: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration + 10, // 52: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 33, // 53: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 10, // 54: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 28, // 55: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + 1, // 56: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 57: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 39, // 58: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration + 39, // 59: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration + 40, // 60: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp + 1, // 61: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 62: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 39, // 63: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration + 33, // 64: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 1, // 65: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 29, // 66: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry + 39, // 67: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration + 1, // 68: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 38, // 69: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 1, // 70: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 71: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 22, // 72: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + 31, // 73: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration + 33, // 74: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 39, // 75: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration + 14, // 76: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration + 1, // 77: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 78, // [78:78] is the sub-list for method output_type + 78, // [78:78] is the sub-list for method input_type + 78, // [78:78] is the sub-list for extension type_name + 78, // [78:78] is the sub-list for extension extendee + 0, // [0:78] is the sub-list for field type_name } func init() { @@ -2293,6 +2371,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob (*BlobAccessConfiguration_WithLabels)(nil), (*BlobAccessConfiguration_Label)(nil), (*BlobAccessConfiguration_DeadlineEnforcing)(nil), + (*BlobAccessConfiguration_ChunkListValidating)(nil), } file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5].OneofWrappers = []any{ (*LocalBlobAccessConfiguration_KeyLocationMapInMemory_)(nil), @@ -2314,7 +2393,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc)), NumEnums: 0, - NumMessages: 29, + NumMessages: 30, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/configuration/blobstore/blobstore.proto b/pkg/proto/configuration/blobstore/blobstore.proto index 68b226711..917df5514 100644 --- a/pkg/proto/configuration/blobstore/blobstore.proto +++ b/pkg/proto/configuration/blobstore/blobstore.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package buildbarn.configuration.blobstore; +import "build/bazel/remote/execution/v2/remote_execution.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto"; @@ -199,6 +200,29 @@ message BlobAccessConfiguration { // value. When gRPC calls are timed out a `DEADLINE_EXCEEDED` error // code will be returned. DeadlineEnforcingBlobAccess deadline_enforcing = 28; + + // Validate that chunk list requests fulfill the api specification. + // Chunk List Storage (CLS) is used to implement the SplitBlob and + // SpliceBlob methods from the REv2 api. The protocol has several + // demands on the state of the Content Addressable Storage (CAS) + // after those methods have been called. + // + // SplitBlob requires that that the blob as well as all chunks of + // the blob are available in the CAS, they have their lifetime + // renewed and that the resulting chunk list composes into the blob. + // This is validated by the Get path by splitting the Blob on demand + // if required. + // + // SpliceBlob requires that the blob as well as all chunks of the + // blob are present in storage, they have their lifetime renewed and + // that the supplied chunk list composes into the blob. Notably it + // does not require the chunks to follow any particular chunking + // algorithm but our implementation ensures that after any call a + // proper rep max cdc chunk list is verified even if the caller + // supplied a different chunk list. + // + // This decorator must be placed on the CLS. + ChunkListValidatingBlobAccessConfiguration chunk_list_validating = 29; } // Was 'redis'. Instead of using Redis, one may run a separate @@ -642,6 +666,18 @@ message LocalBlobAccessConfiguration { // level, e.g., on top of CompletenessCheckingBlobAccess. This can be // achieved by using HierarchicalInstanceNamesBlobAccess. bool hierarchical_instance_names = 14; + + // The chunking parameters advertised via GetCapabilities, setting + // this announces support for the SplitBlob and SpliceBlob api calls. + // + // Adding the chunking parameters does not make the underlying + // blobstore enforce the chunking parameters or implement the split + // and splice blob. There must be a configured Chunk List Storage with + // an outer ChunkValidatingBlobAccess that has view of the entire + // Content Addressable Storage (CAS). + // + // This option is only supported for the CLS. + build.bazel.remote.execution.v2.RepMaxCdcParams chunking_parameters = 15; } message ExistenceCachingBlobAccessConfiguration { @@ -937,3 +973,8 @@ message GrpcBlobAccessConfiguration { // types (AC, ICAS, etc.). bool enable_compression = 2; } + +message ChunkListValidatingBlobAccessConfiguration { + // The backend to which validated operations are delegated. + BlobAccessConfiguration backend = 1; +}