diff --git a/MODULE.bazel b/MODULE.bazel index 3dc48bb5d..8e1d6cfc5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,6 +49,7 @@ use_repo( "com_github_aws_aws_sdk_go_v2_service_sts", "com_github_bazelbuild_buildtools", "com_github_bazelbuild_remote_apis", + "com_github_buildbarn_go_cdc", "com_github_buildbarn_go_sha256tree", "com_github_fxtlabs_primes", "com_github_go_jose_go_jose_v3", diff --git a/cmd/bb_copy/main.go b/cmd/bb_copy/main.go index 54e83489e..6d891a639 100644 --- a/cmd/bb_copy/main.go +++ b/cmd/bb_copy/main.go @@ -42,7 +42,7 @@ func main() { grpcClientFactory := grpc.NewBaseClientFactory(grpc.BaseClientDialer, nil, nil, nil) - blobAccessCreator := blobstore_configuration.NewCASBlobAccessCreator( + blobAccessCreator := blobstore_configuration.NewCSBlobAccessCreator( grpcClientFactory, int(configuration.MaximumMessageSizeBytes), bb_zstd.NewPoolFromConfiguration(nil), @@ -68,7 +68,7 @@ func main() { configuration.Replicator, source.BlobAccess, sink, - blobstore_configuration.NewCASBlobReplicatorCreator(grpcClientFactory), + blobstore_configuration.NewCSBlobReplicatorCreator(grpcClientFactory), ) if err != nil { return util.StatusWrap(err, "Failed to create replicator") diff --git a/cmd/bb_replicator/main.go b/cmd/bb_replicator/main.go index 8f21a8020..8e5b9f994 100644 --- a/cmd/bb_replicator/main.go +++ b/cmd/bb_replicator/main.go @@ -33,7 +33,7 @@ func main() { return util.StatusWrap(err, "Failed to apply global configuration options") } - blobAccessCreator := blobstore_configuration.NewCASBlobAccessCreator( + blobAccessCreator := blobstore_configuration.NewCSBlobAccessCreator( grpcClientFactory, int(configuration.MaximumMessageSizeBytes), bb_zstd.NewPoolFromConfiguration(nil), @@ -59,7 +59,7 @@ func main() { configuration.Replicator, source.BlobAccess, sink, - blobstore_configuration.NewCASBlobReplicatorCreator(grpcClientFactory), + blobstore_configuration.NewCSBlobReplicatorCreator(grpcClientFactory), ) if err != nil { return util.StatusWrap(err, "Failed to create replicator") diff --git a/cmd/bb_storage/BUILD.bazel b/cmd/bb_storage/BUILD.bazel index b9d6bc741..58b47e035 100644 --- a/cmd/bb_storage/BUILD.bazel +++ b/cmd/bb_storage/BUILD.bazel @@ -10,10 +10,12 @@ go_library( "//pkg/auth", "//pkg/auth/configuration", "//pkg/blobstore", + "//pkg/blobstore/cdc", "//pkg/blobstore/configuration", "//pkg/blobstore/grpcservers", "//pkg/builder", "//pkg/capabilities", + "//pkg/cas", "//pkg/global", "//pkg/grpc", "//pkg/program", diff --git a/cmd/bb_storage/main.go b/cmd/bb_storage/main.go index 192fcb5e4..9ef2a3438 100644 --- a/cmd/bb_storage/main.go +++ b/cmd/bb_storage/main.go @@ -9,10 +9,12 @@ import ( "github.com/buildbarn/bb-storage/pkg/auth" auth_configuration "github.com/buildbarn/bb-storage/pkg/auth/configuration" "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/builder" "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/global" bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc" "github.com/buildbarn/bb-storage/pkg/program" @@ -54,34 +56,58 @@ func main() { var cacheCapabilitiesAuthorizers []auth.Authorizer // Content Addressable Storage (CAS). - var contentAddressableStorageInfo *blobstore_configuration.BlobAccessInfo - var contentAddressableStorage blobstore.BlobAccess - if configuration.ContentAddressableStorage != nil { - info, authorizedBackend, allAuthorizers, err := newScannableBlobAccess( + var contentAddressableStorage cas.ContentAddressableStorage + var authorizedContentAddressableStorage cas.ContentAddressableStorage + if configuration.ContentAddressableStorageServer != nil { + var chunkStorage, chunkListStorage blobstore.BlobAccess + var cdcParametersFetcher cdc.ParametersFetcher + contentAddressableStorage, chunkStorage, chunkListStorage, _, cdcParametersFetcher, err = blobstore_configuration.NewCASFromConfiguration( dependenciesGroup, - configuration.ContentAddressableStorage, - blobstore_configuration.NewCASBlobAccessCreator( - grpcClientFactory, - int(configuration.MaximumMessageSizeBytes), - zstdPool, - ), + configuration.ContentAddressableStorageServer.ContentAddressableStorage, grpcClientFactory, + int(configuration.MaximumMessageSizeBytes), + zstdPool, ) if err != nil { return util.StatusWrap(err, "Failed to create Content Addressable Storage") } + + // Create authorizers. + getAuthorizer, err := auth_configuration.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(configuration.ContentAddressableStorageServer.GetAuthorizer, dependenciesGroup, grpcClientFactory) + if err != nil { + return util.StatusWrap(err, "Failed to create Get() authorizer for Content Addressable Storage") + } + putAuthorizer, err := auth_configuration.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(configuration.ContentAddressableStorageServer.PutAuthorizer, dependenciesGroup, grpcClientFactory) + if err != nil { + return util.StatusWrap(err, "Failed to create Put() authorizer for Content Addressable Storage") + } + findMissingAuthorizer, err := auth_configuration.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(configuration.ContentAddressableStorageServer.FindMissingAuthorizer, dependenciesGroup, grpcClientFactory) + if err != nil { + return util.StatusWrap(err, "Failed to create FindMissing() authorizer for Content Addressable Storage") + } + + // Create authorized versions of the backends. + authorizedChunkStorage := blobstore.NewAuthorizingBlobAccess(chunkStorage, getAuthorizer, putAuthorizer, findMissingAuthorizer) + authorizedChunkListStorage := blobstore.NewAuthorizingBlobAccess(chunkListStorage, getAuthorizer, putAuthorizer, findMissingAuthorizer) + authorizedChunkListFetcher := blobstore.NewBlobAccessChunkListFetcher(authorizedChunkListStorage, int(configuration.MaximumMessageSizeBytes)) + authorizedContentAddressableStorage = cas.NewContentAddressableStorage( + authorizedChunkStorage, + authorizedChunkListStorage, + authorizedChunkListFetcher, + cdcParametersFetcher, + contentAddressableStorage.GetDigestKeyFormat(), + ) + // Create the Chunk Storage (CS). cacheCapabilitiesProviders = append( cacheCapabilitiesProviders, - info.BlobAccess, + chunkStorage, capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ CacheCapabilities: &remoteexecution.CacheCapabilities{ SupportedCompressors: configuration.SupportedCompressors, }, }), ) - cacheCapabilitiesAuthorizers = append(cacheCapabilitiesAuthorizers, allAuthorizers...) - contentAddressableStorageInfo = &info - contentAddressableStorage = authorizedBackend + cacheCapabilitiesAuthorizers = append(cacheCapabilitiesAuthorizers, getAuthorizer, putAuthorizer, findMissingAuthorizer) } // Action Cache (AC). @@ -91,7 +117,7 @@ func main() { dependenciesGroup, configuration.ActionCache, blobstore_configuration.NewACBlobAccessCreator( - contentAddressableStorageInfo, + contentAddressableStorage, grpcClientFactory, int(configuration.MaximumMessageSizeBytes), ), @@ -192,19 +218,19 @@ func main() { if err := bb_grpc.NewServersFromConfigurationAndServe( configuration.GrpcServers, func(s grpc.ServiceRegistrar) { - if contentAddressableStorage != nil { + if authorizedContentAddressableStorage != nil { + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer( + authorizedContentAddressableStorage, + configuration.MaximumMessageSizeBytes, + ) remoteexecution.RegisterContentAddressableStorageServer( s, - grpcservers.NewContentAddressableStorageServer( - contentAddressableStorage, - configuration.MaximumMessageSizeBytes, - ), + contentAddressableStorageServer, ) bytestream.RegisterByteStreamServer( s, grpcservers.NewByteStreamServer( - contentAddressableStorage, - 1<<16, + authorizedContentAddressableStorage, zstdPool, ), ) @@ -214,7 +240,10 @@ func main() { s, grpcservers.NewActionCacheServer( actionCache, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*remoteexecution.ActionResult]( + actionCache, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } @@ -223,7 +252,10 @@ func main() { s, grpcservers.NewIndirectContentAddressableStorageServer( indirectContentAddressableStorage, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*icas.Reference]( + indirectContentAddressableStorage, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } @@ -232,7 +264,10 @@ func main() { s, grpcservers.NewInitialSizeClassCacheServer( initialSizeClassCache, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*iscc.PreviousExecutionStats]( + initialSizeClassCache, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } @@ -241,7 +276,10 @@ func main() { s, grpcservers.NewFileSystemAccessCacheServer( fileSystemAccessCache, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*fsac.FileSystemAccessProfile]( + fileSystemAccessCache, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } diff --git a/go.mod b/go.mod index 082763b1a..4fd9fa898 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/bazelbuild/buildtools v0.0.0-20260527135131-3b47c424ecf5 github.com/bazelbuild/remote-apis v0.0.0-20260331222004-becdd8f9ff81 github.com/bazelbuild/rules_go v0.62.0 + github.com/buildbarn/go-cdc v0.0.9 github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449 github.com/go-jose/go-jose/v3 v3.0.5 diff --git a/go.sum b/go.sum index d097ebee7..545e90b57 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/bazelbuild/rules_go v0.62.0/go.mod h1:6YghDRf6l3FSiAwncK+Ww9jE1naoQzN github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/buildbarn/go-cdc v0.0.9 h1:bWfgn92ed8Oo2zZKJdMAfB0APGz7Q8zvnqUn3hPuihM= +github.com/buildbarn/go-cdc v0.0.9/go.mod h1:KUMqSMvoRlby3uak9aKIvgz3KgNqwm2CMUoVX1EDr8k= github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b h1:IKUxixGBm9UxobU7c248z0BF0ojG19uoSLz8MFZM/KA= github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b/go.mod h1:e7g3/yWApcg+PpDqd4eQEEV8pexQmfCgK3frP+1Wuvk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/internal/mock/BUILD.bazel b/internal/mock/BUILD.bazel index 87d7fda65..6e130b604 100644 --- a/internal/mock/BUILD.bazel +++ b/internal/mock/BUILD.bazel @@ -152,6 +152,30 @@ gomock( package = "mock", ) +gomock( + name = "cas", + out = "cas.go", + interfaces = [ + "ContentAddressableStorage", + ], + library = "//pkg/cas", + mockgen_model_library = "@org_uber_go_mock//mockgen/model", + mockgen_tool = "@org_uber_go_mock//mockgen", + package = "mock", +) + +gomock( + name = "cdc", + out = "cdc.go", + interfaces = [ + "ParametersFetcher", + ], + library = "//pkg/blobstore/cdc", + mockgen_model_library = "@org_uber_go_mock//mockgen/model", + mockgen_tool = "@org_uber_go_mock//mockgen", + package = "mock", +) + gomock( name = "clock", out = "clock.go", @@ -320,6 +344,16 @@ gomock( package = "mock", ) +gomock( + name = "storage", + out = "storage.go", + interfaces = ["MessageReader"], + library = "//pkg/storage", + mockgen_tool = "@org_uber_go_mock//mockgen", + package = "mock", + source = "//pkg/storage:message_reader.go", +) + gomock( name = "trace", out = "trace.go", @@ -367,6 +401,8 @@ go_library( "buffer.go", "builder.go", "capabilities.go", + "cas.go", + "cdc.go", "clock.go", "cloud_aws.go", "cloud_gcp.go", @@ -380,6 +416,7 @@ go_library( "prometheus.go", "random.go", "remoteexecution.go", + "storage.go", "trace.go", "trace_wrap.go", "util.go", @@ -391,10 +428,13 @@ go_library( "//pkg/auth", "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/blobstore/local", "//pkg/blobstore/sharding", "//pkg/blobstore/slicing", "//pkg/builder", + "//pkg/cas", "//pkg/clock", "//pkg/cloud/gcp", "//pkg/digest", @@ -416,6 +456,7 @@ go_library( "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//metadata", "@org_golang_google_protobuf//encoding/protowire", + "@org_golang_google_protobuf//proto", "@org_uber_go_mock//gomock", ], ) diff --git a/pkg/blobstore/BUILD.bazel b/pkg/blobstore/BUILD.bazel index 45d054538..f1a04c7f9 100644 --- a/pkg/blobstore/BUILD.bazel +++ b/pkg/blobstore/BUILD.bazel @@ -8,7 +8,10 @@ go_library( "action_result_timestamp_injecting_blob_access.go", "authorizing_blob_access.go", "blob_access.go", + "blob_access_message_reader.go", "cas_read_buffer_factory.go", + "chunk_list_fetcher.go", + "cls_read_buffer_factory.go", "deadline_enforcing_blob_access.go", "demultiplexing_blob_access.go", "empty_blob_injecting_blob_access.go", @@ -21,7 +24,6 @@ go_library( "metrics_blob_access.go", "read_buffer_factory.go", "read_canarying_blob_access.go", - "reference_expanding_blob_access.go", "validation_caching_read_buffer_factory.go", "visit_topologically_sorted_tree.go", "zip_reading_blob_access.go", @@ -32,21 +34,19 @@ go_library( deps = [ "//pkg/auth", "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", "//pkg/blobstore/slicing", "//pkg/capabilities", "//pkg/clock", - "//pkg/cloud/aws", - "//pkg/cloud/gcp", "//pkg/digest", "//pkg/eviction", + "//pkg/proto/blobstore/chunklist", "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", + "//pkg/storage", "//pkg/util", - "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", - "@com_github_aws_aws_sdk_go_v2//aws", - "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", "@com_github_prometheus_client_golang//prometheus", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", @@ -67,7 +67,6 @@ go_test( "existence_caching_blob_access_test.go", "hierarchical_instance_names_blob_access_test.go", "read_canarying_blob_access_test.go", - "reference_expanding_blob_access_test.go", "validation_caching_read_buffer_factory_test.go", "visit_topologically_sorted_tree_test.go", "zip_reading_blob_access_test.go", @@ -79,16 +78,10 @@ go_test( "//pkg/blobstore/buffer", "//pkg/digest", "//pkg/eviction", - "//pkg/proto/icas", "//pkg/testutil", "//pkg/util", - "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", "@bazel_remote_apis//build/bazel/semver:semver_go_proto", - "@com_github_aws_aws_sdk_go_v2//aws", - "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", - "@com_github_aws_aws_sdk_go_v2_service_s3//types", - "@com_github_klauspost_compress//zstd", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", diff --git a/pkg/blobstore/authorizing_blob_access.go b/pkg/blobstore/authorizing_blob_access.go index 25898bdd4..cdb3b480d 100644 --- a/pkg/blobstore/authorizing_blob_access.go +++ b/pkg/blobstore/authorizing_blob_access.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/auth" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -38,13 +37,6 @@ func (ba *authorizingBlobAccess) Get(ctx context.Context, d digest.Digest) buffe return ba.BlobAccess.Get(ctx, d) } -func (ba *authorizingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if err := auth.AuthorizeSingleInstanceName(ctx, ba.getAuthorizer, parentDigest.GetInstanceName()); err != nil { - return buffer.NewBufferFromError(util.StatusWrap(err, "Authorization")) - } - return ba.BlobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer) -} - func (ba *authorizingBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { if err := auth.AuthorizeSingleInstanceName(ctx, ba.putAuthorizer, d.GetInstanceName()); err != nil { return util.StatusWrap(err, "Authorization") diff --git a/pkg/blobstore/authorizing_blob_access_test.go b/pkg/blobstore/authorizing_blob_access_test.go index a13d994dd..6323daec5 100644 --- a/pkg/blobstore/authorizing_blob_access_test.go +++ b/pkg/blobstore/authorizing_blob_access_test.go @@ -54,24 +54,6 @@ func TestAuthorizingBlobAccess(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.PermissionDenied, "Authorization: You shall not pass"), err) }) - t.Run("GetFromComposite-Allowed", func(t *testing.T) { - getAuthorizer.EXPECT().Authorize(ctx, beepSlice).Return([]error{nil}) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - baseBlobAccess.EXPECT().GetFromComposite(ctx, d, d2, blobSlicer).Return(wantBuf) - - gotBuf, err := ba.GetFromComposite(ctx, d, d2, blobSlicer).ToByteSlice(30) - require.NoError(t, err) - require.Equal(t, wantBytes, gotBuf) - }) - - t.Run("GetFromComposite-Denied", func(t *testing.T) { - blobSlicer := mock.NewMockBlobSlicer(ctrl) - getAuthorizer.EXPECT().Authorize(ctx, beepSlice).Return([]error{status.Error(codes.PermissionDenied, "You shall not pass")}) - - _, err := ba.GetFromComposite(ctx, d, d2, blobSlicer).ToByteSlice(30) - testutil.RequireEqualStatus(t, status.Error(codes.PermissionDenied, "Authorization: You shall not pass"), err) - }) - t.Run("Put-Allowed", func(t *testing.T) { putAuthorizer.EXPECT().Authorize(ctx, beepSlice).Return([]error{nil}) baseBlobAccess.EXPECT().Put(ctx, d, wantBuf).Return(nil) diff --git a/pkg/blobstore/blob_access.go b/pkg/blobstore/blob_access.go index 09d6ba963..2f2ed19dc 100644 --- a/pkg/blobstore/blob_access.go +++ b/pkg/blobstore/blob_access.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -16,7 +15,6 @@ type BlobAccess interface { capabilities.Provider Get(ctx context.Context, digest digest.Digest) buffer.Buffer - GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) } diff --git a/pkg/blobstore/blob_access_message_reader.go b/pkg/blobstore/blob_access_message_reader.go new file mode 100644 index 000000000..6c697b1d1 --- /dev/null +++ b/pkg/blobstore/blob_access_message_reader.go @@ -0,0 +1,32 @@ +package blobstore + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/storage" + "google.golang.org/protobuf/proto" +) + +type blobAccessMessageReader[T proto.Message] struct { + blobAccess BlobAccess + maximumMessageSizeBytes int +} + +// NewBlobAccessMessageReader creates a storage.MessageReader that reads +// a message from a BlobAccess up to maximumMessageSizeBytes. +func NewBlobAccessMessageReader[T proto.Message](blobAccess BlobAccess, maximumMessageSizeBytes int) storage.MessageReader[T] { + return blobAccessMessageReader[T]{ + blobAccess: blobAccess, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (mr blobAccessMessageReader[T]) ReadMessage(ctx context.Context, d digest.Digest, msg T) (T, error) { + var zero T + ret, err := mr.blobAccess.Get(ctx, d).ToProto(msg, mr.maximumMessageSizeBytes) + if err != nil { + return zero, err + } + return ret.(T), nil +} diff --git a/pkg/blobstore/cas_read_buffer_factory.go b/pkg/blobstore/cas_read_buffer_factory.go index 2832352b8..b80cf0869 100644 --- a/pkg/blobstore/cas_read_buffer_factory.go +++ b/pkg/blobstore/cas_read_buffer_factory.go @@ -1,15 +1,12 @@ package blobstore import ( - "context" "io" - "math" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/proto" ) type casReadBufferFactory struct{} @@ -30,33 +27,6 @@ func (f casReadBufferFactory) NewBufferFromReaderAt(digest digest.Digest, r buff // stored in the Content Addressable Storage (CAS). var CASReadBufferFactory ReadBufferFactory = casReadBufferFactory{} -// CASPutProto is a helper function for storing Protobuf messages in the -// Content Addressable Storage (CAS). It computes the digest of the -// message and stores it under that key. The digest is then returned, so -// that the object may be referenced. -func CASPutProto(ctx context.Context, blobAccess BlobAccess, message proto.Message, digestFunction digest.Function) (digest.Digest, error) { - b := buffer.NewProtoBufferFromProto(message, buffer.UserProvided) - sizeBytes, err := b.GetSizeBytes() - if err != nil { - b.Discard() - return digest.BadDigest, err - } - - // Compute new digest of data. - digestGenerator := digestFunction.NewGenerator(sizeBytes) - bDigest, bPut := b.CloneCopy(math.MaxInt) - if err := bDigest.IntoWriter(digestGenerator); err != nil { - bPut.Discard() - return digest.BadDigest, err - } - blobDigest := digestGenerator.Sum() - - if err := blobAccess.Put(ctx, blobDigest, bPut); err != nil { - return digest.BadDigest, err - } - return blobDigest, nil -} - // The Protobuf field numbers of the REv2 Tree's "root" and "children" // fields. These are used in combination with util.VisitProtoBytesFields() // to be able to process REv2 Tree objects in a streaming manner. diff --git a/pkg/blobstore/cdc/BUILD.bazel b/pkg/blobstore/cdc/BUILD.bazel new file mode 100644 index 000000000..241c99b54 --- /dev/null +++ b/pkg/blobstore/cdc/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "cdc", + srcs = [ + "bypass.go", + "caching_parameters_fetcher.go", + "chunker.go", + "parameters.go", + "reader_chunker.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/cdc", + visibility = ["//visibility:public"], + deps = [ + "//pkg/capabilities", + "//pkg/digest", + "//pkg/ttlcache", + "//pkg/util", + "@com_github_buildbarn_go_cdc//:go-cdc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) + +go_test( + name = "cdc_test", + srcs = ["reader_chunker_test.go"], + deps = [ + ":cdc", + "//pkg/blobstore/buffer", + "//pkg/digest", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/blobstore/cdc/bypass.go b/pkg/blobstore/cdc/bypass.go new file mode 100644 index 000000000..1564d4e85 --- /dev/null +++ b/pkg/blobstore/cdc/bypass.go @@ -0,0 +1,22 @@ +package cdc + +import "context" + +type chunkListValidationBypassKey struct{} + +// NewContextWithChunkListValidationBypass creates a derived context +// that signals downstream storage layers that the chunk list has +// already been validated (or was freshly generated) and does not need +// expensive re-validation. +func NewContextWithChunkListValidationBypass(ctx context.Context) context.Context { + return context.WithValue(ctx, chunkListValidationBypassKey{}, chunkListValidationBypassKey{}) +} + +// ChunkListValidationBypassed checks if the provided context contains +// the bypass signal. +func ChunkListValidationBypassed(ctx context.Context) bool { + if value := ctx.Value(chunkListValidationBypassKey{}); value != nil { + return true + } + return false +} diff --git a/pkg/blobstore/cdc/caching_parameters_fetcher.go b/pkg/blobstore/cdc/caching_parameters_fetcher.go new file mode 100644 index 000000000..98c7b46c6 --- /dev/null +++ b/pkg/blobstore/cdc/caching_parameters_fetcher.go @@ -0,0 +1,33 @@ +package cdc + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/ttlcache" +) + +type cachingParametersFetcher struct { + ParametersFetcher + cache *ttlcache.TTLCache[digest.InstanceName, Parameters] +} + +// NewCachingParametersFetcher decorates a ParametersFetcher with caching. +func NewCachingParametersFetcher(base ParametersFetcher, ttlCache *ttlcache.TTLCache[digest.InstanceName, Parameters]) ParametersFetcher { + return &cachingParametersFetcher{ + ParametersFetcher: base, + cache: ttlCache, + } +} + +func (f *cachingParametersFetcher) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) { + if v, ok := f.cache.Get(instanceName); ok { + return v, nil + } + params, err := f.ParametersFetcher.FetchCDCParameters(ctx, instanceName) + if err != nil { + return Parameters{}, err + } + f.cache.Put(instanceName, params) + return params, nil +} diff --git a/pkg/blobstore/cdc/chunker.go b/pkg/blobstore/cdc/chunker.go new file mode 100644 index 000000000..ac02682db --- /dev/null +++ b/pkg/blobstore/cdc/chunker.go @@ -0,0 +1,16 @@ +package cdc + +import ( + "github.com/buildbarn/bb-storage/pkg/digest" +) + +// Chunk is a struct of raw binary data with its digest. +type Chunk struct { + Digest digest.Digest + Data []byte +} + +// Chunker is an interface that provides a sequence of chunks. +type Chunker interface { + NextChunk() (Chunk, error) +} diff --git a/pkg/blobstore/cdc/parameters.go b/pkg/blobstore/cdc/parameters.go new file mode 100644 index 000000000..a84cd3515 --- /dev/null +++ b/pkg/blobstore/cdc/parameters.go @@ -0,0 +1,59 @@ +package cdc + +import ( + "context" + "math" + + "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Parameters are the parameters by which to perform Content Defined +// Chunking (CDC). +type Parameters struct { + MinChunkSizeBytes int64 + HorizonSizeBytes int64 +} + +// ParametersFetcher retrieves the Content Defined Chunking (CDC) +// parameters that govern how blobs are decomposed into chunks. +type ParametersFetcher interface { + FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) +} + +type capabilitiesParametersFetcher struct { + provider capabilities.Provider +} + +// NewCapabilitiesParametersFetcher creates a ParametersFetcher that +// obtains the CDC parameters by calling GetCapabilities() on the +// provided capabilities.Provider. +func NewCapabilitiesParametersFetcher(provider capabilities.Provider) ParametersFetcher { + return &capabilitiesParametersFetcher{ + provider: provider, + } +} + +func (f *capabilitiesParametersFetcher) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) { + capabilities, err := f.provider.GetCapabilities(ctx, instanceName) + if err != nil { + return Parameters{}, util.StatusWrap(err, "Unable to GetCapabilities to determine chunking parameters") + } + + params := capabilities.CacheCapabilities.GetRepMaxCdcParams() + if params == nil { + return Parameters{}, status.Error(codes.Unimplemented, "This backend only supports storage backends with RepMaxCDC support") + } + if params.MinChunkSizeBytes < 64 { + return Parameters{}, status.Errorf(codes.Internal, "RepMaxCDC minimum chunk size was %d bytes but a minimum of 64 bytes is required", params.MinChunkSizeBytes) + } + if params.MinChunkSizeBytes > math.MaxInt64 || params.HorizonSizeBytes > math.MaxInt64 { + return Parameters{}, status.Error(codes.Internal, "CDC Parameters could not be represented as int64") + } + + return Parameters{MinChunkSizeBytes: int64(params.MinChunkSizeBytes), HorizonSizeBytes: int64(params.HorizonSizeBytes)}, nil +} diff --git a/pkg/blobstore/cdc/reader_chunker.go b/pkg/blobstore/cdc/reader_chunker.go new file mode 100644 index 000000000..ad7717d72 --- /dev/null +++ b/pkg/blobstore/cdc/reader_chunker.go @@ -0,0 +1,51 @@ +package cdc + +import ( + "bufio" + "io" + + "github.com/buildbarn/bb-storage/pkg/digest" + cdc "github.com/buildbarn/go-cdc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type readerChunker struct { + cdcChunker cdc.ContentDefinedChunker + digestFunction digest.Function +} + +func (c *readerChunker) NextChunk() (Chunk, error) { + chunk, err := c.cdcChunker.ReadNextChunk() + if err != nil { + return Chunk{}, err + } + + digestGenerator := c.digestFunction.NewGenerator(int64(len(chunk))) + if _, err := digestGenerator.Write(chunk); err != nil { + return Chunk{}, status.Error(codes.Internal, "Could not compute digest of chunk") + } + chunkDigest := digestGenerator.Sum() + + return Chunk{ + Data: chunk, + Digest: chunkDigest, + }, nil +} + +// NewReaderChunker creates a chunker that reads from an io.Reader +func NewReaderChunker(digestFunction digest.Function, reader io.Reader, minChunkSizeBytes, horizonSizeBytes int64) Chunker { + // The internal RepMaxContentDefinedChunker may peek up to this many + // bytes. We therefore make sure that the underlying buffer is big + // enough to prevent bufio.ErrBufferFull errors. + bufferSizeBytes := 2*minChunkSizeBytes + horizonSizeBytes + return &readerChunker{ + cdc.NewRepMaxContentDefinedChunker( + bufio.NewReaderSize(reader, int(bufferSizeBytes)), + &cdc.FastContentDefinedChunkerGearTable, + int(minChunkSizeBytes), + int(horizonSizeBytes), + ), + digestFunction, + } +} diff --git a/pkg/blobstore/cdc/reader_chunker_test.go b/pkg/blobstore/cdc/reader_chunker_test.go new file mode 100644 index 000000000..d92778bbd --- /dev/null +++ b/pkg/blobstore/cdc/reader_chunker_test.go @@ -0,0 +1,107 @@ +package cdc_test + +import ( + "io" + "math/rand" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" +) + +const ( + minChunkSize = 256 << 10 // 256 KiB + maxChunkSize = 2*minChunkSize - 1 + horizonLookaheadBytes = 8 * minChunkSize +) + +func FuzzReaderChunker(f *testing.F) { + for i := range 20 { + // Fuzz test i+1 MB of data with seed i. + f.Add((i+1)<<20, int64(i)) + } + f.Fuzz(func(t *testing.T, dataSizeBytes int, seed int64) { + require := require.New(t) + rng := rand.New(rand.NewSource(seed)) + originalData := make([]byte, dataSizeBytes) + rng.Read(originalData) + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + chunker := cdc.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + composedData := make([]byte, 0, dataSizeBytes) + var numberOfChunks int + for numberOfChunks = 0; ; numberOfChunks++ { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + require.NoError(err, "Failed to generate chunk %d.", numberOfChunks) + + chunkSize := int64(len(chunk.Data)) + chunkHasher := chunk.Digest.NewHasher(chunkSize) + chunkHasher.Write(chunk.Data) + + require.Equal(chunk.Digest.GetHashBytes(), chunkHasher.Sum(nil), "Digest mismatch for %d.", numberOfChunks) + composedData = append(composedData, chunk.Data...) + } + + require.Equal(originalData, composedData) + + originalDigestGen := digestFunc.NewGenerator(int64(dataSizeBytes)) + originalDigestGen.Write(originalData) + + composedDigestGen := digestFunc.NewGenerator(int64(dataSizeBytes)) + composedDigestGen.Write(composedData) + + require.Equal(originalDigestGen.Sum(), composedDigestGen.Sum(), "The digest of the composed data does not match the digest of the original data.") + + minNumberOfChunks := dataSizeBytes / maxChunkSize + require.GreaterOrEqual(numberOfChunks, minNumberOfChunks, "Produced fewer chunks than should be possible.") + + maxNumberOfChunks := dataSizeBytes / minChunkSize + require.LessOrEqual(numberOfChunks, maxNumberOfChunks, "Produced more chunks than should be possible.") + }) +} + +func TestReaderChunkerSmallBlob(t *testing.T) { + // Test with a small blob that should produce a single chunk + originalData := []byte("Hello, World!") + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + chunker := cdc.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + chunks := make([][]byte, 0, 1) + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + require.NoError(t, err) + + chunks = append(chunks, chunk.Data) + } + require.Len(t, chunks, 1) + require.Equal(t, originalData, chunks[0]) +} + +func TestReaderChunkerEmptyBlob(t *testing.T) { + // Test with empty blob + originalData := []byte{} + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + chunker := cdc.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + chunk, err := chunker.NextChunk() + require.ErrorIs(t, io.EOF, err) + require.Empty(t, chunk.Data) +} diff --git a/pkg/blobstore/chunk_list_fetcher.go b/pkg/blobstore/chunk_list_fetcher.go new file mode 100644 index 000000000..0817e2e3c --- /dev/null +++ b/pkg/blobstore/chunk_list_fetcher.go @@ -0,0 +1,33 @@ +package blobstore + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/util" +) + +type blobAccessChunkListFetcher struct { + chunkListStorage BlobAccess + maximumMessageSizeBytes int +} + +// NewBlobAccessChunkListFetcher creates a ChunkListFetcher that reads +// and decodes chunk lists from the provided BlobAccess. +func NewBlobAccessChunkListFetcher(chunkListStorage BlobAccess, maximumMessageSizeBytes int) chunklist.Fetcher { + return &blobAccessChunkListFetcher{ + chunkListStorage: chunkListStorage, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (f *blobAccessChunkListFetcher) FetchChunkList(ctx context.Context, d digest.Digest) (chunklist.ChunkList, error) { + msg, err := f.chunkListStorage.Get(ctx, d).ToProto(&chunklist_pb.ChunkList{}, f.maximumMessageSizeBytes) + if err != nil { + return nil, util.StatusWrap(err, "Failed to fetch chunk list") + } + proto := msg.(*chunklist_pb.ChunkList) + return chunklist.NewChunkListFromProto(proto, d.GetInstanceName()) +} diff --git a/pkg/blobstore/chunklist/BUILD.bazel b/pkg/blobstore/chunklist/BUILD.bazel new file mode 100644 index 000000000..cd8927589 --- /dev/null +++ b/pkg/blobstore/chunklist/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "chunklist", + srcs = [ + "caching_fetcher.go", + "chunk_concatenating_reader.go", + "chunk_fetcher.go", + "chunk_list.go", + "encoding.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist", + visibility = ["//visibility:public"], + deps = [ + "//pkg/digest", + "//pkg/proto/blobstore/chunklist", + "//pkg/ttlcache", + "//pkg/util", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) diff --git a/pkg/blobstore/chunklist/caching_fetcher.go b/pkg/blobstore/chunklist/caching_fetcher.go new file mode 100644 index 000000000..2b51933ca --- /dev/null +++ b/pkg/blobstore/chunklist/caching_fetcher.go @@ -0,0 +1,33 @@ +package chunklist + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/ttlcache" +) + +type cachingFetcher struct { + Fetcher + cache *ttlcache.TTLCache[digest.Digest, ChunkList] +} + +// NewCachingFetcher decorates a Fetcher with a cache. +func NewCachingFetcher(base Fetcher, cache *ttlcache.TTLCache[digest.Digest, ChunkList]) Fetcher { + return &cachingFetcher{ + Fetcher: base, + cache: cache, + } +} + +func (f *cachingFetcher) FetchChunkList(ctx context.Context, d digest.Digest) (ChunkList, error) { + if v, ok := f.cache.Get(d); ok { + return v, nil + } + chunkList, err := f.Fetcher.FetchChunkList(ctx, d) + if err != nil { + return nil, err + } + f.cache.Put(d, chunkList) + return chunkList, nil +} diff --git a/pkg/blobstore/chunklist/chunk_concatenating_reader.go b/pkg/blobstore/chunklist/chunk_concatenating_reader.go new file mode 100644 index 000000000..dbaf6e42a --- /dev/null +++ b/pkg/blobstore/chunklist/chunk_concatenating_reader.go @@ -0,0 +1,74 @@ +package chunklist + +import ( + "context" + "io" + + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// chunkConcatenatingReadCloser is an io.ReadCloser that stitches +// together the contents of a blob based on a ChunkList. +type chunkConcatenatingReadCloser struct { + ctx context.Context + chunkFetcher ChunkFetcher + chunkList ChunkList + + currentChunkIndex int + currentChunkData []byte + currentChunkOffset int + closed bool +} + +// NewChunkConcatenatingReader creates an io.ReadCloser that yields the +// concatenated contents of the chunks in a ChunkList. +func NewChunkConcatenatingReader(ctx context.Context, chunkList ChunkList, chunkFetcher ChunkFetcher) io.ReadCloser { + return &chunkConcatenatingReadCloser{ + ctx: ctx, + chunkFetcher: chunkFetcher, + chunkList: chunkList, + } +} + +func (r *chunkConcatenatingReadCloser) Read(p []byte) (int, error) { + if r.closed { + return 0, status.Error(codes.Internal, "Reader is already closed") + } + + // Advance to the next chunk if the current one is exhausted, + // skipping any zero-length chunks. + for r.currentChunkOffset >= len(r.currentChunkData) { + if r.currentChunkIndex >= len(r.chunkList) { + return 0, io.EOF + } + + chunkDigest := r.chunkList[r.currentChunkIndex].Digest + chunkData, err := r.chunkFetcher.FetchChunk(r.ctx, chunkDigest) + if err != nil { + return 0, util.StatusWrapf(err, "Failed to fetch chunk at index %d", r.currentChunkIndex) + } + + r.currentChunkData = chunkData + r.currentChunkOffset = 0 + r.currentChunkIndex++ + } + + // Copy as much data as available from the current chunk into p. + n := copy(p, r.currentChunkData[r.currentChunkOffset:]) + r.currentChunkOffset += n + + if r.currentChunkOffset >= len(r.currentChunkData) { + r.currentChunkData = nil + } + + return n, nil +} + +func (r *chunkConcatenatingReadCloser) Close() error { + r.closed = true + r.currentChunkData = nil + return nil +} diff --git a/pkg/blobstore/chunklist/chunk_fetcher.go b/pkg/blobstore/chunklist/chunk_fetcher.go new file mode 100644 index 000000000..146567714 --- /dev/null +++ b/pkg/blobstore/chunklist/chunk_fetcher.go @@ -0,0 +1,26 @@ +package chunklist + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" +) + +// ChunkFetcher retrieves the contents of an individual chunk. +type ChunkFetcher interface { + FetchChunk(ctx context.Context, digest digest.Digest) ([]byte, error) +} + +type functionChunkFetcher struct { + fetcher func(ctx context.Context, digest digest.Digest) ([]byte, error) +} + +// NewChunkFetcherFromFunction creates a chunk fetcher that wraps a +// function which fetches chunks. +func NewChunkFetcherFromFunction(fetcher func(ctx context.Context, digest digest.Digest) ([]byte, error)) ChunkFetcher { + return functionChunkFetcher{fetcher: fetcher} +} + +func (f functionChunkFetcher) FetchChunk(ctx context.Context, digest digest.Digest) ([]byte, error) { + return f.fetcher(ctx, digest) +} diff --git a/pkg/blobstore/chunklist/chunk_list.go b/pkg/blobstore/chunklist/chunk_list.go new file mode 100644 index 000000000..894565ae8 --- /dev/null +++ b/pkg/blobstore/chunklist/chunk_list.go @@ -0,0 +1,43 @@ +package chunklist + +import ( + "context" + "slices" + + "github.com/buildbarn/bb-storage/pkg/digest" +) + +// Entry represents a single chunk within a chunk list. +type Entry struct { + Offset uint64 + Digest digest.Digest +} + +// ChunkList represents the ordered list of chunks that compose a blob. +type ChunkList []Entry + +// Fetcher retrieves a ChunkList for a digest. +type Fetcher interface { + FetchChunkList(ctx context.Context, digest digest.Digest) (ChunkList, error) +} + +// FindChunkOffset returns the index of the chunk containing the given +// offset and the offset within that chunk. +func FindChunkOffset(chunkList ChunkList, off uint64) (index int, chunkOffset int64) { + i, ok := slices.BinarySearchFunc(chunkList, off, func(e Entry, offset uint64) int { + next := e.Offset + uint64(e.Digest.GetSizeBytes()) + if offset < e.Offset { + return 1 + } + if offset >= next { + return -1 + } + return 0 + }) + + if ok { + return i, int64(off - chunkList[i].Offset) + } + + return len(chunkList), 0 +} diff --git a/pkg/blobstore/chunklist/encoding.go b/pkg/blobstore/chunklist/encoding.go new file mode 100644 index 000000000..a6fbf12c5 --- /dev/null +++ b/pkg/blobstore/chunklist/encoding.go @@ -0,0 +1,56 @@ +package chunklist + +import ( + "bytes" + + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/util" +) + +// EncodeToBinary converts a ChunkList into a compact binary +// representation. Each digest is encoded using +// digest.Digest.GetCompactBinary(). +func EncodeToBinary(chunkDigests []digest.Digest) []byte { + var buf bytes.Buffer + for _, d := range chunkDigests { + buf.Write(d.GetCompactBinary()) + } + return buf.Bytes() +} + +// DecodeFromBinary parses a compact binary representation of a chunk +// list back into a ChunkList. +func DecodeFromBinary(data []byte, instanceName digest.InstanceName) (ChunkList, error) { + r := bytes.NewReader(data) + var cl ChunkList + var offset uint64 + for r.Len() > 0 { + d, err := instanceName.NewDigestFromCompactBinary(r) + if err != nil { + return nil, util.StatusWrap(err, "Failed to parse binary coded digest in chunk list") + } + cl = append(cl, Entry{ + Offset: offset, + Digest: d, + }) + offset += uint64(d.GetSizeBytes()) + } + return cl, nil +} + +// ToProto converts a ChunkList to the protobuf storage format. +func ToProto(cl ChunkList) *chunklist_pb.ChunkList { + digests := make([]digest.Digest, 0, len(cl)) + for _, entry := range cl { + digests = append(digests, entry.Digest) + } + return &chunklist_pb.ChunkList{ + Data: EncodeToBinary(digests), + } +} + +// NewChunkListFromProto converts a protobuf ChunkList into a ChunkList. +func NewChunkListFromProto(proto *chunklist_pb.ChunkList, instanceName digest.InstanceName) (ChunkList, error) { + return DecodeFromBinary(proto.Data, instanceName) +} diff --git a/pkg/blobstore/chunklistvalidating/BUILD.bazel b/pkg/blobstore/chunklistvalidating/BUILD.bazel new file mode 100644 index 000000000..cac7baa20 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "chunklistvalidating", + srcs = ["chunk_list_validating_blob_access.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", + "//pkg/digest", + "//pkg/proto/blobstore/chunklist", + "//pkg/util", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) + +go_test( + name = "chunklistvalidating_test", + srcs = [ + "chunk_list_validating_blob_access_test.go", + "fake_blob_access_test.go", + ], + deps = [ + ":chunklistvalidating", + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", + "//pkg/digest", + "//pkg/proto/blobstore/chunklist", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go new file mode 100644 index 000000000..acb88c372 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go @@ -0,0 +1,303 @@ +package chunklistvalidating + +import ( + "context" + "io" + "slices" + + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type chunkListValidatingBlobAccess struct { + blobstore.BlobAccess + cdcParametersFetcher cdc.ParametersFetcher + chunkListFetcher chunklist.Fetcher + chunkFetcher chunklist.ChunkFetcher + chunkStorage blobstore.BlobAccess + maximumMessageSizeBytes int +} + +// NewChunkListValidatingBlobAccess creates a wrapper around a Chunk +// List Storage (CLS) that ensures only valid chunk lists are stored in +// the CLS. A valid chunk list is a chunk list which follows the +// chunking parameters, has all the chunks present in the Content +// Addressable Storage (CAS) and where the chunks concatenate into the +// appropriate digest. +// +// This validation is fairly expensive and validation should only be +// done at a single layer as close as possible to the CAS where the full +// view of the CAS is available. +func NewChunkListValidatingBlobAccess(chunkListStorage, chunkStorage blobstore.BlobAccess, maximumMessageSizeBytes int) blobstore.BlobAccess { + return &chunkListValidatingBlobAccess{ + BlobAccess: chunkListStorage, + cdcParametersFetcher: cdc.NewCapabilitiesParametersFetcher(chunkStorage), + chunkListFetcher: blobstore.NewBlobAccessChunkListFetcher(chunkListStorage, maximumMessageSizeBytes), + chunkStorage: chunkStorage, + maximumMessageSizeBytes: maximumMessageSizeBytes, + chunkFetcher: chunklist.NewChunkFetcherFromFunction(func(ctx context.Context, digest digest.Digest) ([]byte, error) { + return chunkStorage.Get(ctx, digest).ToByteSlice(maximumMessageSizeBytes) + }), + } +} + +// Get the split result from the downstream blob access, should one +// exist return it only if all its constituent chunks exist. +func (ba *chunkListValidatingBlobAccess) getComplete(ctx context.Context, d digest.Digest) buffer.Buffer { + // Verify the existence of the blob itself against the chunk + // list storage. This renews the lifetime of the blob even when + // the chunk list is served from a caching chunk list storage's + // local cache, because the FindMissing is punched through to + // the authoritative chunk list storage. + missing, err := ba.BlobAccess.FindMissing(ctx, d.ToSingletonSet()) + if err != nil || !missing.Empty() { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob could not be found")) + } + + storedChunkList, err := ba.chunkListFetcher.FetchChunkList(ctx, d) + if err != nil { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Failed to get chunk list")) + } + + digestSetBuilder := digest.NewSetBuilder(len(storedChunkList)) + for _, entry := range storedChunkList { + digestSetBuilder.Add(entry.Digest) + } + + missing, err = ba.chunkStorage.FindMissing(ctx, digestSetBuilder.Build()) + if err == nil && missing.Empty() { + return buffer.NewProtoBufferFromProto(chunklist.ToProto(storedChunkList), buffer.UserProvided) + } + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob could not be found")) +} + +// Get returns a valid chunk list for the given digest, chunking the +// blob and storing the chunk list if needed. +func (ba *chunkListValidatingBlobAccess) Get(ctx context.Context, d digest.Digest) buffer.Buffer { + params, err := ba.cdcParametersFetcher.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return buffer.NewBufferFromError(err) + } + + // Check for the trivial case, the blob is small enough that it will + // always decompose to a single chunk of the same size as the + // original blob. We verify the existence of the blob in CAS and + // break out early. + blobSize := d.GetSizeBytes() + if blobSize < 2*params.MinChunkSizeBytes { + missing, err := ba.chunkStorage.FindMissing(ctx, d.ToSingletonSet()) + if err != nil { + return buffer.NewBufferFromError(util.StatusWrap(err, "Failed to verify blob existence")) + } + if !missing.Empty() { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found in CAS")) + } + + chunkList := chunklist.ChunkList{ + {Offset: 0, Digest: d}, + } + return buffer.NewProtoBufferFromProto(chunklist.ToProto(chunkList), buffer.UserProvided) + } + + // Return upstream split result if complete. + return ba.getComplete(ctx, d) +} + +// matchesStoredChunkList checks if the user-provided chunk digests +// match the chunk list already stored for the given digest. +func (ba *chunkListValidatingBlobAccess) matchesStoredChunkList(ctx context.Context, d digest.Digest, userChunkList chunklist.ChunkList) bool { + msg, err := ba.BlobAccess.Get(ctx, d).ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) + if err != nil { + return false + } + + storedChunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), d.GetInstanceName()) + if err != nil { + return false + } + + return slices.Equal(userChunkList, storedChunkList) +} + +func (ba *chunkListValidatingBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { + // Parse the buffer as a ChunkList proto. + msg, err := b.ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) + if err != nil { + return util.StatusWrap(err, "Failed to parse input as ChunkList") + } + userChunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), d.GetInstanceName()) + if err != nil { + return util.StatusWrap(err, "Failed to decode chunk list") + } + + params, err := ba.cdcParametersFetcher.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return err + } + + // No chunks given, blob must be the empty blob. + if len(userChunkList) == 0 { + if d.GetSizeBytes() != 0 { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + if d.GetDigestFunction().NewGenerator(0).Sum() != d { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + return nil + } + + // Check that all referenced chunks are present in storage. + userChunkList, err = ba.flattenChunks(ctx, params, userChunkList) + if err != nil { + return status.Error(codes.NotFound, "At least one chunk is missing from storage.") + } + + // Chunk list is marked for validation bypass, push it directy to + // downstream blob store. + if cdc.ChunkListValidationBypassed(ctx) { + return ba.BlobAccess.Put(ctx, d, b) + } + + // Check the trivial case without hitting the downstream blob + // stores. + + if ba.matchesStoredChunkList(ctx, d, userChunkList) { + return nil + } + + // No more shortcuts available go through the heavy path of + // concatenating/verifying and chunking the blobs. + canonicalChunkList := make(chunklist.ChunkList, 0, len(userChunkList)) + offset := uint64(0) + reader := chunklist.NewChunkConcatenatingReader(ctx, userChunkList, ba.chunkFetcher) + digestFunction := d.GetDigestFunction() + wholeGen := digestFunction.NewGenerator(d.GetSizeBytes()) + chunker := cdc.NewReaderChunker(d.GetDigestFunction(), reader, params.MinChunkSizeBytes, params.HorizonSizeBytes) + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if _, err := wholeGen.Write(chunk.Data); err != nil { + return status.Error(codes.Internal, "Could not compute digest of blob") + } + + missing, err := ba.chunkStorage.FindMissing(ctx, chunk.Digest.ToSingletonSet()) + if err != nil { + return err + } + if !missing.Empty() { + if err := ba.chunkStorage.Put(ctx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { + return util.StatusWrap(err, "Failed to save chunk") + } + } + canonicalChunkList = append(canonicalChunkList, chunklist.Entry{Digest: chunk.Digest, Offset: offset}) + offset += uint64(chunk.Digest.GetSizeBytes()) + } + + // Verify the whole blob against the advertised digest. + if actual := wholeGen.Sum(); actual != d { + return status.Errorf(codes.InvalidArgument, "Blob digest mismatch: advertised %s, actual %s", d, actual) + } + + // Store the canonical chunk list. + canonicalBuffer := buffer.NewProtoBufferFromProto(chunklist.ToProto(canonicalChunkList), buffer.UserProvided) + if err := ba.BlobAccess.Put(ctx, d, canonicalBuffer); err != nil { + return util.StatusWrap(err, "Failed to save canonical chunk list") + } + return nil +} + +func (ba *chunkListValidatingBlobAccess) flattenChunks(ctx context.Context, params cdc.Parameters, userChunkList chunklist.ChunkList) (chunklist.ChunkList, error) { + maxChunkSize := int64(2*params.MinChunkSizeBytes - 1) + bigDigests := digest.NewSetBuilder(len(userChunkList)) + for _, entry := range userChunkList { + if entry.Digest.GetSizeBytes() > maxChunkSize { + bigDigests.Add(entry.Digest) + } + } + missing, err := ba.BlobAccess.FindMissing(ctx, bigDigests.Build()) + if err != nil { + return nil, util.StatusWrap(err, "Error checking for chunk lists of big chunks") + } + if !missing.Empty() { + return nil, status.Error(codes.NotFound, "Chunk lists not found for big chunks") + } + flattenedChunkList := make(chunklist.ChunkList, 0, len(userChunkList)) + flattenedChunksBuilder := digest.NewSetBuilder(len(userChunkList)) + for _, outerEntry := range userChunkList { + outerDigest := outerEntry.Digest + if outerDigest.GetSizeBytes() <= maxChunkSize { + flattenedChunkList = append(flattenedChunkList, outerEntry) + flattenedChunksBuilder.Add(outerDigest) + } else { + innerChunkList, err := ba.chunkListFetcher.FetchChunkList(ctx, outerDigest) + if err != nil { + return nil, util.StatusWrap(err, "Error fetching inner chunk list") + } + for _, innerChunkEntry := range innerChunkList { + innerEntry := chunklist.Entry{ + Digest: innerChunkEntry.Digest, + Offset: innerChunkEntry.Offset + outerEntry.Offset, + } + flattenedChunkList = append(flattenedChunkList, innerEntry) + flattenedChunksBuilder.Add(innerEntry.Digest) + } + } + } + missing, err = ba.chunkStorage.FindMissing(ctx, flattenedChunksBuilder.Build()) + if err != nil { + return nil, util.StatusWrap(err, "Error checking for existence of flattened chunks.") + } + if !missing.Empty() { + return nil, status.Error(codes.NotFound, "At least one chunk among flattened chunks are missing.") + } + return flattenedChunkList, nil +} + +func (ba *chunkListValidatingBlobAccess) findMissingChunks(ctx context.Context, d digest.Digest) (digest.Set, error) { + msg, err := ba.BlobAccess.Get(ctx, d).ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) + if err != nil { + return digest.EmptySet, err + } + storedChunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), d.GetInstanceName()) + if err != nil { + return digest.EmptySet, util.StatusWrap(err, "Failed to decode chunk list") + } + builder := digest.NewSetBuilder(len(storedChunkList)) + for _, entry := range storedChunkList { + builder.Add(entry.Digest) + } + return ba.chunkStorage.FindMissing(ctx, builder.Build()) +} + +func (ba *chunkListValidatingBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + missingBlobs, err := ba.BlobAccess.FindMissing(ctx, digests) + if err != nil { + return digest.EmptySet, err + } + nonMissingBlobs, _, _ := digest.GetDifferenceAndIntersection(digests, missingBlobs) + missings := make([]digest.Set, 1, 1+nonMissingBlobs.Length()) + missings[0] = missingBlobs + for _, d := range nonMissingBlobs.Items() { + missingChunks, err := ba.findMissingChunks(ctx, d) + if err != nil { + return digest.EmptySet, err + } + if !missingChunks.Empty() { + missings = append(missings, d.ToSingletonSet()) + } + } + return digest.GetUnion(missings), nil +} diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go new file mode 100644 index 000000000..340d4e23f --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go @@ -0,0 +1,367 @@ +package chunklistvalidating_test + +import ( + "bytes" + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" + "github.com/stretchr/testify/require" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// mustComputeDigest is a test helper to easily generate digests from +// byte slices. +func mustComputeDigest(t *testing.T, digestFunction digest.Function, data []byte) digest.Digest { + t.Helper() + generator := digestFunction.NewGenerator(int64(len(data))) + _, err := generator.Write(data) + require.NoError(t, err) + return generator.Sum() +} + +// makeChunkListBuffer creates a buffer containing a ChunkList proto +// from a list of chunk digests, suitable for passing to Put(). +func makeChunkListBuffer(chunkDigests ...digest.Digest) buffer.Buffer { + cl := make(chunklist.ChunkList, 0, len(chunkDigests)) + var offset uint64 + for _, d := range chunkDigests { + cl = append(cl, chunklist.Entry{Offset: offset, Digest: d}) + offset += uint64(d.GetSizeBytes()) + } + return buffer.NewProtoBufferFromProto(chunklist.ToProto(cl), buffer.UserProvided) +} + +// getChunkListDigests reads a ChunkList proto from a buffer and +// returns the chunk digests it contains. +func getChunkListDigests(t *testing.T, buf buffer.Buffer, instanceName digest.InstanceName) []digest.Digest { + t.Helper() + msg, err := buf.ToProto(&chunklist_pb.ChunkList{}, maximumMessageSizeBytes) + require.NoError(t, err) + cl, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), instanceName) + require.NoError(t, err) + digests := make([]digest.Digest, 0, len(cl)) + for _, entry := range cl { + digests = append(digests, entry.Digest) + } + return digests +} + +var testCDCParams = &remoteexecution.RepMaxCdcParams{ + MinChunkSizeBytes: 1024, + HorizonSizeBytes: 8 * 1024, +} +var maximumMessageSizeBytes = 1024 * 1024 + +func TestChunkListValidatingBlobAccessGetTrivialSmallBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobData := []byte("Small trivial blob") + blobDigest := mustComputeDigest(t, digestFunction, blobData) + + require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) + + fakeCAS.ResetTouches() + chunkDigests := getChunkListDigests(t, validatingCLS.Get(ctx, blobDigest), digestFunction.GetInstanceName()) + + require.Len(t, chunkDigests, 1) + require.Equal(t, blobDigest, chunkDigests[0]) + require.Greater(t, fakeCAS.GetTouches(blobDigest), 0, "Blob did not have its lifetime renewed.") +} + +func TestChunkListValidatingBlobAccessGetExtendsLifetimes(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + // The blob must be at least 2*MinChunkSizeBytes, so that Get + // goes down the path of returning the stored chunk list. + blobData := bytes.Repeat([]byte("testdatafortests"), 250) + blobData = blobData[:2048] + chunk1Data := blobData[:len(blobData)/2] + chunk2Data := blobData[len(blobData)/2:] + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobDigest := mustComputeDigest(t, digestFunction, blobData) + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + require.NoError(t, fakeCLS.Put(ctx, blobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest))) + + // Reset touches. + fakeCLS.ResetTouches() + fakeCAS.ResetTouches() + + // Perform a cached split. + cachedDigests := getChunkListDigests(t, validatingCLS.Get(ctx, blobDigest), digestFunction.GetInstanceName()) + require.Equal(t, []digest.Digest{chunk1Digest, chunk2Digest}, cachedDigests) + + // The original blob's chunk list MUST have had its lifetime + // extended. + require.Greater(t, fakeCLS.GetTouches(blobDigest), 0, "Original blob's chunk list lifetime was not extended during call to Get") + + // Every chunk MUST have had its lifetime extended. + for _, chunkDigest := range cachedDigests { + require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk's lifetime was not extended during call to Get") + } +} + +func TestChunkListValidatingBlobAccessGetLargeBlobMissingUnderlyingChunk(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunk1Data := bytes.Repeat([]byte("A"), 1500) + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + chunk2Data := bytes.Repeat([]byte("B"), 1500) + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + // Chunk 2 is not uploaded to the chunk storage. + + expectedFullData := append(chunk1Data, chunk2Data...) + blobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + require.NoError(t, fakeCLS.Put(ctx, blobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest))) + + _, err := validatingCLS.Get(ctx, blobDigest).ToProto(&chunklist_pb.ChunkList{}, maximumMessageSizeBytes) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err), "Incorrect error code from Get request: %s", err.Error()) +} + +func TestChunkListValidatingBlobAccessGetMissingBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + ghostDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) + + _, err := validatingCLS.Get(ctx, ghostDigest).ToProto(&chunklist_pb.ChunkList{}, maximumMessageSizeBytes) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) +} + +func TestChunkListValidatingBlobAccessPutManualSplice(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunk1Data := []byte("Hello, ") + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Data := []byte("World!") + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + expectedFullData := []byte("Hello, World!") + fullBlobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + + err := validatingCLS.Put(ctx, fullBlobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest)) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, fullBlobDigest).ToByteSlice(len(expectedFullData)) + require.NoError(t, err) + require.Equal(t, expectedFullData, composedData) +} + +func TestChunkListValidatingBlobAccessPutCanonicalization(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + blobData := bytes.Repeat([]byte("testdatafortests"), 250) + chunk1Data := blobData[:len(blobData)/2] + chunk2Data := blobData[len(blobData)/2:] + + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + fullBlobDigest := mustComputeDigest(t, digestFunction, blobData) + + err := validatingCLS.Put(ctx, fullBlobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest)) + require.NoError(t, err) + + // The stored chunk list should be the canonical CDC chunking, + // not the non-standard chunks that were provided. + canonicalDigests := getChunkListDigests(t, fakeCLS.Get(ctx, fullBlobDigest), digestFunction.GetInstanceName()) + require.Greater(t, len(canonicalDigests), 0) + require.NotEqual(t, chunk1Digest, canonicalDigests[0], "Server should not have echoed back the non-standard chunks") +} + +func TestChunkListValidatingBlobAccessPutMissingChunk(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + chunkDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) + + err := validatingCLS.Put(ctx, chunkDigest, makeChunkListBuffer(chunkDigest)) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) +} + +func TestChunkListValidatingBlobAccessPutDigestMismatch(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkData := []byte("Valid chunk data") + chunkDigest := mustComputeDigest(t, digestFunction, chunkData) + require.NoError(t, fakeCAS.Put(ctx, chunkDigest, buffer.NewValidatedBufferFromByteSlice(chunkData))) + + wrongBlobDigest := mustComputeDigest(t, digestFunction, []byte("Different data")) + + err := validatingCLS.Put(ctx, wrongBlobDigest, makeChunkListBuffer(chunkDigest)) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err), "Incorrect error code from Put request: %s", err.Error()) +} + +func TestChunkListValidatingBlobAccessPutEmptyBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + emptyDigest := mustComputeDigest(t, digestFunction, nil) + + err := validatingCLS.Put(ctx, emptyDigest, makeChunkListBuffer()) + require.NoError(t, err) +} + +func TestChunkListValidatingBlobAccessPutRepeatedChunks(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkA := []byte("A") + digestA := mustComputeDigest(t, digestFunction, chunkA) + require.NoError(t, fakeCAS.Put(ctx, digestA, buffer.NewValidatedBufferFromByteSlice(chunkA))) + + chunkB := []byte("B") + digestB := mustComputeDigest(t, digestFunction, chunkB) + require.NoError(t, fakeCAS.Put(ctx, digestB, buffer.NewValidatedBufferFromByteSlice(chunkB))) + + expectedData := []byte("AABA") + expectedDigest := mustComputeDigest(t, digestFunction, expectedData) + + err := validatingCLS.Put(ctx, expectedDigest, makeChunkListBuffer(digestA, digestA, digestB, digestA)) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, expectedDigest).ToByteSlice(len(expectedData)) + require.NoError(t, err) + require.Equal(t, expectedData, composedData) +} + +func TestChunkListValidatingBlobAccessPutInlineEmptyChunk(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkData := []byte("Valid") + chunkDigest := mustComputeDigest(t, digestFunction, chunkData) + require.NoError(t, fakeCAS.Put(ctx, chunkDigest, buffer.NewValidatedBufferFromByteSlice(chunkData))) + + emptyDigest := mustComputeDigest(t, digestFunction, nil) + require.NoError(t, fakeCAS.Put(ctx, emptyDigest, buffer.NewValidatedBufferFromByteSlice(nil))) + + expectedDigest := mustComputeDigest(t, digestFunction, chunkData) + + err := validatingCLS.Put(ctx, expectedDigest, makeChunkListBuffer(chunkDigest, emptyDigest)) + require.NoError(t, err) +} + +func TestChunkListValidatingBlobAccessPutExtendsLifetimes(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunk1Data := []byte("Hello, ") + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Data := []byte("World!") + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + expectedFullData := []byte("Hello, World!") + fullBlobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + + fakeCAS.ResetTouches() + + err := validatingCLS.Put(ctx, fullBlobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest)) + + // From the REAPI, the server may either process the splice and + // return OK, OR it may return ALREADY_EXISTS if the blob is + // already composed and the server chooses not to extend the + // lifetime of the user's specific chunks. + if status.Code(err) == codes.AlreadyExists { + // The server is free not to touch the user's chunks. + // However, it MUST still have verified/touched the original + // blob. + require.Greater(t, fakeCAS.GetTouches(fullBlobDigest), 0, "Composed blob lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") + } else { + // Because the server accepted the Splice request, it is + // strictly obligated to extend the lifetimes of BOTH the + // provided chunks and the composed blob. + require.NoError(t, err) + + require.Greater(t, fakeCAS.GetTouches(chunk1Digest), 0, "Chunk 1 lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCAS.GetTouches(chunk2Digest), 0, "Chunk 2 lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCAS.GetTouches(fullBlobDigest), 0, "Composed blob lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") + } +} diff --git a/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go b/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go new file mode 100644 index 000000000..ffd7922c1 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go @@ -0,0 +1,88 @@ +package chunklistvalidating_test + +import ( + "context" + "sync" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// fakeBlobAccess provides a thread-safe, in-memory BlobAccess for +// testing. +type fakeBlobAccess struct { + blobstore.BlobAccess + lock sync.Mutex + blobs map[digest.Digest][]byte + touches map[digest.Digest]int // Tracks lifetime extensions + chunkingParameters *remoteexecution.RepMaxCdcParams +} + +func newFakeBlobAccess(chunkingParameters *remoteexecution.RepMaxCdcParams) *fakeBlobAccess { + return &fakeBlobAccess{ + blobs: make(map[digest.Digest][]byte), + touches: make(map[digest.Digest]int), + chunkingParameters: chunkingParameters, + } +} + +func (f *fakeBlobAccess) Get(ctx context.Context, d digest.Digest) buffer.Buffer { + f.lock.Lock() + defer f.lock.Unlock() + data, ok := f.blobs[d] + if !ok { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found")) + } + return buffer.NewValidatedBufferFromByteSlice(data) +} + +func (f *fakeBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { + data, err := b.ToByteSlice(100 * 1024 * 1024) + if err != nil { + return err + } + f.lock.Lock() + defer f.lock.Unlock() + f.touches[d]++ + f.blobs[d] = data + return nil +} + +func (f *fakeBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + f.lock.Lock() + defer f.lock.Unlock() + missing := digest.NewSetBuilder(digests.Length()) + for _, d := range digests.Items() { + if _, ok := f.blobs[d]; !ok { + missing.Add(d) + } else { + f.touches[d]++ + } + } + return missing.Build(), nil +} + +func (f *fakeBlobAccess) GetCapabilities(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.ServerCapabilities, error) { + return &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + RepMaxCdcParams: f.chunkingParameters, + }, + }, nil +} + +func (f *fakeBlobAccess) GetTouches(d digest.Digest) int { + f.lock.Lock() + defer f.lock.Unlock() + return f.touches[d] +} + +func (f *fakeBlobAccess) ResetTouches() { + f.lock.Lock() + defer f.lock.Unlock() + f.touches = make(map[digest.Digest]int) +} diff --git a/pkg/blobstore/cls_read_buffer_factory.go b/pkg/blobstore/cls_read_buffer_factory.go new file mode 100644 index 000000000..071781c55 --- /dev/null +++ b/pkg/blobstore/cls_read_buffer_factory.go @@ -0,0 +1,27 @@ +package blobstore + +import ( + "io" + + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" +) + +type clsReadBufferFactory struct{} + +func (clsReadBufferFactory) NewBufferFromByteSlice(digest digest.Digest, data []byte, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return buffer.NewProtoBufferFromByteSlice(&chunklist_pb.ChunkList{}, data, buffer.BackendProvided(dataIntegrityCallback)) +} + +func (clsReadBufferFactory) NewBufferFromReader(digest digest.Digest, r io.ReadCloser, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return buffer.NewProtoBufferFromReader(&chunklist_pb.ChunkList{}, r, buffer.BackendProvided(dataIntegrityCallback)) +} + +func (f clsReadBufferFactory) NewBufferFromReaderAt(digest digest.Digest, r buffer.ReadAtCloser, sizeBytes int64, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return f.NewBufferFromReader(digest, newReaderFromReaderAt(r), dataIntegrityCallback) +} + +// CLSReadBufferFactory is capable of creating identifiers and buffers +// for objects stored in the Chunk List Storage (CLS). +var CLSReadBufferFactory ReadBufferFactory = clsReadBufferFactory{} diff --git a/pkg/blobstore/completenesschecking/BUILD.bazel b/pkg/blobstore/completenesschecking/BUILD.bazel index 17cf2c617..21028340d 100644 --- a/pkg/blobstore/completenesschecking/BUILD.bazel +++ b/pkg/blobstore/completenesschecking/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "//pkg/blobstore", "//pkg/blobstore/buffer", "//pkg/blobstore/slicing", + "//pkg/cas", "//pkg/digest", "//pkg/util", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", @@ -25,6 +26,8 @@ go_test( ":completenesschecking", "//internal/mock", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/digest", "//pkg/testutil", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go b/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go index 992d94d4d..cd7663a5f 100644 --- a/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go +++ b/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go @@ -8,6 +8,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -21,7 +22,7 @@ import ( type findMissingQueue struct { context context.Context digestFunction digest.Function - contentAddressableStorage blobstore.BlobAccess + contentAddressableStorage cas.ContentAddressableStorage batchSize int pending digest.SetBuilder @@ -73,7 +74,7 @@ func (q *findMissingQueue) finalize() error { type completenessCheckingBlobAccess struct { blobstore.BlobAccess - contentAddressableStorage blobstore.BlobAccess + contentAddressableStorage cas.ContentAddressableStorage batchSize int maximumMessageSizeBytes int maximumTotalTreeSizeBytes int64 @@ -93,7 +94,7 @@ type completenessCheckingBlobAccess struct { // needs to be rebuilt. By calling it, Bazel indicates that all // associated output files must remain present during the build for // forward progress to be made. -func NewCompletenessCheckingBlobAccess(actionCache, contentAddressableStorage blobstore.BlobAccess, batchSize, maximumMessageSizeBytes int, maximumTotalTreeSizeBytes int64) blobstore.BlobAccess { +func NewCompletenessCheckingBlobAccess(actionCache blobstore.BlobAccess, contentAddressableStorage cas.ContentAddressableStorage, batchSize, maximumMessageSizeBytes int, maximumTotalTreeSizeBytes int64) blobstore.BlobAccess { return &completenessCheckingBlobAccess{ BlobAccess: actionCache, contentAddressableStorage: contentAddressableStorage, @@ -151,8 +152,10 @@ func (ba *completenessCheckingBlobAccess) checkCompleteness(ctx context.Context, return status.Errorf(codes.NotFound, "Combined size of all output directories exceeds maximum limit of %d bytes", ba.maximumTotalTreeSizeBytes) } remainingTreeSizeBytes -= sizeBytes - - r := ba.contentAddressableStorage.Get(ctx, treeDigest).ToReader() + r, err := cas.GetReadCloser(ctx, ba.contentAddressableStorage, treeDigest) + if err != nil { + return err + } if err := util.VisitProtoBytesFields(r, func(fieldNumber protowire.Number, offsetBytes, sizeBytes int64, fieldReader io.Reader) error { if fieldNumber == blobstore.TreeRootFieldNumber || fieldNumber == blobstore.TreeChildrenFieldNumber { directoryMessage, err := buffer.NewProtoBufferFromReader( @@ -185,13 +188,6 @@ func (ba *completenessCheckingBlobAccess) checkCompleteness(ctx context.Context, } return nil }); err != nil { - // Any errors generated above may be caused by - // data corruption on the Tree object. Force - // reading the Tree until completion, and prefer - // read errors over any errors generated above. - if _, copyErr := io.Copy(io.Discard, r); copyErr != nil { - err = copyErr - } r.Close() return util.StatusWrapf(err, "Output directory %#v", outputDirectory.Path) } diff --git a/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go b/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go index 0fe3d6820..b789c4889 100644 --- a/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go +++ b/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go @@ -2,12 +2,13 @@ package completenesschecking_test import ( "context" - "io" "testing" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/completenesschecking" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -24,7 +25,8 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) actionCache := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) + completenessCheckingBlobAccess := completenesschecking.NewCompletenessCheckingBlobAccess( actionCache, contentAddressableStorage, @@ -149,13 +151,27 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { buffer.BackendProvided(dataIntegrityCallback.Call), ), ) - contentAddressableStorage.EXPECT().Get( + treeDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("hello"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 2, HorizonSizeBytes: 4}, nil) + contentAddressableStorage.EXPECT().GetManifest( ctx, - digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - ).Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Hard disk has a case of the Mondays"))) + treeDigest, + ).Return( + chunklist.ChunkList{{Offset: 0, Digest: treeDigest}}, + nil, + ) + // The tree's only chunk fails while the tree is being + // traversed. + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + treeDigest, + ).Return(nil, status.Error(codes.Internal, "Hard disk has a case of the Mondays")) _, err := completenessCheckingBlobAccess.Get(ctx, actionDigest).ToProto(&remoteexecution.ActionResult{}, 1000) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Output directory \"bazel-out/foo\": Hard disk has a case of the Mondays"), err) + testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Output directory \"bazel-out/foo\": Failed to fetch chunk at index 0: Hard disk has a case of the Mondays"), err) }) t.Run("GetTreeTooLarge", func(t *testing.T) { @@ -185,102 +201,6 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Combined size of all output directories exceeds maximum limit of 10000 bytes"), err) }) - t.Run("GetTreeDataCorruption", func(t *testing.T) { - // Because Tree objects are processed in a streaming - // fashion, it may be the case that we call - // FindMissing() against the CAS, even though we later - // discover that the Tree object was corrupted. - // - // This means that even if FindMissing() reports objects - // as being absent, we cannot terminate immediately. We - // must process the Tree object in its entirety. - dataIntegrityCallback1 := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback1.EXPECT().Call(true) - actionCache.EXPECT().Get(ctx, actionDigest).Return( - buffer.NewProtoBufferFromProto( - &remoteexecution.ActionResult{ - OutputDirectories: []*remoteexecution.OutputDirectory{ - { - Path: "bazel-out/foo", - TreeDigest: &remoteexecution.Digest{ - Hash: "8f0450aa5f4602d93968daba6f2e7611", - SizeBytes: 4000, - }, - }, - }, - }, - buffer.BackendProvided(dataIntegrityCallback1.Call), - ), - ) - - treeReader := mock.NewMockReadCloser(ctrl) - treeReader.EXPECT().Read(gomock.Any()). - DoAndReturn(func(p []byte) (int, error) { - treeData, err := proto.Marshal(&remoteexecution.Tree{ - Root: &remoteexecution.Directory{ - Files: []*remoteexecution.FileNode{ - { - Digest: &remoteexecution.Digest{ - Hash: "024ced29f1fdef2f644f34a071ade5be", - SizeBytes: 1, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "8b3b146b1c4df062a2dc35168cbf4ce6", - SizeBytes: 2, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "4a4a6ebb3f8b062653cb957cbdc047d9", - SizeBytes: 3, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "69778ed3e4dcf4e0c40df49e4ca5bd37", - SizeBytes: 4, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "ff7816e0353299e801a30e37aee1758c", - SizeBytes: 5, - }, - }, - }, - }, - }) - require.NoError(t, err) - return copy(p, treeData), nil - }) - treeReader.EXPECT().Read(gomock.Any()). - DoAndReturn(func(p []byte) (int, error) { - return copy(p, "Garbage"), io.EOF - }) - treeReader.EXPECT().Close() - dataIntegrityCallback2 := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback2.EXPECT().Call(false) - treeDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8f0450aa5f4602d93968daba6f2e7611", 4000) - contentAddressableStorage.EXPECT().Get(ctx, treeDigest).Return( - buffer.NewCASBufferFromReader(treeDigest, treeReader, buffer.BackendProvided(dataIntegrityCallback2.Call)), - ) - contentAddressableStorage.EXPECT().FindMissing( - ctx, - digest.NewSetBuilder(0). - Add(treeDigest). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "024ced29f1fdef2f644f34a071ade5be", 1)). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b3b146b1c4df062a2dc35168cbf4ce6", 2)). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "4a4a6ebb3f8b062653cb957cbdc047d9", 3)). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "69778ed3e4dcf4e0c40df49e4ca5bd37", 4)). - Build(), - ).Return(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "4a4a6ebb3f8b062653cb957cbdc047d9", 3).ToSingletonSet(), nil) - - _, err := completenessCheckingBlobAccess.Get(ctx, actionDigest).ToProto(&remoteexecution.ActionResult{}, 1000) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Output directory \"bazel-out/foo\": Buffer is 210 bytes in size, while 4000 bytes were expected"), err) - }) - t.Run("Success", func(t *testing.T) { // Successful checking of existence of dependencies. // Below is an ActionResult that contains five @@ -335,12 +255,7 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { buffer.BackendProvided(dataIntegrityCallback1.Call), ), ) - dataIntegrityCallback2 := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback2.EXPECT().Call(true) - contentAddressableStorage.EXPECT().Get( - ctx, - digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 200), - ).Return(buffer.NewProtoBufferFromProto(&remoteexecution.Tree{ + tree := &remoteexecution.Tree{ Root: &remoteexecution.Directory{ // Directory digests should not be part of // FindMissing(), as references to directories @@ -383,7 +298,25 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { }, {}, }, - }, buffer.BackendProvided(dataIntegrityCallback2.Call))) + } + treeBytes, err := proto.Marshal(tree) + require.NoError(t, err) + treeDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 200) + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("hello"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 2, HorizonSizeBytes: 4}, nil) + contentAddressableStorage.EXPECT().GetManifest( + ctx, + treeDigest, + ).Return( + chunklist.ChunkList{{Offset: 0, Digest: treeDigest}}, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + treeDigest, + ).Return(treeBytes, nil) contentAddressableStorage.EXPECT().FindMissing( ctx, digest.NewSetBuilder(0). @@ -410,3 +343,11 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { testutil.RequireEqualProto(t, &actionResult, actualResult) }) } + +func mustNewInstanceName(name string) digest.InstanceName { + instanceName, err := digest.NewInstanceName(name) + if err != nil { + panic(err) + } + return instanceName +} diff --git a/pkg/blobstore/configuration/BUILD.bazel b/pkg/blobstore/configuration/BUILD.bazel index 33ebe095b..580c67a57 100644 --- a/pkg/blobstore/configuration/BUILD.bazel +++ b/pkg/blobstore/configuration/BUILD.bazel @@ -6,8 +6,9 @@ go_library( "ac_blob_access_creator.go", "blob_access_creator.go", "blob_replicator_creator.go", - "cas_blob_access_creator.go", - "cas_blob_replicator_creator.go", + "cls_blob_access_creator.go", + "cs_blob_access_creator.go", + "cs_blob_replicator_creator.go", "fsac_blob_access_creator.go", "icas_blob_access_creator.go", "icas_blob_replicator_creator.go", @@ -21,16 +22,21 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/blobstore", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", + "//pkg/blobstore/chunklistvalidating", "//pkg/blobstore/completenesschecking", "//pkg/blobstore/grpcclients", "//pkg/blobstore/local", "//pkg/blobstore/mirrored", "//pkg/blobstore/readcaching", "//pkg/blobstore/readfallback", + "//pkg/blobstore/referenceexpanding", "//pkg/blobstore/replication", "//pkg/blobstore/sharding", "//pkg/blockdevice", "//pkg/capabilities", + "//pkg/cas", "//pkg/clock", "//pkg/cloud/aws", "//pkg/cloud/gcp", @@ -44,6 +50,7 @@ go_library( "//pkg/proto/configuration/blobstore", "//pkg/proto/configuration/digest", "//pkg/random", + "//pkg/ttlcache", "//pkg/util", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/configuration/ac_blob_access_creator.go b/pkg/blobstore/configuration/ac_blob_access_creator.go index 14999a233..b31a95fb3 100644 --- a/pkg/blobstore/configuration/ac_blob_access_creator.go +++ b/pkg/blobstore/configuration/ac_blob_access_creator.go @@ -6,6 +6,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/completenesschecking" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/grpc" @@ -31,7 +32,7 @@ type acBlobAccessCreator struct { protoBlobAccessCreator protoBlobReplicatorCreator - contentAddressableStorage *BlobAccessInfo + contentAddressableStorage cas.ContentAddressableStorage grpcClientFactory grpc.ClientFactory maximumMessageSizeBytes int } @@ -39,7 +40,7 @@ type acBlobAccessCreator struct { // NewACBlobAccessCreator creates a BlobAccessCreator that can be // provided to NewBlobAccessFromConfiguration() to construct a // BlobAccess that is suitable for accessing the Action Cache. -func NewACBlobAccessCreator(contentAddressableStorage *BlobAccessInfo, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { +func NewACBlobAccessCreator(contentAddressableStorage cas.ContentAddressableStorage, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { return &acBlobAccessCreator{ contentAddressableStorage: contentAddressableStorage, grpcClientFactory: grpcClientFactory, @@ -100,20 +101,22 @@ func (bac *acBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gro return BlobAccessInfo{ BlobAccess: completenesschecking.NewCompletenessCheckingBlobAccess( base.BlobAccess, - bac.contentAddressableStorage.BlobAccess, + bac.contentAddressableStorage, blobstore.RecommendedFindMissingDigestsCount, bac.maximumMessageSizeBytes, backend.CompletenessChecking.MaximumTotalTreeSizeBytes, ), - DigestKeyFormat: base.DigestKeyFormat.Combine(bac.contentAddressableStorage.DigestKeyFormat), + DigestKeyFormat: base.DigestKeyFormat.Combine(bac.contentAddressableStorage.GetDigestKeyFormat()), }, "completeness_checking", nil case *pb.BlobAccessConfiguration_Grpc: - client, err := bac.grpcClientFactory.NewClientFromConfiguration(backend.Grpc.Client, terminationGroup) + grpc := backend.Grpc + client, err := bac.grpcClientFactory.NewClientFromConfiguration(grpc.Client, terminationGroup) if err != nil { return BlobAccessInfo{}, "", err } + ba := grpcclients.NewACBlobAccess(client, bac.maximumMessageSizeBytes) return BlobAccessInfo{ - BlobAccess: grpcclients.NewACBlobAccess(client, bac.maximumMessageSizeBytes), + BlobAccess: ba, DigestKeyFormat: digest.KeyWithInstance, }, "grpc", nil default: diff --git a/pkg/blobstore/configuration/blob_access_creator.go b/pkg/blobstore/configuration/blob_access_creator.go index 6b44c1171..50d6032ce 100644 --- a/pkg/blobstore/configuration/blob_access_creator.go +++ b/pkg/blobstore/configuration/blob_access_creator.go @@ -48,7 +48,7 @@ type BlobAccessCreator interface { // NewHierarchicalInstanceNamesLocalBlobAccess() creates a // BlobAccess suitable for storing data on the local system that // uses hierarchical instance names. - NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) + NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) // NewCustomBlobAccess() can be used as a fallback to create // BlobAccess instances that only apply to this storage type. // For example, CompletenessCheckingBlobAccess is only diff --git a/pkg/blobstore/configuration/cls_blob_access_creator.go b/pkg/blobstore/configuration/cls_blob_access_creator.go new file mode 100644 index 000000000..7fd9e9c49 --- /dev/null +++ b/pkg/blobstore/configuration/cls_blob_access_creator.go @@ -0,0 +1,82 @@ +package configuration + +import ( + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" + "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/grpc" + "github.com/buildbarn/bb-storage/pkg/program" + pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" +) + +type clsBlobAccessCreator struct { + protoBlobAccessCreator + protoBlobReplicatorCreator + + chunkStorage *BlobAccessInfo + grpcClientFactory grpc.ClientFactory + maximumMessageSizeBytes int +} + +// NewCLSBlobAccessCreator creates a BlobAccessCreator that can be +// provided to NewBlobAccessFromConfiguration() to construct a +// BlobAccess that is suitable for querying for chunk list. +func NewCLSBlobAccessCreator(chunkStorage *BlobAccessInfo, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { + return &clsBlobAccessCreator{ + chunkStorage: chunkStorage, + grpcClientFactory: grpcClientFactory, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (clsBlobAccessCreator) GetReadBufferFactory() blobstore.ReadBufferFactory { + return blobstore.CLSReadBufferFactory +} + +func (clsBlobAccessCreator) GetStorageTypeName() string { + return "cls" +} + +func (clsBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provider { + return capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{}) +} + +func (bac *clsBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { + switch backend := configuration.Backend.(type) { + case *pb.BlobAccessConfiguration_ChunkListValidating: + base, err := nestedCreator.NewNestedBlobAccess(backend.ChunkListValidating.Backend, bac) + if err != nil { + return BlobAccessInfo{}, "", err + } + return BlobAccessInfo{ + BlobAccess: chunklistvalidating.NewChunkListValidatingBlobAccess( + base.BlobAccess, + bac.chunkStorage.BlobAccess, + bac.maximumMessageSizeBytes, + ), + DigestKeyFormat: base.DigestKeyFormat.Combine(bac.chunkStorage.DigestKeyFormat), + }, "chunk_list_validating", nil + + case *pb.BlobAccessConfiguration_Grpc: + grpc := backend.Grpc + client, err := bac.grpcClientFactory.NewClientFromConfiguration(grpc.Client, terminationGroup) + if err != nil { + return BlobAccessInfo{}, "", err + } + ba := grpcclients.NewCLSBlobAccess(client, bac.maximumMessageSizeBytes) + return BlobAccessInfo{ + BlobAccess: ba, + DigestKeyFormat: digest.KeyWithInstance, + }, "grpc", nil + + default: + return newProtoCustomBlobAccess(configuration, nestedCreator, bac) + } +} + +func (clsBlobAccessCreator) WrapTopLevelBlobAccess(blobAccess blobstore.BlobAccess) blobstore.BlobAccess { + return blobAccess +} diff --git a/pkg/blobstore/configuration/cas_blob_access_creator.go b/pkg/blobstore/configuration/cs_blob_access_creator.go similarity index 70% rename from pkg/blobstore/configuration/cas_blob_access_creator.go rename to pkg/blobstore/configuration/cs_blob_access_creator.go index 2677982a9..043d344bf 100644 --- a/pkg/blobstore/configuration/cas_blob_access_creator.go +++ b/pkg/blobstore/configuration/cs_blob_access_creator.go @@ -10,7 +10,9 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" "github.com/buildbarn/bb-storage/pkg/blobstore/local" + "github.com/buildbarn/bb-storage/pkg/blobstore/referenceexpanding" "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/cloud/aws" "github.com/buildbarn/bb-storage/pkg/cloud/gcp" "github.com/buildbarn/bb-storage/pkg/digest" @@ -28,27 +30,26 @@ import ( "cloud.google.com/go/storage" ) -var casCapabilitiesProvider = capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ +var csCapabilitiesProvider = capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ CacheCapabilities: &remoteexecution.CacheCapabilities{ DigestFunctions: digest.SupportedDigestFunctions, // MaxBatchTotalSize: Not used by Bazel yet. }, }) -type casBlobAccessCreator struct { - casBlobReplicatorCreator +type csBlobAccessCreator struct { + csBlobReplicatorCreator maximumMessageSizeBytes int zstdPool bb_zstd.Pool } -// NewCASBlobAccessCreator creates a BlobAccessCreator that can be +// NewCSBlobAccessCreator creates a BlobAccessCreator that can be // provided to NewBlobAccessFromConfiguration() to construct a -// BlobAccess that is suitable for accessing the Content Addressable -// Storage. -func NewCASBlobAccessCreator(grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) BlobAccessCreator { - return &casBlobAccessCreator{ - casBlobReplicatorCreator: casBlobReplicatorCreator{ +// BlobAccess that is suitable for accessing the Chunk Storage. +func NewCSBlobAccessCreator(grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) BlobAccessCreator { + return &csBlobAccessCreator{ + csBlobReplicatorCreator: csBlobReplicatorCreator{ grpcClientFactory: grpcClientFactory, }, maximumMessageSizeBytes: maximumMessageSizeBytes, @@ -56,27 +57,27 @@ func NewCASBlobAccessCreator(grpcClientFactory grpc.ClientFactory, maximumMessag } } -func (casBlobAccessCreator) GetBaseDigestKeyFormat() digest.KeyFormat { +func (csBlobAccessCreator) GetBaseDigestKeyFormat() digest.KeyFormat { return digest.KeyWithoutInstance } -func (casBlobAccessCreator) GetReadBufferFactory() blobstore.ReadBufferFactory { +func (csBlobAccessCreator) GetReadBufferFactory() blobstore.ReadBufferFactory { return blobstore.CASReadBufferFactory } -func (casBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provider { - return casCapabilitiesProvider +func (csBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provider { + return csCapabilitiesProvider } -func (casBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks int) (local.BlockListGrowthPolicy, error) { +func (csBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks int) (local.BlockListGrowthPolicy, error) { return local.NewImmutableBlockListGrowthPolicy(currentBlocks, newBlocks), nil } -func (casBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) { - return local.NewHierarchicalCASBlobAccess(keyLocationMap, locationBlobMap, globalLock, casCapabilitiesProvider), nil +func (csBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { + return local.NewHierarchicalCASBlobAccess(keyLocationMap, locationBlobMap, globalLock, capabilitiesProvider), nil } -func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { +func (bac *csBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { switch backend := configuration.Backend.(type) { case *pb.BlobAccessConfiguration_ExistenceCaching: base, err := nestedCreator.NewNestedBlobAccess(backend.ExistenceCaching.Backend, bac) @@ -92,7 +93,8 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr DigestKeyFormat: base.DigestKeyFormat, }, "existence_caching", nil case *pb.BlobAccessConfiguration_Grpc: - client, err := bac.grpcClientFactory.NewClientFromConfiguration(backend.Grpc.Client, terminationGroup) + grpc := backend.Grpc + client, err := bac.grpcClientFactory.NewClientFromConfiguration(grpc.Client, terminationGroup) if err != nil { return BlobAccessInfo{}, "", err } @@ -100,10 +102,11 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr if backend.Grpc.EnableCompression { zstdPool = bac.zstdPool } + ba := grpcclients.NewCSBlobAccess(client, uuid.NewRandom, 64<<10, zstdPool) // TODO: Should we provide a configuration option, so // that digest.KeyWithoutInstance can be used? return BlobAccessInfo{ - BlobAccess: grpcclients.NewCASBlobAccess(client, uuid.NewRandom, 64<<10, zstdPool), + BlobAccess: ba, DigestKeyFormat: digest.KeyWithInstance, }, "grpc", nil case *pb.BlobAccessConfiguration_ReferenceExpanding: @@ -124,15 +127,12 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr return BlobAccessInfo{}, "", err } - var contentAddressableStorage blobstore.BlobAccess + var contentAddressableStorage cas.ContentAddressableStorage if backend.ReferenceExpanding.ContentAddressableStorage != nil { - backend, err := nestedCreator.NewNestedBlobAccess(backend.ReferenceExpanding.ContentAddressableStorage, bac) - if err != nil { - return BlobAccessInfo{}, "", err - } - contentAddressableStorage = backend.BlobAccess + contentAddressableStorage, _, _, _, _, err = NewCASFromConfiguration(terminationGroup, backend.ReferenceExpanding.ContentAddressableStorage, bac.grpcClientFactory, bac.maximumMessageSizeBytes, bac.zstdPool) } else { - contentAddressableStorage = blobstore.NewErrorBlobAccess(status.Error(codes.Unimplemented, "No Content Addressable Storage configured")) + // TODO: is an equivalent of this needed? + // contentAddressableStorage = blobstore.NewErrorBlobAccess(status.Error(codes.Unimplemented, "No Content Addressable Storage configured")) } awsConfig, err := aws.NewConfigFromConfiguration(backend.ReferenceExpanding.AwsSession, "S3ReferenceExpandingBlobAccess") @@ -159,7 +159,7 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr } return BlobAccessInfo{ - BlobAccess: blobstore.NewReferenceExpandingBlobAccess( + BlobAccess: referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage.BlobAccess, contentAddressableStorage, &http.Client{ @@ -177,7 +177,7 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr } } -func (casBlobAccessCreator) WrapTopLevelBlobAccess(blobAccess blobstore.BlobAccess) blobstore.BlobAccess { +func (csBlobAccessCreator) WrapTopLevelBlobAccess(blobAccess blobstore.BlobAccess) blobstore.BlobAccess { // For the Content Addressable Storage it is required that the empty // blob is always present. This decorator ensures that requests // for the empty blob never contact the storage backend. diff --git a/pkg/blobstore/configuration/cas_blob_replicator_creator.go b/pkg/blobstore/configuration/cs_blob_replicator_creator.go similarity index 66% rename from pkg/blobstore/configuration/cas_blob_replicator_creator.go rename to pkg/blobstore/configuration/cs_blob_replicator_creator.go index ed3172fef..215ebaf03 100644 --- a/pkg/blobstore/configuration/cas_blob_replicator_creator.go +++ b/pkg/blobstore/configuration/cs_blob_replicator_creator.go @@ -11,25 +11,25 @@ import ( "google.golang.org/grpc/status" ) -type casBlobReplicatorCreator struct { +type csBlobReplicatorCreator struct { grpcClientFactory grpc.ClientFactory } -// NewCASBlobReplicatorCreator creates a BlobReplicatorCreator that can +// NewCSBlobReplicatorCreator creates a BlobReplicatorCreator that can // be provided to NewBlobReplicatorFromConfiguration() to construct a -// BlobReplicator that is suitable for replicating Content Addressable -// Storage objects. -func NewCASBlobReplicatorCreator(grpcClientFactory grpc.ClientFactory) BlobReplicatorCreator { - return &casBlobReplicatorCreator{ +// BlobReplicator that is suitable for replicating Chunk Storage +// objects. +func NewCSBlobReplicatorCreator(grpcClientFactory grpc.ClientFactory) BlobReplicatorCreator { + return &csBlobReplicatorCreator{ grpcClientFactory: grpcClientFactory, } } -func (casBlobReplicatorCreator) GetStorageTypeName() string { - return "cas" +func (csBlobReplicatorCreator) GetStorageTypeName() string { + return "cs" } -func (brc *casBlobReplicatorCreator) NewCustomBlobReplicator(terminationGroup program.Group, configuration *pb.BlobReplicatorConfiguration, source blobstore.BlobAccess, sink BlobAccessInfo) (replication.BlobReplicator, error) { +func (brc *csBlobReplicatorCreator) NewCustomBlobReplicator(terminationGroup program.Group, configuration *pb.BlobReplicatorConfiguration, source blobstore.BlobAccess, sink BlobAccessInfo) (replication.BlobReplicator, error) { switch mode := configuration.Mode.(type) { case *pb.BlobReplicatorConfiguration_Deduplicating: base, err := NewBlobReplicatorFromConfiguration(terminationGroup, mode.Deduplicating, source, sink, brc) diff --git a/pkg/blobstore/configuration/new_blob_access.go b/pkg/blobstore/configuration/new_blob_access.go index 941494d66..39a1e93f7 100644 --- a/pkg/blobstore/configuration/new_blob_access.go +++ b/pkg/blobstore/configuration/new_blob_access.go @@ -7,13 +7,18 @@ import ( "sync" "time" + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/local" "github.com/buildbarn/bb-storage/pkg/blobstore/mirrored" "github.com/buildbarn/bb-storage/pkg/blobstore/readcaching" "github.com/buildbarn/bb-storage/pkg/blobstore/readfallback" "github.com/buildbarn/bb-storage/pkg/blobstore/sharding" "github.com/buildbarn/bb-storage/pkg/blockdevice" + "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/eviction" @@ -24,6 +29,7 @@ import ( pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" digest_pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/digest" "github.com/buildbarn/bb-storage/pkg/random" + "github.com/buildbarn/bb-storage/pkg/ttlcache" "github.com/buildbarn/bb-storage/pkg/util" bb_zstd "github.com/buildbarn/bb-storage/pkg/zstd" "github.com/fxtlabs/primes" @@ -352,11 +358,26 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration * ) var localBlobAccess blobstore.BlobAccess + capabilitiesProvider := creator.GetDefaultCapabilitiesProvider() + chunkingParameters := backend.Local.GetChunkingParameters() + if chunkingParameters != nil { + capabilitiesProvider = capabilities.NewMergingProvider([]capabilities.Provider{ + capabilitiesProvider, + capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: true, + SpliceBlobSupport: true, + RepMaxCdcParams: chunkingParameters, + }, + }), + }) + } if backend.Local.HierarchicalInstanceNames { localBlobAccess, err = creator.NewHierarchicalInstanceNamesLocalBlobAccess( keyLocationMap, locationBlobMap, &globalLock, + capabilitiesProvider, ) if err != nil { return BlobAccessInfo{}, "", err @@ -368,7 +389,7 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration * digestKeyFormat, &globalLock, storageTypeName, - creator.GetDefaultCapabilitiesProvider(), + capabilitiesProvider, ) } return BlobAccessInfo{ @@ -616,16 +637,12 @@ func NewBlobAccessFromConfiguration(terminationGroup program.Group, configuratio }, nil } -// NewCASAndACBlobAccessFromConfiguration is a convenience function to -// create BlobAccess objects for both the Content Addressable Storage -// and Action Cache. Most Buildbarn components tend to require access to -// both these data stores. -func NewCASAndACBlobAccessFromConfiguration(terminationGroup program.Group, configuration *pb.BlobstoreConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (blobstore.BlobAccess, blobstore.BlobAccess, error) { - contentAddressableStorage, err := NewBlobAccessFromConfiguration( - terminationGroup, - configuration.GetContentAddressableStorage(), - NewCASBlobAccessCreator(grpcClientFactory, maximumMessageSizeBytes, zstdPool), - ) +// NewCASAndACFromConfiguration is a convenience function to create a +// Content Addressable Storage (CAS) and a BlobAccess for the Action +// Cache. Most Buildbarn components tend to require access to both these +// data stores. +func NewCASAndACFromConfiguration(terminationGroup program.Group, configuration *pb.BlobstoreConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cas.ContentAddressableStorage, blobstore.BlobAccess, error) { + contentAddressableStorage, _, _, _, _, err := NewCASFromConfiguration(terminationGroup, configuration.ContentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes, zstdPool) if err != nil { return nil, nil, util.StatusWrap(err, "Failed to create Content Addressable Storage") } @@ -634,7 +651,7 @@ func NewCASAndACBlobAccessFromConfiguration(terminationGroup program.Group, conf terminationGroup, configuration.GetActionCache(), NewACBlobAccessCreator( - &contentAddressableStorage, + contentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes, ), @@ -643,5 +660,61 @@ func NewCASAndACBlobAccessFromConfiguration(terminationGroup program.Group, conf return nil, nil, util.StatusWrap(err, "Failed to create Action Cache") } - return contentAddressableStorage.BlobAccess, actionCache.BlobAccess, nil + return contentAddressableStorage, actionCache.BlobAccess, nil +} + +// NewCASFromConfiguration is a convenience function to create a +// ContentAddressableStorage (CAS) and its constituent parts from +// configuration. +func NewCASFromConfiguration(terminationGroup program.Group, configuration *pb.ContentAddressableStorageConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cas.ContentAddressableStorage, blobstore.BlobAccess, blobstore.BlobAccess, chunklist.Fetcher, cdc.ParametersFetcher, error) { + chunkStorageInfo, err := NewBlobAccessFromConfiguration( + terminationGroup, + configuration.GetChunkStorage(), + NewCSBlobAccessCreator(grpcClientFactory, maximumMessageSizeBytes, zstdPool), + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create Chunk Storage") + } + chunkStorage := chunkStorageInfo.BlobAccess + + chunkListStorageInfo, err := NewBlobAccessFromConfiguration( + terminationGroup, + configuration.GetChunkListStorage(), + NewCLSBlobAccessCreator(&chunkStorageInfo, grpcClientFactory, maximumMessageSizeBytes), + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create Chunk List Storage") + } + chunkListStorage := chunkListStorageInfo.BlobAccess + var chunkListFetcher chunklist.Fetcher = blobstore.NewBlobAccessChunkListFetcher(chunkListStorage, maximumMessageSizeBytes) + if configuration.GetChunkListCache() != nil { + cache, err := ttlcache.NewTTLCacheFromConfiguration[digest.Digest, chunklist.ChunkList]( + configuration.ChunkListCache, + clock.SystemClock, + "ChunkListCache", + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create chunk list cache") + } + chunkListFetcher = chunklist.NewCachingFetcher(chunkListFetcher, cache) + } + + // The chunking parameters are a property of the Chunk Storage + // (CS), so the CDC parameters are fetched from there. + cdcParametersFetcher := cdc.NewCapabilitiesParametersFetcher(chunkStorage) + if configuration.GetCdcParameterCache() != nil { + cache, err := ttlcache.NewTTLCacheFromConfiguration[digest.InstanceName, cdc.Parameters]( + configuration.CdcParameterCache, + clock.SystemClock, + "CDCParameterCache", + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create cdc parameter cache") + } + cdcParametersFetcher = cdc.NewCachingParametersFetcher(cdcParametersFetcher, cache) + } + + contentAddressableStorage := cas.NewContentAddressableStorage(chunkStorage, chunkListStorage, chunkListFetcher, cdcParametersFetcher, chunkStorageInfo.DigestKeyFormat.Combine(chunkListStorageInfo.DigestKeyFormat)) + + return contentAddressableStorage, chunkStorage, chunkListStorage, chunkListFetcher, cdcParametersFetcher, nil } diff --git a/pkg/blobstore/configuration/proto_blob_access_creator.go b/pkg/blobstore/configuration/proto_blob_access_creator.go index aa7daf857..d7ff593ba 100644 --- a/pkg/blobstore/configuration/proto_blob_access_creator.go +++ b/pkg/blobstore/configuration/proto_blob_access_creator.go @@ -5,6 +5,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/local" + "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/digest" pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" @@ -25,7 +26,7 @@ func (protoBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks return local.NewMutableBlockListGrowthPolicy(currentBlocks), nil } -func (protoBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) { +func (protoBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { return nil, status.Error(codes.InvalidArgument, "The hierarchical instance names option can only be used for the Content Addressable Storage") } diff --git a/pkg/blobstore/deadline_enforcing_blob_access.go b/pkg/blobstore/deadline_enforcing_blob_access.go index 0faad1f16..79b9db92e 100644 --- a/pkg/blobstore/deadline_enforcing_blob_access.go +++ b/pkg/blobstore/deadline_enforcing_blob_access.go @@ -6,7 +6,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -40,15 +39,6 @@ func (d *deadlineEnforcingBlobAccess) Get(ctx context.Context, digest digest.Dig ) } -func (d *deadlineEnforcingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - ctxWithTimeout, cancel := context.WithTimeout(ctx, d.timeout) - - return buffer.WithErrorHandler( - d.delegate.GetFromComposite(ctxWithTimeout, parentDigest, childDigest, slicer), - &contextCancelingErrorHandler{cancel: cancel}, - ) -} - func (d *deadlineEnforcingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { ctxWithTimeout, cancel := context.WithTimeout(ctx, d.timeout) defer cancel() diff --git a/pkg/blobstore/demultiplexing_blob_access.go b/pkg/blobstore/demultiplexing_blob_access.go index 0835162bb..3cb4c7040 100644 --- a/pkg/blobstore/demultiplexing_blob_access.go +++ b/pkg/blobstore/demultiplexing_blob_access.go @@ -5,7 +5,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -49,17 +48,6 @@ func (ba *demultiplexingBlobAccess) Get(ctx context.Context, digest digest.Diges ) } -func (ba *demultiplexingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - backend, backendName, patcher, err := ba.getBackend(parentDigest.GetInstanceName()) - if err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - backend.GetFromComposite(ctx, patcher.PatchDigest(parentDigest), patcher.PatchDigest(childDigest), slicer), - backendNamePrefixingErrorHandler{backendName: backendName}, - ) -} - func (ba *demultiplexingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { backend, backendName, patcher, err := ba.getBackend(digest.GetInstanceName()) if err != nil { diff --git a/pkg/blobstore/demultiplexing_blob_access_test.go b/pkg/blobstore/demultiplexing_blob_access_test.go index da2e68446..5f0f6e150 100644 --- a/pkg/blobstore/demultiplexing_blob_access_test.go +++ b/pkg/blobstore/demultiplexing_blob_access_test.go @@ -87,90 +87,6 @@ func TestDemultiplexingBlobAccessGet(t *testing.T) { }) } -func TestDemultiplexingBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - demultiplexedBlobAccessGetter := mock.NewMockDemultiplexedBlobAccessGetter(ctrl) - blobAccess := blobstore.NewDemultiplexingBlobAccess(demultiplexedBlobAccessGetter.Call) - - t.Run("UnknownInstanceName", func(t *testing.T) { - // This request cannot be forwarded to any backend. - demultiplexedBlobAccessGetter.EXPECT().Call(util.Must(digest.NewInstanceName("unknown"))).Return( - nil, - "", - digest.NoopInstanceNamePatcher, - status.Error(codes.InvalidArgument, "Unknown instance name"), - ) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - - _, err := blobAccess.GetFromComposite( - ctx, - digest.MustNewDigest("unknown", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("unknown", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Unknown instance name"), err) - }) - - t.Run("BackendFailure", func(t *testing.T) { - // Error messages should have the backend name prepended. - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - demultiplexedBlobAccessGetter.EXPECT().Call(util.Must(digest.NewInstanceName("hello/world"))).Return( - baseBlobAccess, - "Primary", - digest.NewInstanceNamePatcher( - util.Must(digest.NewInstanceName("hello")), - util.Must(digest.NewInstanceName("goodbye")), - ), - nil, - ) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - baseBlobAccess.EXPECT().GetFromComposite( - ctx, - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Server on fire"))) - - _, err := blobAccess.GetFromComposite( - ctx, - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Backend \"Primary\": Server on fire"), err) - }) - - t.Run("Success", func(t *testing.T) { - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - demultiplexedBlobAccessGetter.EXPECT().Call(util.Must(digest.NewInstanceName("hello/world"))).Return( - baseBlobAccess, - "Primary", - digest.NewInstanceNamePatcher( - util.Must(digest.NewInstanceName("hello")), - util.Must(digest.NewInstanceName("goodbye")), - ), - nil, - ) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - baseBlobAccess.EXPECT().GetFromComposite( - ctx, - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("ell"))) - - data, err := blobAccess.GetFromComposite( - ctx, - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("ell"), data) - }) -} - func TestDemultiplexingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/empty_blob_injecting_blob_access.go b/pkg/blobstore/empty_blob_injecting_blob_access.go index 33eb8caa7..f6d27f05d 100644 --- a/pkg/blobstore/empty_blob_injecting_blob_access.go +++ b/pkg/blobstore/empty_blob_injecting_blob_access.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -45,13 +44,6 @@ func (ba *emptyBlobInjectingBlobAccess) Get(ctx context.Context, digest digest.D return ba.BlobAccess.Get(ctx, digest) } -func (ba *emptyBlobInjectingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if childDigest.GetSizeBytes() == 0 { - return buffer.NewCASBufferFromByteSlice(childDigest, nil, buffer.UserProvided) - } - return ba.BlobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer) -} - func (ba *emptyBlobInjectingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { if digest.GetSizeBytes() == 0 { _, err := b.ToByteSlice(0) diff --git a/pkg/blobstore/empty_blob_injecting_blob_access_test.go b/pkg/blobstore/empty_blob_injecting_blob_access_test.go index 518f99a27..83fb50dd2 100644 --- a/pkg/blobstore/empty_blob_injecting_blob_access_test.go +++ b/pkg/blobstore/empty_blob_injecting_blob_access_test.go @@ -63,53 +63,6 @@ func TestEmptyBlobInjectingBlobAccessGet(t *testing.T) { }) } -func TestEmptyBlobInjectingBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - blobAccess := blobstore.NewEmptyBlobInjectingBlobAccess(baseBlobAccess) - parentDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA256, "d0607e3f454f88e2925ef995f549503b7f67541dc7ac0dcf4662265aed7a749f", 1000) - slicer := mock.NewMockBlobSlicer(ctrl) - - t.Run("NonEmptySuccess", func(t *testing.T) { - // Requests for non-empty blobs should be forwarded. - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "7fc56270e7a70fa81a5935b72eacbe29", 1) - baseBlobAccess.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("A")), - ) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(1) - require.NoError(t, err) - require.Equal(t, []byte("A"), data) - }) - - t.Run("NonEmptyFailure", func(t *testing.T) { - // Errors from the backend should be propagated. - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "7fc56270e7a70fa81a5935b72eacbe29", 1) - baseBlobAccess.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewBufferFromError( - status.Error(codes.Internal, "Server on fire"), - ), - ) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(1) - testutil.RequireEqualStatus(t, err, status.Error(codes.Internal, "Server on fire")) - }) - - t.Run("EmptySuccess", func(t *testing.T) { - // Requests for the empty blob should be processed directly. - data, err := blobAccess.GetFromComposite(ctx, parentDigest, digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "d41d8cd98f00b204e9800998ecf8427e", 0), slicer).ToByteSlice(0) - require.NoError(t, err) - require.Empty(t, data) - }) - - t.Run("EmptyInvalid", func(t *testing.T) { - // Validation should still be performed on empty blobs. - _, err := blobAccess.GetFromComposite(ctx, parentDigest, digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 0), slicer).ToByteSlice(0) - testutil.RequireEqualStatus(t, err, status.Error(codes.InvalidArgument, "Buffer has checksum d41d8cd98f00b204e9800998ecf8427e, while 3e25960a79dbc69b674cd4ec67a72c62 was expected")) - }) -} - func TestEmptyBlobInjectingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/grpcclients/BUILD.bazel b/pkg/blobstore/grpcclients/BUILD.bazel index 257d8fd04..81d2e8c44 100644 --- a/pkg/blobstore/grpcclients/BUILD.bazel +++ b/pkg/blobstore/grpcclients/BUILD.bazel @@ -4,7 +4,8 @@ go_library( name = "grpcclients", srcs = [ "ac_blob_access.go", - "cas_blob_access.go", + "cls_blob_access.go", + "cs_blob_access.go", "fsac_blob_access.go", "icas_blob_access.go", "iscc_blob_access.go", @@ -14,8 +15,10 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", "//pkg/blobstore/slicing", "//pkg/digest", + "//pkg/proto/blobstore/chunklist", "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", @@ -33,7 +36,7 @@ go_library( go_test( name = "grpcclients_test", - srcs = ["cas_blob_access_test.go"], + srcs = ["cs_blob_access_test.go"], deps = [ ":grpcclients", "//internal/mock", diff --git a/pkg/blobstore/grpcclients/cls_blob_access.go b/pkg/blobstore/grpcclients/cls_blob_access.go new file mode 100644 index 000000000..bdc84b212 --- /dev/null +++ b/pkg/blobstore/grpcclients/cls_blob_access.go @@ -0,0 +1,122 @@ +package grpcclients + +import ( + "context" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" + + "google.golang.org/grpc" +) + +type clsBlobAccess struct { + contentAddressableStorageClient remoteexecution.ContentAddressableStorageClient + capabilitiesClient remoteexecution.CapabilitiesClient + maximumMessageSizeBytes int +} + +// NewCLSBlobAccess creates a BlobAccess that relays any requests to a +// gRPC server that implements the split and splice API calls of a +// remoteexecution.ContentAddressableStorage service. +func NewCLSBlobAccess(client grpc.ClientConnInterface, maximumMessageSizeBytes int) blobstore.BlobAccess { + return &clsBlobAccess{ + contentAddressableStorageClient: remoteexecution.NewContentAddressableStorageClient(client), + capabilitiesClient: remoteexecution.NewCapabilitiesClient(client), + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (ba *clsBlobAccess) Get(ctx context.Context, blobDigest digest.Digest) buffer.Buffer { + digestFunction := blobDigest.GetDigestFunction() + splitBlobsResponse, err := ba.contentAddressableStorageClient.SplitBlob(ctx, &remoteexecution.SplitBlobRequest{ + InstanceName: digestFunction.GetInstanceName().String(), + BlobDigest: blobDigest.GetProto(), + DigestFunction: digestFunction.GetEnumValue(), + }) + if err != nil { + return buffer.NewBufferFromError(err) + } + + // Convert wire format to storage format. + chunkDigests := make([]digest.Digest, 0, len(splitBlobsResponse.ChunkDigests)) + for _, proto := range splitBlobsResponse.ChunkDigests { + d, err := digestFunction.NewDigestFromProto(proto) + if err != nil { + return buffer.NewBufferFromError(err) + } + chunkDigests = append(chunkDigests, d) + } + return buffer.NewProtoBufferFromProto( + &chunklist_pb.ChunkList{ + Data: chunklist.EncodeToBinary(chunkDigests), + }, + buffer.BackendProvided(buffer.Irreparable(blobDigest)), + ) +} + +func (ba *clsBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { + b, _ := slicer.Slice(ba.Get(ctx, parentDigest), childDigest) + return b +} + +func (ba *clsBlobAccess) Put(ctx context.Context, blobDigest digest.Digest, b buffer.Buffer) error { + msg, err := b.ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) + if err != nil { + return err + } + + // Convert storage format to wire format. + chunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), blobDigest.GetInstanceName()) + if err != nil { + return err + } + chunkDigests := make([]*remoteexecution.Digest, 0, len(chunkList)) + for _, entry := range chunkList { + chunkDigests = append(chunkDigests, entry.Digest.GetProto()) + } + + digestFunction := blobDigest.GetDigestFunction() + _, err = ba.contentAddressableStorageClient.SpliceBlob(ctx, &remoteexecution.SpliceBlobRequest{ + InstanceName: digestFunction.GetInstanceName().String(), + DigestFunction: digestFunction.GetEnumValue(), + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + BlobDigest: blobDigest.GetProto(), + }) + return err +} + +func (ba *clsBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + // Semantically an REv2 server which supports the Split and Splice + // apis should be able to answer the SplitBlob call for any blob + // which it has in its storage. Thus we can safely say that we are + // able to Get a chunk list from an upstream server as long as it + // has the blob. We can therefore reuse the existing + // FindMissingBlobs API for this purpose. + // + // In Buildbarn we implement this on the server side by segregating + // FMB requests for blobs larger than the maximum chunk size to the + // Chunk List Storage (CLS) and to the Chunk Storage (CS) for other + // blobs. + return findMissingBlobsInternal(ctx, digests, ba.contentAddressableStorageClient) +} + +func (ba *clsBlobAccess) GetCapabilities(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.ServerCapabilities, error) { + serverCapabilities, err := getServerCapabilitiesWithCacheCapabilities(ctx, ba.capabilitiesClient, instanceName) + if err != nil { + return nil, err + } + cacheCapabilities := serverCapabilities.CacheCapabilities + // Only return fields that pertain to Chunk List Storage. + return &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: cacheCapabilities.SplitBlobSupport, + SpliceBlobSupport: cacheCapabilities.SpliceBlobSupport, + }, + }, nil +} diff --git a/pkg/blobstore/grpcclients/cas_blob_access.go b/pkg/blobstore/grpcclients/cs_blob_access.go similarity index 95% rename from pkg/blobstore/grpcclients/cas_blob_access.go rename to pkg/blobstore/grpcclients/cs_blob_access.go index 8c7b3abca..3e853a729 100644 --- a/pkg/blobstore/grpcclients/cas_blob_access.go +++ b/pkg/blobstore/grpcclients/cs_blob_access.go @@ -33,7 +33,7 @@ type casBlobAccess struct { zstdPool bb_zstd.Pool } -// NewCASBlobAccess creates a BlobAccess handle that relays any requests +// NewCSBlobAccess creates a BlobAccess handle that relays any requests // to a gRPC service that implements the bytestream.ByteStream and // remoteexecution.ContentAddressableStorage services. Those are the // services that Bazel uses to access blobs stored in the Content @@ -41,7 +41,7 @@ type casBlobAccess struct { // // If zstdPool is non-nil, the client will use ZSTD compression for // ByteStream operations if the server supports it. -func NewCASBlobAccess(client grpc.ClientConnInterface, uuidGenerator util.UUIDGenerator, readChunkSize int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { +func NewCSBlobAccess(client grpc.ClientConnInterface, uuidGenerator util.UUIDGenerator, readChunkSize int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { return &casBlobAccess{ byteStreamClient: bytestream.NewByteStreamClient(client), contentAddressableStorageClient: remoteexecution.NewContentAddressableStorageClient(client), @@ -395,7 +395,10 @@ func (ba *casBlobAccess) GetCapabilities(ctx context.Context, instanceName diges // issue batch operations. return &remoteexecution.ServerCapabilities{ CacheCapabilities: &remoteexecution.CacheCapabilities{ - DigestFunctions: digest.RemoveUnsupportedDigestFunctions(cacheCapabilities.DigestFunctions), + DigestFunctions: digest.RemoveUnsupportedDigestFunctions(cacheCapabilities.DigestFunctions), + SplitBlobSupport: cacheCapabilities.SplitBlobSupport, + SpliceBlobSupport: cacheCapabilities.SpliceBlobSupport, + RepMaxCdcParams: cacheCapabilities.RepMaxCdcParams, }, DeprecatedApiVersion: serverCapabilities.DeprecatedApiVersion, LowApiVersion: serverCapabilities.LowApiVersion, diff --git a/pkg/blobstore/grpcclients/cas_blob_access_test.go b/pkg/blobstore/grpcclients/cs_blob_access_test.go similarity index 95% rename from pkg/blobstore/grpcclients/cas_blob_access_test.go rename to pkg/blobstore/grpcclients/cs_blob_access_test.go index 4465d1ded..de60aa128 100644 --- a/pkg/blobstore/grpcclients/cas_blob_access_test.go +++ b/pkg/blobstore/grpcclients/cs_blob_access_test.go @@ -37,12 +37,12 @@ func newTestZstdPool(maxEncoders, maxDecoders int64) bb_zstd.Pool { ) } -func TestCASBlobAccessPut(t *testing.T) { +func TestCSBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, nil) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, nil) blobDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) uuid := uuid.Must(uuid.Parse("7d659e5f-0e4b-48f0-ad9f-3489db6e103b")) @@ -180,7 +180,7 @@ func TestCASBlobAccessGet(t *testing.T) { client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, nil) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, nil) t.Run("Success", func(t *testing.T) { blobDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) @@ -276,12 +276,12 @@ func TestCASBlobAccessGet(t *testing.T) { }) } -func TestCASBlobAccessGetCapabilities(t *testing.T) { +func TestCSBlobAccessGetCapabilities(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, nil) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, nil) t.Run("BackendFailure", func(t *testing.T) { client.EXPECT().Invoke( @@ -377,12 +377,12 @@ func TestCASBlobAccessGetCapabilities(t *testing.T) { }) } -func TestCASBlobAccessPutWithCompression(t *testing.T) { +func TestCSBlobAccessPutWithCompression(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, newTestZstdPool(16, 16)) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, newTestZstdPool(16, 16)) expectGetCapabilitiesWithZSTD(client) @@ -455,12 +455,12 @@ func expectGetCapabilitiesWithZSTD(client *mock.MockClientConnInterface) { }).AnyTimes() } -func TestCASBlobAccessGetWithCompression(t *testing.T) { +func TestCSBlobAccessGetWithCompression(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 100, newTestZstdPool(16, 16)) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 100, newTestZstdPool(16, 16)) expectGetCapabilitiesWithZSTD(client) @@ -500,14 +500,14 @@ func TestCASBlobAccessGetWithCompression(t *testing.T) { }) } -func TestCASBlobAccessPutPoolExhaustion(t *testing.T) { +func TestCSBlobAccessPutPoolExhaustion(t *testing.T) { // Create a pool with only 1 concurrent encoder to test backpressure. pool := bb_zstd.NewBoundedPool(1, 1, nil, nil) ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, pool) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, pool) expectGetCapabilitiesWithZSTD(client) @@ -579,7 +579,7 @@ func TestCASBlobAccessPutPoolExhaustion(t *testing.T) { wg.Wait() } -func TestCASBlobAccessPutPoolReleasesEncoder(t *testing.T) { +func TestCSBlobAccessPutPoolReleasesEncoder(t *testing.T) { // Pool with 1 encoder: if encoder isn't released after the first Put, // the second Put would deadlock. pool := bb_zstd.NewBoundedPool(1, 1, nil, nil) @@ -587,7 +587,7 @@ func TestCASBlobAccessPutPoolReleasesEncoder(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, pool) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, pool) expectGetCapabilitiesWithZSTD(client) @@ -619,7 +619,7 @@ func TestCASBlobAccessPutPoolReleasesEncoder(t *testing.T) { } } -func TestCASBlobAccessGetPoolReleasesDecoder(t *testing.T) { +func TestCSBlobAccessGetPoolReleasesDecoder(t *testing.T) { // Pool with 1 decoder: if decoder isn't released after the first Get, // the second Get would deadlock. pool := bb_zstd.NewBoundedPool(1, 1, nil, nil) @@ -627,7 +627,7 @@ func TestCASBlobAccessGetPoolReleasesDecoder(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 100, pool) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 100, pool) expectGetCapabilitiesWithZSTD(client) diff --git a/pkg/blobstore/grpcservers/BUILD.bazel b/pkg/blobstore/grpcservers/BUILD.bazel index 556ad91e8..0216ac6c2 100644 --- a/pkg/blobstore/grpcservers/BUILD.bazel +++ b/pkg/blobstore/grpcservers/BUILD.bazel @@ -15,10 +15,13 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", + "//pkg/cas", "//pkg/digest", "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", + "//pkg/storage", "//pkg/util", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", @@ -32,16 +35,23 @@ go_library( go_test( name = "grpcservers_test", srcs = [ + "action_cache_server_test.go", "byte_stream_server_test.go", "content_addressable_storage_server_test.go", + "file_system_access_cache_server_test.go", "indirect_content_addressable_storage_server_test.go", + "initial_size_class_cache_server_test.go", ], deps = [ ":grpcservers", "//internal/mock", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/digest", + "//pkg/proto/fsac", "//pkg/proto/icas", + "//pkg/proto/iscc", "//pkg/testutil", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/grpcservers/action_cache_server.go b/pkg/blobstore/grpcservers/action_cache_server.go index e8d5d5942..2b92b7316 100644 --- a/pkg/blobstore/grpcservers/action_cache_server.go +++ b/pkg/blobstore/grpcservers/action_cache_server.go @@ -7,20 +7,21 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" ) type actionCacheServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + resultReader storage.MessageReader[*remoteexecution.ActionResult] } // NewActionCacheServer creates a GRPC service for serving the contents // of a Bazel Action Cache (AC) to Bazel. -func NewActionCacheServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) remoteexecution.ActionCacheServer { +func NewActionCacheServer(blobAccess blobstore.BlobAccess, resultReader storage.MessageReader[*remoteexecution.ActionResult]) remoteexecution.ActionCacheServer { return &actionCacheServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + resultReader: resultReader, } } @@ -37,14 +38,7 @@ func (s *actionCacheServer) GetActionResult(ctx context.Context, in *remoteexecu if err != nil { return nil, err } - actionResult, err := s.blobAccess.Get(ctx, digest).ToProto( - &remoteexecution.ActionResult{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return actionResult.(*remoteexecution.ActionResult), nil + return s.resultReader.ReadMessage(ctx, digest, &remoteexecution.ActionResult{}) } func (s *actionCacheServer) UpdateActionResult(ctx context.Context, in *remoteexecution.UpdateActionResultRequest) (*remoteexecution.ActionResult, error) { diff --git a/pkg/blobstore/grpcservers/action_cache_server_test.go b/pkg/blobstore/grpcservers/action_cache_server_test.go new file mode 100644 index 000000000..5f3bcd60c --- /dev/null +++ b/pkg/blobstore/grpcservers/action_cache_server_test.go @@ -0,0 +1,46 @@ +package grpcservers_test + +import ( + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/internal/mock" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" + + "go.uber.org/mock/gomock" +) + +func TestActionCacheServer(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + blobAccess := mock.NewMockBlobAccess(ctrl) + resultReader := mock.NewMockMessageReader[*remoteexecution.ActionResult](ctrl) + + server := grpcservers.NewActionCacheServer(blobAccess, resultReader) + + t.Run("GetActionResultSuccess", func(t *testing.T) { + digest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 100) + result := &remoteexecution.ActionResult{} + + resultReader.EXPECT().ReadMessage( + gomock.Any(), + digest, + gomock.Any(), + ).Return(result, nil) + + resp, err := server.GetActionResult(ctx, &remoteexecution.GetActionResultRequest{ + InstanceName: "ubuntu1804", + DigestFunction: remoteexecution.DigestFunction_MD5, + ActionDigest: &remoteexecution.Digest{ + Hash: "09f7e02f1290be211da707a266f153b3", + SizeBytes: 100, + }, + }) + + require.NoError(t, err) + require.Equal(t, result, resp) + }) +} diff --git a/pkg/blobstore/grpcservers/byte_stream_server.go b/pkg/blobstore/grpcservers/byte_stream_server.go index 0f9cb18a2..60b196aee 100644 --- a/pkg/blobstore/grpcservers/byte_stream_server.go +++ b/pkg/blobstore/grpcservers/byte_stream_server.go @@ -6,8 +6,7 @@ import ( "io" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" - "github.com/buildbarn/bb-storage/pkg/blobstore" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" bb_zstd "github.com/buildbarn/bb-storage/pkg/zstd" @@ -18,19 +17,17 @@ import ( ) type byteStreamServer struct { - blobAccess blobstore.BlobAccess - readChunkSize int - zstdPool bb_zstd.Pool + contentAddressableStorage cas.ContentAddressableStorage + zstdPool bb_zstd.Pool } // NewByteStreamServer creates a GRPC service for reading blobs from and // writing blobs to a BlobAccess. It is used by Bazel to access the // Content Addressable Storage (CAS). -func NewByteStreamServer(blobAccess blobstore.BlobAccess, readChunkSize int, zstdPool bb_zstd.Pool) bytestream.ByteStreamServer { +func NewByteStreamServer(contentAddressableStorage cas.ContentAddressableStorage, zstdPool bb_zstd.Pool) bytestream.ByteStreamServer { return &byteStreamServer{ - blobAccess: blobAccess, - readChunkSize: readChunkSize, - zstdPool: zstdPool, + contentAddressableStorage: contentAddressableStorage, + zstdPool: zstdPool, } } @@ -38,41 +35,33 @@ func (s *byteStreamServer) Read(in *bytestream.ReadRequest, out bytestream.ByteS if in.ReadLimit != 0 { return status.Error(codes.Unimplemented, "This service does not support downloading partial files") } - digest, compressor, err := digest.NewDigestFromByteStreamReadPath(in.ResourceName) + d, compressor, err := digest.NewDigestFromByteStreamReadPath(in.ResourceName) if err != nil { return err } + if in.ReadOffset < 0 { + return status.Errorf(codes.InvalidArgument, "Negative read offset: %d", in.ReadOffset) + } + if in.ReadOffset > d.GetSizeBytes() { + return status.Errorf(codes.InvalidArgument, "Buffer is %d bytes in size, while a read at offset %d was requested", d.GetSizeBytes(), in.ReadOffset) + } + ctx := out.Context() + var w io.Writer = readStreamWriter{out: out} switch compressor { case remoteexecution.Compressor_IDENTITY: - r := s.blobAccess.Get(ctx, digest).ToChunkReader(in.ReadOffset, s.readChunkSize) - defer r.Close() - - for { - readBuf, readErr := r.Read() - if readErr == io.EOF { - return nil - } - if readErr != nil { - return readErr - } - if writeErr := out.Send(&bytestream.ReadResponse{Data: readBuf}); writeErr != nil { - return writeErr - } - } - case remoteexecution.Compressor_ZSTD: - b := s.blobAccess.Get(ctx, digest) - encoder, err := s.zstdPool.NewEncoder(ctx, &readStreamWriter{out: out}) + encoder, err := s.zstdPool.NewEncoder(ctx, w) if err != nil { - b.Discard() return status.Errorf(codes.ResourceExhausted, "Failed to acquire ZSTD encoder: %v", err) } defer encoder.Close() - return b.IntoWriter(encoder) + w = encoder + return cas.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, encoder) default: return status.Errorf(codes.Unimplemented, "This service does not support downloading compression type: %s", compressor) } + return cas.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, w) } // readStreamWriter adapts the ByteStream_ReadServer to an io.Writer. @@ -80,56 +69,13 @@ type readStreamWriter struct { out bytestream.ByteStream_ReadServer } -func (w *readStreamWriter) Write(p []byte) (int, error) { +func (w readStreamWriter) Write(p []byte) (int, error) { if err := w.out.Send(&bytestream.ReadResponse{Data: p}); err != nil { return 0, err } return len(p), nil } -type byteStreamWriteServerChunkReader struct { - stream bytestream.ByteStream_WriteServer - writeOffset int64 - data []byte - finishedWrite bool -} - -func (r *byteStreamWriteServerChunkReader) setRequest(request *bytestream.WriteRequest) error { - if r.finishedWrite { - return status.Error(codes.InvalidArgument, "Client closed stream twice") - } - if request.WriteOffset != r.writeOffset { - return status.Errorf(codes.InvalidArgument, "Attempted to write at offset %d, while %d was expected", request.WriteOffset, r.writeOffset) - } - - r.writeOffset += int64(len(request.Data)) - r.data = request.Data - r.finishedWrite = request.FinishWrite - return nil -} - -func (r *byteStreamWriteServerChunkReader) Read() ([]byte, error) { - // Read next chunk if no data is present. - if len(r.data) == 0 { - request, err := r.stream.Recv() - if err != nil { - if err == io.EOF && !r.finishedWrite { - return nil, status.Error(codes.InvalidArgument, "Client closed stream without finishing write") - } - return nil, err - } - if err := r.setRequest(request); err != nil { - return nil, err - } - } - - data := r.data - r.data = nil - return data, nil -} - -func (byteStreamWriteServerChunkReader) Close() {} - func (s *byteStreamServer) Write(stream bytestream.ByteStream_WriteServer) error { request, err := stream.Recv() if err != nil { @@ -138,45 +84,51 @@ func (s *byteStreamServer) Write(stream bytestream.ByteStream_WriteServer) error } return err } - digest, compressor, err := digest.NewDigestFromByteStreamWritePath(request.ResourceName) + + d, compressor, err := digest.NewDigestFromByteStreamWritePath(request.ResourceName) if err != nil { return err } + + ctx := stream.Context() + rawReader := &writeStreamReader{ + stream: stream, + nextOffset: int64(len(request.Data)), + finished: request.FinishWrite, + pendingData: request.Data, + } + + var r io.Reader switch compressor { case remoteexecution.Compressor_IDENTITY: - return s.writeIdentity(stream, request, digest) + r = rawReader case remoteexecution.Compressor_ZSTD: - return s.writeZstd(stream, request, digest) + zr, err := bb_zstd.NewReadCloser(ctx, s.zstdPool, rawReader) + if err != nil { + return util.StatusWrap(err, "Failed to acquire ZSTD decoder") + } + defer zr.Close() + r = zr default: return status.Errorf(codes.Unimplemented, "This service does not support uploading compression type: %s", compressor) } -} -func (s *byteStreamServer) writeIdentity(stream bytestream.ByteStream_WriteServer, request *bytestream.WriteRequest, digest digest.Digest) error { - r := &byteStreamWriteServerChunkReader{stream: stream} - if err := r.setRequest(request); err != nil { - return err - } - if err := s.blobAccess.Put( - stream.Context(), - digest, - buffer.NewCASBufferFromChunkReader(digest, r, buffer.UserProvided), - ); err != nil { + if err := cas.PutReader(ctx, s.contentAddressableStorage, d, r); err != nil { return err } return stream.SendAndClose(&bytestream.WriteResponse{ - CommittedSize: digest.GetSizeBytes(), + CommittedSize: rawReader.nextOffset, }) } -type zstdWriteStreamReader struct { +type writeStreamReader struct { stream bytestream.ByteStream_WriteServer nextOffset int64 finished bool pendingData []byte } -func (r *zstdWriteStreamReader) Read(p []byte) (n int, err error) { +func (r *writeStreamReader) Read(p []byte) (n int, err error) { if len(r.pendingData) > 0 { n = copy(p, r.pendingData) r.pendingData = r.pendingData[n:] @@ -184,7 +136,18 @@ func (r *zstdWriteStreamReader) Read(p []byte) (n int, err error) { } if r.finished { - return 0, io.EOF + // The client indicated the write was finished. Check for trailing garbage. + req, err := r.stream.Recv() + if err == nil { + if req.FinishWrite { + return 0, status.Error(codes.InvalidArgument, "Client closed stream twice") + } + return 0, status.Error(codes.InvalidArgument, "Client sent extra data after finishing write") + } + if errors.Is(err, io.EOF) { + return 0, io.EOF + } + return 0, err } req, err := r.stream.Recv() @@ -207,36 +170,10 @@ func (r *zstdWriteStreamReader) Read(p []byte) (n int, err error) { return n, nil } -func (zstdWriteStreamReader) Close() error { +func (writeStreamReader) Close() error { return nil } -func (s *byteStreamServer) writeZstd(stream bytestream.ByteStream_WriteServer, request *bytestream.WriteRequest, digest digest.Digest) error { - ctx := stream.Context() - streamReader := &zstdWriteStreamReader{ - stream: stream, - nextOffset: int64(len(request.Data)), - finished: request.FinishWrite, - pendingData: request.Data, - } - - zstdReader, err := bb_zstd.NewReadCloser(ctx, s.zstdPool, streamReader) - if err != nil { - return util.StatusWrap(err, "Failed to acquire ZSTD decoder") - } - - if err := s.blobAccess.Put( - ctx, - digest, - buffer.NewCASBufferFromReader(digest, zstdReader, buffer.UserProvided), - ); err != nil { - return err - } - return stream.SendAndClose(&bytestream.WriteResponse{ - CommittedSize: streamReader.nextOffset, - }) -} - func (byteStreamServer) QueryWriteStatus(ctx context.Context, in *bytestream.QueryWriteStatusRequest) (*bytestream.QueryWriteStatusResponse, error) { return nil, status.Error(codes.Unimplemented, "This service does not support querying write status") } diff --git a/pkg/blobstore/grpcservers/byte_stream_server_test.go b/pkg/blobstore/grpcservers/byte_stream_server_test.go index 542eb40f3..04a09ded6 100644 --- a/pkg/blobstore/grpcservers/byte_stream_server_test.go +++ b/pkg/blobstore/grpcservers/byte_stream_server_test.go @@ -10,7 +10,8 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -33,8 +34,8 @@ func TestByteStreamServer(t *testing.T) { // Create an RPC server/client pair. l := bufconn.Listen(1 << 20) server := grpc.NewServer() - blobAccess := mock.NewMockBlobAccess(ctrl) - bytestream.RegisterByteStreamServer(server, grpcservers.NewByteStreamServer(blobAccess, 10, bb_zstd.NewUnboundedPool( + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) + bytestream.RegisterByteStreamServer(server, grpcservers.NewByteStreamServer(contentAddressableStorage, bb_zstd.NewUnboundedPool( []zstd.EOption{zstd.WithEncoderConcurrency(1)}, []zstd.DOption{zstd.WithDecoderConcurrency(1)}, ))) @@ -89,12 +90,14 @@ func TestByteStreamServer(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Invalid digest size: -42 bytes"), err) }) + singleChunkParameters := cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20} + twoChunksParameters := cdc.Parameters{MinChunkSizeBytes: 10, HorizonSizeBytes: 20} + t.Run("ReadSuccessEmptyInstance", func(t *testing.T) { // Attempt to fetch the small blob without an instance name. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 5), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 5) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return([]byte("Hello"), nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "blobs/09f7e02f1290be211da707a266f153b3/5", @@ -109,10 +112,34 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadSuccessNonEmptyInstance", func(t *testing.T) { // Attempt to fetch the large blob with an instance name. - blobAccess.EXPECT().Get( + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "3538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "0538d378083b9afa5ffad767f7269509", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "1538d378083b9afa5ffad767f7269509", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "2538d378083b9afa5ffad767f7269509", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest1, + ).Return([]byte("This is a "), nil) + contentAddressableStorage.EXPECT().FetchChunk( gomock.Any(), - digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "3538d378083b9afa5ffad767f7269509", 22), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("This is a long message"))) + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "debian8/blobs/3538d378083b9afa5ffad767f7269509/22", @@ -134,10 +161,9 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadZSTDCompression", func(t *testing.T) { // Test reading with ZSTD compression. originalData := []byte("This is a test message that should be compressed with ZSTD") - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "8b2c3f8a9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f61", 58), - ).Return(buffer.NewValidatedBufferFromByteSlice(originalData)) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "8b2c3f8a9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f61", 58) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return(originalData, nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "compressed-blobs/zstd/8b2c3f8a9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f61/58", @@ -162,7 +188,6 @@ func TestByteStreamServer(t *testing.T) { require.NoError(t, err) require.Equal(t, originalData, decompressedData) }) - t.Run("ReadZSTDCompressionLargeData", func(t *testing.T) { // Test reading large data with ZSTD compression. originalData := make([]byte, 100000) @@ -170,10 +195,9 @@ func TestByteStreamServer(t *testing.T) { originalData[i] = byte(i % 256) } - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", 100000), - ).Return(buffer.NewValidatedBufferFromByteSlice(originalData)) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", 100000) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return(originalData, nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "compressed-blobs/zstd/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2/100000", @@ -214,16 +238,12 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadNegativeReadOffset", func(t *testing.T) { // Attempt to fetch a blob with a negative offset. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "6fc422233a40a75a1f028e11c3cd1140", 7), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Goodbye"))) - req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "ubuntu1804/blobs/6fc422233a40a75a1f028e11c3cd1140/7", ReadOffset: -4, }) require.NoError(t, err) + _, err = req.Recv() testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Negative read offset: -4"), err) }) @@ -231,11 +251,6 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadOffsetBeyondEnd", func(t *testing.T) { // Attempt to fetch a blob with a offset beyond the size // of the blob. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "ad3c8ac9eef32188da352082244b3598", 13), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("short message"))) - req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "ubuntu1804/blobs/ad3c8ac9eef32188da352082244b3598/13", ReadOffset: 100, @@ -247,10 +262,9 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadSuccessWithOffset", func(t *testing.T) { // Attempt to fetch a lblob with an instance name and offset. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "da39a3ee5e6b4b0d3255bfef95601890", 19), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("This offset message"))) + digest1 := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "da39a3ee5e6b4b0d3255bfef95601890", 19) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("ubuntu1804")).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return([]byte("This offset message"), nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "ubuntu1804/blobs/da39a3ee5e6b4b0d3255bfef95601890/19", @@ -259,20 +273,185 @@ func TestByteStreamServer(t *testing.T) { require.NoError(t, err) readResponse, err := req.Recv() require.NoError(t, err) - require.Equal(t, []byte(" offset me"), readResponse.Data) + require.Equal(t, []byte(" offset message"), readResponse.Data) + _, err = req.Recv() + require.Equal(t, io.EOF, err) + }) + + t.Run("ReadSuccessChunkedWithOffset", func(t *testing.T) { + // Fetch the tail of a multi-chunk blob starting in the + // middle of the second chunk. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + // The first chunk lies completely before the read offset + // and must not be fetched. + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/blobs/4538d378083b9afa5ffad767f7269509/22", + ReadOffset: 14, + }) + require.NoError(t, err) + readResponse, err := req.Recv() + require.NoError(t, err) + require.Equal(t, []byte(" messa"), readResponse.Data) readResponse, err = req.Recv() require.NoError(t, err) - require.Equal(t, []byte("ssage"), readResponse.Data) + require.Equal(t, []byte("ge"), readResponse.Data) _, err = req.Recv() require.Equal(t, io.EOF, err) }) - t.Run("ReadNonexistentBlob", func(t *testing.T) { - // Attempt to fetch a nonexistent blob. - blobAccess.EXPECT().Get( + t.Run("ReadSuccessChunkedWithOffsetAtChunkBoundary", func(t *testing.T) { + // A read offset that coincides with a chunk boundary + // should start the stream at the beginning of that chunk. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( gomock.Any(), - digest.MustNewDigest("fedora28", remoteexecution.DigestFunction_MD5, "09f34d28e9c8bb445ec996388968a9e8", 7), - ).Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found"))) + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/blobs/5538d378083b9afa5ffad767f7269509/22", + ReadOffset: 10, + }) + require.NoError(t, err) + readResponse, err := req.Recv() + require.NoError(t, err) + require.Equal(t, []byte("long messa"), readResponse.Data) + readResponse, err = req.Recv() + require.NoError(t, err) + require.Equal(t, []byte("ge"), readResponse.Data) + _, err = req.Recv() + require.Equal(t, io.EOF, err) + }) + + t.Run("ReadSuccessChunkedWithOffsetAtEnd", func(t *testing.T) { + // A read offset equal to the blob size is valid and + // should yield an empty stream. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/blobs/6538d378083b9afa5ffad767f7269509/22", + ReadOffset: 22, + }) + require.NoError(t, err) + _, err = req.Recv() + require.Equal(t, io.EOF, err) + }) + + t.Run("ReadChunkedZSTDCompressionWithOffset", func(t *testing.T) { + // Test reading with ZSTD compression where the read + // offset references the uncompressed stream. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/compressed-blobs/zstd/7538d378083b9afa5ffad767f7269509/22", + ReadOffset: 14, + }) + require.NoError(t, err) + + var compressedData []byte + for { + response, err := req.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + compressedData = append(compressedData, response.Data...) + } + + decoder, err := zstd.NewReader(nil) + require.NoError(t, err) + defer decoder.Close() + + decompressedData, err := decoder.DecodeAll(compressedData, nil) + require.NoError(t, err) + require.Equal(t, []byte(" message"), decompressedData) + }) + + t.Run("ReadNonexistentBlob", func(t *testing.T) { + digest1 := digest.MustNewDigest("fedora28", remoteexecution.DigestFunction_MD5, "09f34d28e9c8bb445ec996388968a9e8", 7) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("fedora28")).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return(nil, status.Error(codes.NotFound, "Blob not found")) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "///fedora28//blobs/09f34d28e9c8bb445ec996388968a9e8/////7/", @@ -304,16 +483,9 @@ func TestByteStreamServer(t *testing.T) { t.Run("WriteSuccessEmptyInstance", func(t *testing.T) { // Attempt to write a blob without an instance name. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "581c1053f832a1c719fb6528a588ccfd", 14), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("LaputanMachine"), data) - return nil - }) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "581c1053f832a1c719fb6528a588ccfd", 14) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), digest1, []byte("LaputanMachine")).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -345,16 +517,8 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), actualDigest, originalData).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -382,16 +546,8 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), actualDigest, originalData).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -431,16 +587,8 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(100000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), actualDigest, originalData).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -485,15 +633,9 @@ func TestByteStreamServer(t *testing.T) { // Test writing with invalid ZSTD data. invalidData := []byte("This is not valid ZSTD compressed data") - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5", 10), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(1000) - require.Error(t, err) - return err - }) + // The decompressor fails while the blob is being chunked, so + // nothing gets stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -520,16 +662,9 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + // An empty blob decomposes into no chunks at all, so it + // requires no storage. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -552,16 +687,10 @@ func TestByteStreamServer(t *testing.T) { }) t.Run("WriteSuccessWithoutFinish", func(t *testing.T) { - // Attempt to write without finishing properly. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA1, "f10e562d8825ec2e17e0d9f58646f8084a658cfa", 6), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream without finishing write"), err) - return err - }) + // Attempt to write without finishing properly. The chunking of + // the blob aborts while reading the stream, so nothing gets + // stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -570,20 +699,13 @@ func TestByteStreamServer(t *testing.T) { Data: []byte("Foo"), })) _, err = stream.CloseAndRecv() - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream without finishing write"), err) + testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Failed to chunk write stream: Client closed stream without finishing write"), err) }) t.Run("WriteFailFinishTwice", func(t *testing.T) { - // Attempted to write while finishing twice. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("fedora28", remoteexecution.DigestFunction_MD5, "cbd8f7984c654c25512e3d9241ae569f", 3), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream twice"), err) - return err - }) + // Attempted to write while finishing twice. The chunking of the + // blob aborts while reading the stream, so nothing gets stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("fedora28")).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -598,20 +720,14 @@ func TestByteStreamServer(t *testing.T) { FinishWrite: true, })) _, err = stream.CloseAndRecv() - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream twice"), err) + testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Failed to chunk write stream: Client closed stream twice"), err) }) t.Run("WriteFailBadOffset", func(t *testing.T) { - // Attempted to write with a bad write offset. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("windows10", remoteexecution.DigestFunction_MD5, "68e109f0f40ca72a15e05cc22786f8e6", 10), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to write at offset 4, while 5 was expected"), err) - return err - }) + // Attempted to write with a bad write offset. The chunking of + // the blob aborts while reading the stream, so nothing gets + // stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("windows10")).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -625,7 +741,7 @@ func TestByteStreamServer(t *testing.T) { FinishWrite: true, })) _, err = stream.CloseAndRecv() - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to write at offset 4, while 5 was expected"), err) + testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Failed to chunk write stream: Attempted to write at offset 4, while 5 was expected"), err) }) t.Run("QueryWriteStatus", func(t *testing.T) { @@ -635,3 +751,11 @@ func TestByteStreamServer(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.Unimplemented, "This service does not support querying write status"), err) }) } + +func mustNewInstanceName(name string) digest.InstanceName { + instanceName, err := digest.NewInstanceName(name) + if err != nil { + panic(err) + } + return instanceName +} diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server.go b/pkg/blobstore/grpcservers/content_addressable_storage_server.go index 18802b959..4f1631c65 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server.go @@ -4,8 +4,8 @@ import ( "context" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" - "github.com/buildbarn/bb-storage/pkg/blobstore" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -14,13 +14,13 @@ import ( ) type contentAddressableStorageServer struct { - contentAddressableStorage blobstore.BlobAccess + contentAddressableStorage cas.ContentAddressableStorage maximumMessageSizeBytes int64 } // NewContentAddressableStorageServer creates a GRPC service for serving // the contents of a Bazel Content Addressable Storage (CAS) to Bazel. -func NewContentAddressableStorageServer(contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { +func NewContentAddressableStorageServer(contentAddressableStorage cas.ContentAddressableStorage, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { return &contentAddressableStorageServer{ contentAddressableStorage: contentAddressableStorage, maximumMessageSizeBytes: maximumMessageSizeBytes, @@ -41,23 +41,26 @@ func (s *contentAddressableStorageServer) FindMissingBlobs(ctx context.Context, } inDigests := digest.NewSetBuilder(len(in.BlobDigests)) - for _, partialDigest := range in.BlobDigests { - digest, err := digestFunction.NewDigestFromProto(partialDigest) + for _, inDigest := range in.BlobDigests { + digest, err := digestFunction.NewDigestFromProto(inDigest) if err != nil { return nil, err } inDigests.Add(digest) } - outDigests, err := s.contentAddressableStorage.FindMissing(ctx, inDigests.Build()) + + missing, err := s.contentAddressableStorage.FindMissing(ctx, inDigests.Build()) if err != nil { return nil, err } - partialDigests := make([]*remoteexecution.Digest, 0, outDigests.Length()) - for _, outDigest := range outDigests.Items() { - partialDigests = append(partialDigests, outDigest.GetProto()) + + outDigests := make([]*remoteexecution.Digest, 0, missing.Length()) + for _, outDigest := range missing.Items() { + outDigests = append(outDigests, outDigest.GetProto()) } + return &remoteexecution.FindMissingBlobsResponse{ - MissingBlobDigests: partialDigests, + MissingBlobDigests: outDigests, }, nil } @@ -74,6 +77,7 @@ func (s *contentAddressableStorageServer) BatchReadBlobs(ctx context.Context, in return nil, err } + // TODO: Compensate for message overhead. bytesRemaining := s.maximumMessageSizeBytes digests := make([]digest.Digest, 0, len(in.Digests)) for _, reqDigest := range in.Digests { @@ -98,10 +102,11 @@ func (s *contentAddressableStorageServer) BatchReadBlobs(ctx context.Context, in Responses: make([]*remoteexecution.BatchReadBlobsResponse_Response, 0, len(in.Digests)), } for i, reqDigest := range in.Digests { - data, err := s.contentAddressableStorage.Get( + data, err := cas.GetBytes( ctx, + s.contentAddressableStorage, digests[i], - ).ToByteSlice(int(digests[i].GetSizeBytes())) + ) response.Responses = append(response.Responses, &remoteexecution.BatchReadBlobsResponse_Response{ Digest: reqDigest, Data: data, @@ -131,10 +136,11 @@ func (s *contentAddressableStorageServer) BatchUpdateBlobs(ctx context.Context, for _, request := range in.Requests { digest, err := digestFunction.NewDigestFromProto(request.Digest) if err == nil { - err = s.contentAddressableStorage.Put( + err = cas.PutBytes( ctx, + s.contentAddressableStorage, digest, - buffer.NewCASBufferFromByteSlice(digest, request.Data, buffer.UserProvided), + request.Data, ) } response.Responses = append(response.Responses, @@ -150,10 +156,86 @@ func (contentAddressableStorageServer) GetTree(in *remoteexecution.GetTreeReques return status.Error(codes.Unimplemented, "This service does not support downloading directory trees") } -func (contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *remoteexecution.SpliceBlobRequest) (*remoteexecution.SpliceBlobResponse, error) { - return nil, status.Error(codes.Unimplemented, "This service does not support splicing blobs") +func (s *contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *remoteexecution.SpliceBlobRequest) (*remoteexecution.SpliceBlobResponse, error) { + instanceName, err := digest.NewInstanceName(in.InstanceName) + if err != nil { + return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) + } + digestFunction, err := instanceName.GetDigestFunction(in.DigestFunction, len(in.BlobDigest.GetHash())) + if err != nil { + return nil, err + } + blobDigest, err := digestFunction.NewDigestFromProto(in.BlobDigest) + if err != nil { + return nil, err + } + + chunkList := make(chunklist.ChunkList, 0, len(in.ChunkDigests)) + offset := uint64(0) + for _, chunkDigestProto := range in.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + return nil, err + } + chunkList = append(chunkList, chunklist.Entry{Digest: chunkDigest, Offset: offset}) + offset += uint64(chunkDigest.GetSizeBytes()) + } + + if err := s.contentAddressableStorage.PutManifest(ctx, blobDigest, chunkList); err != nil { + return nil, err + } + + return &remoteexecution.SpliceBlobResponse{ + BlobDigest: in.BlobDigest, + }, nil } -func (contentAddressableStorageServer) SplitBlob(ctx context.Context, in *remoteexecution.SplitBlobRequest) (*remoteexecution.SplitBlobResponse, error) { - return nil, status.Error(codes.Unimplemented, "This service does not support splitting blobs") +func (s *contentAddressableStorageServer) SplitBlob(ctx context.Context, in *remoteexecution.SplitBlobRequest) (*remoteexecution.SplitBlobResponse, error) { + instanceName, err := digest.NewInstanceName(in.InstanceName) + if err != nil { + return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) + } + digestFunction, err := instanceName.GetDigestFunction(in.DigestFunction, len(in.BlobDigest.GetHash())) + if err != nil { + return nil, err + } + blobDigest, err := digestFunction.NewDigestFromProto(in.BlobDigest) + if err != nil { + return nil, err + } + + // Blobs stored as a single chunk have no chunk list; synthesize + // the trivial answer, provided the blob exists. + params, err := s.contentAddressableStorage.FetchCDCParameters(ctx, instanceName) + if err != nil { + return nil, err + } + if cas.IsSingleChunk(params, blobDigest) { + missing, err := s.contentAddressableStorage.FindMissing(ctx, blobDigest.ToSingletonSet()) + if err != nil { + return nil, util.StatusWrap(err, "Failed to check blob existence") + } + if !missing.Empty() { + return nil, status.Errorf(codes.NotFound, "Blob %s not found", blobDigest) + } + return &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{in.BlobDigest}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + }, nil + } + + storedChunkList, err := s.contentAddressableStorage.GetManifest(ctx, blobDigest) + if err != nil { + return nil, util.StatusWrap(err, "Failed to decode chunk list") + } + + chunkDigests := make([]*remoteexecution.Digest, 0, len(storedChunkList)) + for _, entry := range storedChunkList { + chunkDigests = append(chunkDigests, entry.Digest.GetProto()) + } + + return &remoteexecution.SplitBlobResponse{ + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + }, nil } diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go index 89a278297..12878182c 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go @@ -6,7 +6,8 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -44,18 +45,18 @@ func TestContentAddressableStorageServerBatchReadBlobsSuccess(t *testing.T) { InstanceName: "ubuntu1804", } - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + cas := mock.NewMockContentAddressableStorage(ctrl) + singleChunkParameters := cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20} + cas.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("ubuntu1804")).Return(singleChunkParameters, nil).Times(3) a := make([]byte, 123) - buf := buffer.NewValidatedBufferFromByteSlice(a) - contentAddressableStorage.EXPECT().Get(ctx, digest1).Return(buf) + cas.EXPECT().FetchChunk(ctx, digest1).Return(a, nil) b := make([]byte, 234) - buf2 := buffer.NewValidatedBufferFromByteSlice(b) - contentAddressableStorage.EXPECT().Get(ctx, digest2).Return(buf2) - buf3 := buffer.NewBufferFromError(status.Error(codes.NotFound, "The object you requested could not be found")) - contentAddressableStorage.EXPECT().Get(ctx, digest3).Return(buf3) + cas.EXPECT().FetchChunk(ctx, digest2).Return(b, nil) + cas.EXPECT().FetchChunk(ctx, digest3).Return(nil, status.Error(codes.NotFound, "The object you requested could not be found")) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, 1<<16) + maximumMessageSizeBytes := 4 << 20 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) response, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) require.NoError(t, err) @@ -106,10 +107,256 @@ func TestContentAddressableStorageServerBatchReadBlobsFailure(t *testing.T) { InstanceName: "ubuntu1804", } - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + cas := mock.NewMockContentAddressableStorage(ctrl) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, 200) + maximumMessageSizeBytes := 200 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) _, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to read a total of at least 357 bytes, while a maximum of 200 bytes is permitted"), err) } + +func TestContentAddressableStorageServerBatchUpdateBlobs(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + digest1 := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", 5) + digest2 := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0479688f99e8cbc70291ce272876ff8e0db71a0889daf2752884b0996056b4a0", 5) + + request := &remoteexecution.BatchUpdateBlobsRequest{ + Requests: []*remoteexecution.BatchUpdateBlobsRequest_Request{ + { + Digest: digest1.GetProto(), + Data: []byte("Hello"), + }, + { + Digest: digest2.GetProto(), + Data: []byte("World"), + }, + }, + InstanceName: "ubuntu1804", + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("ubuntu1804"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil).Times(2) + cas.EXPECT().PutChunk(ctx, digest1, []byte("Hello")).Return(nil) + cas.EXPECT().PutChunk(ctx, digest2, []byte("World")).Return(status.Error(codes.Internal, "Hard disk has a case of the Mondays")) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, 4<<20) + + response, err := contentAddressableStorageServer.BatchUpdateBlobs(ctx, request) + require.NoError(t, err) + testutil.RequireEqualProto(t, &remoteexecution.BatchUpdateBlobsResponse{ + Responses: []*remoteexecution.BatchUpdateBlobsResponse_Response{ + { + Digest: digest1.GetProto(), + }, + { + Digest: digest2.GetProto(), + Status: &status_pb.Status{ + Code: int32(codes.Internal), + Message: "Hard disk has a case of the Mondays", + }, + }, + }, + }, response) +} + +func TestContentAddressableStorageServerFindMissingBlobs(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + digest1 := digest.MustNewDigest("my_instance_name", remoteexecution.DigestFunction_SHA256, "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", 16) + digest2 := digest.MustNewDigest("my_instance_name", remoteexecution.DigestFunction_SHA256, "0479688f99e8cbc70291ce272876ff8e0db71a0889daf2752884b0996056b4a0", 256) + + request := &remoteexecution.FindMissingBlobsRequest{ + InstanceName: "my_instance_name", + BlobDigests: []*remoteexecution.Digest{ + {Hash: digest1.GetHashString(), SizeBytes: digest1.GetSizeBytes()}, + {Hash: digest2.GetHashString(), SizeBytes: digest2.GetSizeBytes()}, + }, + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + + digests := digest.NewSetBuilder(2) + digests.Add(digest1) + digests.Add(digest2) + + cas.EXPECT().FindMissing(ctx, digests.Build()).Return(digest.EmptySet, nil) + + maximumMessageSizeBytes := 200 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) + + response, err := contentAddressableStorageServer.FindMissingBlobs(ctx, request) + require.NoError(t, err) + require.Empty(t, response.GetMissingBlobDigests()) +} + +func TestContentAddressableStorageServerSplitBlob(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SplitBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + InstanceName: "my_instance_name", + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + instanceName, err := digest.NewInstanceName(request.InstanceName) + require.NoError(t, err) + digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) + require.NoError(t, err) + blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + + chunk1Digest, err := digestFunction.NewDigestFromProto(&remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }) + require.NoError(t, err) + chunk2Digest, err := digestFunction.NewDigestFromProto(&remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }) + require.NoError(t, err) + + cas := mock.NewMockContentAddressableStorage(ctrl) + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("my_instance_name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 4, HorizonSizeBytes: 8}, nil) + cas.EXPECT().GetManifest(ctx, blobDigest).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunk1Digest}, + {Offset: 8, Digest: chunk2Digest}, + }, + nil, + ) + + maximumMessageSizeBytes := 200 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) + + response, err := contentAddressableStorageServer.SplitBlob(ctx, request) + require.NoError(t, err) + require.Len(t, response.ChunkDigests, 2) + require.Equal(t, remoteexecution.ChunkingFunction_REP_MAX_CDC, response.ChunkingFunction) +} + +func TestContentAddressableStorageServerSplitBlobSingleChunk(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SplitBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + InstanceName: "my_instance_name", + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + + // The blob is small enough to fit in a single chunk. Its chunk + // list is never consulted; the blob itself is the single chunk. + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("my_instance_name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + blobDigest, err := digest.MustNewFunction("my_instance_name", remoteexecution.DigestFunction_SHA256).NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + cas.EXPECT().FindMissing(ctx, blobDigest.ToSingletonSet()).Return(digest.EmptySet, nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, 200) + + response, err := contentAddressableStorageServer.SplitBlob(ctx, request) + require.NoError(t, err) + require.Len(t, response.ChunkDigests, 1) + testutil.RequireEqualProto(t, &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, response.ChunkDigests[0]) + require.Equal(t, remoteexecution.ChunkingFunction_REP_MAX_CDC, response.ChunkingFunction) +} + +func TestContentAddressableStorageServerSplitBlobNotFound(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SplitBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + InstanceName: "my_instance_name", + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + + // A single-chunk blob that does not exist. + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("my_instance_name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + blobDigest, err := digest.MustNewFunction("my_instance_name", remoteexecution.DigestFunction_SHA256).NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + cas.EXPECT().FindMissing(ctx, blobDigest.ToSingletonSet()).Return(blobDigest.ToSingletonSet(), nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, 200) + + _, err = contentAddressableStorageServer.SplitBlob(ctx, request) + testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Blob 1-409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49-16-my_instance_name not found"), err) +} + +func TestContentAddressableStorageServerSpliceBlob(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SpliceBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + ChunkDigests: []*remoteexecution.Digest{ + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + }, + InstanceName: "my_instance_name", + } + + instanceName, err := digest.NewInstanceName(request.InstanceName) + require.NoError(t, err) + digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) + require.NoError(t, err) + blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + + chunk1Digest, err := digestFunction.NewDigestFromProto(request.ChunkDigests[0]) + require.NoError(t, err) + chunk2Digest, err := digestFunction.NewDigestFromProto(request.ChunkDigests[1]) + require.NoError(t, err) + + expectedChunkList := chunklist.ChunkList{ + {Offset: 0, Digest: chunk1Digest}, + {Offset: 8, Digest: chunk2Digest}, + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + cas.EXPECT().PutManifest(ctx, blobDigest, expectedChunkList).Return(nil) + + maximumMessageSizeBytes := 200 + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) + response, err := contentAddressableStorageServer.SpliceBlob(ctx, request) + require.NoError(t, err) + require.Equal(t, request.BlobDigest, response.BlobDigest) +} diff --git a/pkg/blobstore/grpcservers/file_system_access_cache_server.go b/pkg/blobstore/grpcservers/file_system_access_cache_server.go index 6d7bab695..90c2ed756 100644 --- a/pkg/blobstore/grpcservers/file_system_access_cache_server.go +++ b/pkg/blobstore/grpcservers/file_system_access_cache_server.go @@ -7,14 +7,15 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/fsac" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" "google.golang.org/protobuf/types/known/emptypb" ) type fileSystemAccessCacheServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + profileReader storage.MessageReader[*fsac.FileSystemAccessProfile] } // NewFileSystemAccessCacheServer creates a gRPC service for serving the @@ -22,10 +23,10 @@ type fileSystemAccessCacheServer struct { // that is specific to Buildbarn, used to store profiles of file system // access patterns of build actions. These profiles can be used to // perform readahead of objects stored in the action's input root. -func NewFileSystemAccessCacheServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) fsac.FileSystemAccessCacheServer { +func NewFileSystemAccessCacheServer(blobAccess blobstore.BlobAccess, profileReader storage.MessageReader[*fsac.FileSystemAccessProfile]) fsac.FileSystemAccessCacheServer { return &fileSystemAccessCacheServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + profileReader: profileReader, } } @@ -43,14 +44,7 @@ func (s *fileSystemAccessCacheServer) GetFileSystemAccessProfile(ctx context.Con if err != nil { return nil, err } - previousExecutionStats, err := s.blobAccess.Get(ctx, digest).ToProto( - &fsac.FileSystemAccessProfile{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return previousExecutionStats.(*fsac.FileSystemAccessProfile), nil + return s.profileReader.ReadMessage(ctx, digest, &fsac.FileSystemAccessProfile{}) } func (s *fileSystemAccessCacheServer) UpdateFileSystemAccessProfile(ctx context.Context, in *fsac.UpdateFileSystemAccessProfileRequest) (*emptypb.Empty, error) { diff --git a/pkg/blobstore/grpcservers/file_system_access_cache_server_test.go b/pkg/blobstore/grpcservers/file_system_access_cache_server_test.go new file mode 100644 index 000000000..a302c5124 --- /dev/null +++ b/pkg/blobstore/grpcservers/file_system_access_cache_server_test.go @@ -0,0 +1,47 @@ +package grpcservers_test + +import ( + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/internal/mock" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/proto/fsac" + "github.com/stretchr/testify/require" + + "go.uber.org/mock/gomock" +) + +func TestFileSystemAccessCacheServer(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + blobAccess := mock.NewMockBlobAccess(ctrl) + profileReader := mock.NewMockMessageReader[*fsac.FileSystemAccessProfile](ctrl) + + server := grpcservers.NewFileSystemAccessCacheServer(blobAccess, profileReader) + + t.Run("GetFileSystemAccessProfileSuccess", func(t *testing.T) { + digest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 100) + profile := &fsac.FileSystemAccessProfile{} + + profileReader.EXPECT().ReadMessage( + gomock.Any(), + digest, + gomock.Any(), + ).Return(profile, nil) + + resp, err := server.GetFileSystemAccessProfile(ctx, &fsac.GetFileSystemAccessProfileRequest{ + InstanceName: "ubuntu1804", + DigestFunction: remoteexecution.DigestFunction_MD5, + ReducedActionDigest: &remoteexecution.Digest{ + Hash: "09f7e02f1290be211da707a266f153b3", + SizeBytes: 100, + }, + }) + + require.NoError(t, err) + require.Equal(t, profile, resp) + }) +} diff --git a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go index b38557309..17bdd4c44 100644 --- a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go +++ b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go @@ -8,24 +8,25 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/icas" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" "google.golang.org/grpc/status" ) type indirectContentAddressableStorageServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + referenceReader storage.MessageReader[*icas.Reference] } // NewIndirectContentAddressableStorageServer creates a gRPC service for // serving the contents of an Indirect Content Addressable Storage // (ICAS). The ICAS is a Buildbarn specific extension for integrating // external corpora into the CAS. -func NewIndirectContentAddressableStorageServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) icas.IndirectContentAddressableStorageServer { +func NewIndirectContentAddressableStorageServer(blobAccess blobstore.BlobAccess, referenceReader storage.MessageReader[*icas.Reference]) icas.IndirectContentAddressableStorageServer { return &indirectContentAddressableStorageServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + referenceReader: referenceReader, } } @@ -105,12 +106,5 @@ func (s *indirectContentAddressableStorageServer) GetReference(ctx context.Conte if err != nil { return nil, err } - actionResult, err := s.blobAccess.Get(ctx, digest).ToProto( - &icas.Reference{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return actionResult.(*icas.Reference), nil + return s.referenceReader.ReadMessage(ctx, digest, &icas.Reference{}) } diff --git a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go index c4e2b9761..1acc80470 100644 --- a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go +++ b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go @@ -23,7 +23,8 @@ func TestIndirectContentAddressableStorageServerFindMissingReferences(t *testing ctrl, ctx := gomock.WithContext(context.Background(), t) blobAccess := mock.NewMockBlobAccess(ctrl) - s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, 1000) + referenceReader := mock.NewMockMessageReader[*icas.Reference](ctrl) + s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, referenceReader) t.Run("BadDigest", func(t *testing.T) { // Malformed requests cannot be executed. @@ -107,7 +108,8 @@ func TestIndirectContentAddressableStorageServerBatchUpdateReferences(t *testing ctrl, ctx := gomock.WithContext(context.Background(), t) blobAccess := mock.NewMockBlobAccess(ctrl) - s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, 1000) + referenceReader := mock.NewMockMessageReader[*icas.Reference](ctrl) + s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, referenceReader) t.Run("Mixed", func(t *testing.T) { // Send a single batch update request containing three @@ -212,7 +214,8 @@ func TestIndirectContentAddressableStorageServerGetReference(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) blobAccess := mock.NewMockBlobAccess(ctrl) - s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, 1000) + referenceReader := mock.NewMockMessageReader[*icas.Reference](ctrl) + s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, referenceReader) t.Run("BadDigest", func(t *testing.T) { // Malformed requests cannot be executed. @@ -228,11 +231,12 @@ func TestIndirectContentAddressableStorageServerGetReference(t *testing.T) { t.Run("BackendFailure", func(t *testing.T) { // Errors returned by the backend should be forwarded. - blobAccess.EXPECT().Get( + referenceReader.EXPECT().ReadMessage( ctx, digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), + gomock.Any(), ). - Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Hardware failure"))) + Return(nil, status.Error(codes.Internal, "Hardware failure")) _, err := s.GetReference(ctx, &icas.GetReferenceRequest{ InstanceName: "example", @@ -246,20 +250,18 @@ func TestIndirectContentAddressableStorageServerGetReference(t *testing.T) { }) t.Run("Success", func(t *testing.T) { - dataIntegrityCallback := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback.EXPECT().Call(true) - blobAccess.EXPECT().Get( + referenceReader.EXPECT().ReadMessage( ctx, digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - ). - Return(buffer.NewProtoBufferFromProto( - &icas.Reference{ - Medium: &icas.Reference_HttpUrl{ - HttpUrl: "http://example.com/file3.txt", - }, + gomock.Any(), + ).Return( + &icas.Reference{ + Medium: &icas.Reference_HttpUrl{ + HttpUrl: "http://example.com/file3.txt", }, - buffer.BackendProvided(dataIntegrityCallback.Call), - )) + }, + nil, + ) resp, err := s.GetReference(ctx, &icas.GetReferenceRequest{ InstanceName: "example", diff --git a/pkg/blobstore/grpcservers/initial_size_class_cache_server.go b/pkg/blobstore/grpcservers/initial_size_class_cache_server.go index fbc4ae3ca..3bcab85f8 100644 --- a/pkg/blobstore/grpcservers/initial_size_class_cache_server.go +++ b/pkg/blobstore/grpcservers/initial_size_class_cache_server.go @@ -7,14 +7,15 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/iscc" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" "google.golang.org/protobuf/types/known/emptypb" ) type initialSizeClassCacheServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + statsReader storage.MessageReader[*iscc.PreviousExecutionStats] } // NewInitialSizeClassCacheServer creates a gRPC service for serving the @@ -23,10 +24,10 @@ type initialSizeClassCacheServer struct { // execution times of actions, so that it can make better predictions // about which size class to pick during future invocations of similar // actions. -func NewInitialSizeClassCacheServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) iscc.InitialSizeClassCacheServer { +func NewInitialSizeClassCacheServer(blobAccess blobstore.BlobAccess, statsReader storage.MessageReader[*iscc.PreviousExecutionStats]) iscc.InitialSizeClassCacheServer { return &initialSizeClassCacheServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + statsReader: statsReader, } } @@ -44,14 +45,7 @@ func (s *initialSizeClassCacheServer) GetPreviousExecutionStats(ctx context.Cont if err != nil { return nil, err } - previousExecutionStats, err := s.blobAccess.Get(ctx, digest).ToProto( - &iscc.PreviousExecutionStats{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return previousExecutionStats.(*iscc.PreviousExecutionStats), nil + return s.statsReader.ReadMessage(ctx, digest, &iscc.PreviousExecutionStats{}) } func (s *initialSizeClassCacheServer) UpdatePreviousExecutionStats(ctx context.Context, in *iscc.UpdatePreviousExecutionStatsRequest) (*emptypb.Empty, error) { diff --git a/pkg/blobstore/grpcservers/initial_size_class_cache_server_test.go b/pkg/blobstore/grpcservers/initial_size_class_cache_server_test.go new file mode 100644 index 000000000..3d687683a --- /dev/null +++ b/pkg/blobstore/grpcservers/initial_size_class_cache_server_test.go @@ -0,0 +1,47 @@ +package grpcservers_test + +import ( + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/internal/mock" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/proto/iscc" + "github.com/stretchr/testify/require" + + "go.uber.org/mock/gomock" +) + +func TestInitialSizeClassCacheServer(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + blobAccess := mock.NewMockBlobAccess(ctrl) + statsReader := mock.NewMockMessageReader[*iscc.PreviousExecutionStats](ctrl) + + server := grpcservers.NewInitialSizeClassCacheServer(blobAccess, statsReader) + + t.Run("GetPreviousExecutionStatsSuccess", func(t *testing.T) { + digest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 100) + stats := &iscc.PreviousExecutionStats{} + + statsReader.EXPECT().ReadMessage( + gomock.Any(), + digest, + gomock.Any(), + ).Return(stats, nil) + + resp, err := server.GetPreviousExecutionStats(ctx, &iscc.GetPreviousExecutionStatsRequest{ + InstanceName: "ubuntu1804", + DigestFunction: remoteexecution.DigestFunction_MD5, + ReducedActionDigest: &remoteexecution.Digest{ + Hash: "09f7e02f1290be211da707a266f153b3", + SizeBytes: 100, + }, + }) + + require.NoError(t, err) + require.Equal(t, stats, resp) + }) +} diff --git a/pkg/blobstore/grpcservers/integration/BUILD.bazel b/pkg/blobstore/grpcservers/integration/BUILD.bazel new file mode 100644 index 000000000..4381b6dbf --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "integration_test", + timeout = "short", + srcs = [ + "byte_stream_server_test.go", + "content_addressable_storage_server_test.go", + "utils_test.go", + ], + data = [ + "//cmd/bb_replicator", + "//cmd/bb_storage", + ], + env = { + "BB_STORAGE_RUNFILE_PATH": "$(rlocationpath //cmd/bb_storage:bb_storage)", + "BB_REPLICATOR_RUNFILE_PATH": "$(rlocationpath //cmd/bb_replicator:bb_replicator)", + }, + deps = [ + "//pkg/digest", + "//pkg/util", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_google_uuid//:uuid", + "@com_github_klauspost_compress//zstd", + "@com_github_stretchr_testify//require", + "@org_golang_google_genproto_googleapis_bytestream//:bytestream", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//credentials/insecure", + "@org_golang_google_grpc//status", + "@org_golang_google_protobuf//proto", + "@rules_go//go/runfiles", + ], +) diff --git a/pkg/blobstore/grpcservers/integration/byte_stream_server_test.go b/pkg/blobstore/grpcservers/integration/byte_stream_server_test.go new file mode 100644 index 000000000..e7a944888 --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/byte_stream_server_test.go @@ -0,0 +1,50 @@ +package integration + +import ( + "context" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/stretchr/testify/require" +) + +func TestByteStreamAPI(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + blobData := makeRandomData(t, 2*maximumMessageSizeBytes, 0) + digest := computeDigest(blobData) + + tests := []struct { + name string + compressor remoteexecution.Compressor_Value + data []byte + }{ + {name: "IDENTITY", compressor: remoteexecution.Compressor_IDENTITY, data: blobData}, + {name: "ZSTD", compressor: remoteexecution.Compressor_ZSTD, data: zstdEncode(blobData)}, + } + + t.Run("Write and read back binary data", func(t *testing.T) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, _, bsClient := setupServers(t) + // Write blob. + err := bytestreamWriteBlob(ctx, bsClient, test.data, digest, test.compressor) + require.NoError(t, err, "Could not write blob") + + // Read back uncompressed. + receivedData, err := bytestreamReadBlob(ctx, bsClient, digest, remoteexecution.Compressor_IDENTITY) + require.NoError(t, err, "Could not read back uploaded data") + require.Equal(t, blobData, receivedData, "Downloaded payload does not match uploaded data") + + // Read back compressed. + receivedZstdData, err := bytestreamReadBlob(ctx, bsClient, digest, remoteexecution.Compressor_ZSTD) + require.NoError(t, err, "Could not read back uploaded data") + decompressedData, err := zstdDecode(receivedZstdData) + require.NoError(t, err, "Failed to decompress ZSTD payload") + require.Equal(t, blobData, decompressedData, "Downloaded compressed payload does not match uploaded data") + }) + } + }) +} diff --git a/pkg/blobstore/grpcservers/integration/content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/integration/content_addressable_storage_server_test.go new file mode 100644 index 000000000..da569c595 --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/content_addressable_storage_server_test.go @@ -0,0 +1,404 @@ +package integration + +import ( + "context" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +func TestContentAddressableStorageAPI(t *testing.T) { + blobData := makeRandomData(t, maximumMessageSizeBytes/2, 0) + blobDigest := computeDigest(blobData) + + t.Run("GetCapabilities", func(t *testing.T) { + capabilitiesClient, _, _, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + capabilities, err := capabilitiesClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: "allowed_instance", + }) + require.NoError(t, err) + + cacheCaps := capabilities.GetCacheCapabilities() + require.NotNil(t, cacheCaps) + require.Contains(t, cacheCaps.DigestFunctions, remoteexecution.DigestFunction_SHA256) + + require.True(t, cacheCaps.SpliceBlobSupport) + require.True(t, cacheCaps.SplitBlobSupport) + + chunkingParameters := cacheCaps.GetRepMaxCdcParams() + require.NotNil(t, chunkingParameters) + require.Equal(t, minChunkSizeBytes, int(chunkingParameters.MinChunkSizeBytes)) + require.Equal(t, 8*minChunkSizeBytes, int(chunkingParameters.HorizonSizeBytes)) + + _, err = capabilitiesClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: "forbidden_instance", + }) + require.Error(t, err) + status, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.PermissionDenied, status.Code()) + }) + + t.Run("UploadSplitFMBDownload", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Upload a test blob. + err := batchUploadBlob(ctx, casClient, blobData, blobDigest) + require.NoError(t, err, "Failed to upload test data") + + // Ask server to split. + chunkDigests, err := splitBlob(ctx, casClient, blobDigest) + require.NoError(t, err, "Failed to split uploaded blob") + + // Check that all digests exist in CAS. + allDigests := append(chunkDigests, blobDigest) + missing, err := findMissingBlobs(ctx, casClient, allDigests) + require.NoError(t, err, "Failed to find missing blobs") + require.Empty(t, missing, "Blobs were unexpectedly missing") + + // Read back chunks and stitch them together. + chunks, err := batchDownloadBlobs(ctx, casClient, chunkDigests) + require.NoError(t, err, "Failed to download blobs") + rebuiltBlob := make([]byte, 0, len(blobData)) + for _, chunk := range chunks { + rebuiltBlob = append(rebuiltBlob, chunk...) + } + require.Equal(t, blobData, rebuiltBlob, "Blob did not stitch back into expected result") + + // Read back blob. + blobs, err := batchDownloadBlobs(ctx, casClient, []digest.Digest{blobDigest}) + require.NoError(t, err) + require.Len(t, blobs, 1) + require.Equal(t, blobData, blobs[0]) + }) +} + +func TestRepMaxCDCSplitAndSpliceBehaviors(t *testing.T) { + t.Run("RoundTripSplitThenSplice", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + dataSize := (minChunkSizeBytes * 4) + 128 + data := makeRandomData(t, int(dataSize), 0) + blobDigest := computeDigest(data) + err := batchUploadBlob(ctx, casClient, data, blobDigest) + require.NoError(t, err) + + digests, err := splitBlob(ctx, casClient, blobDigest) + require.NoError(t, err, "Unexpected error when splitting blob.") + + err = spliceBlob(ctx, casClient, blobDigest, digests) + require.NoError(t, err, "Unexpected error when splicing blob.") + }) + + t.Run("SpliceNonStandardChunkingThenSplit", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := makeRandomData(t, int(minChunkSizeBytes*2), 0) + + chunk1 := blobData[:1] + chunk2 := blobData[1:] + + digest1 := computeDigest(chunk1) + digest2 := computeDigest(chunk2) + blobDigest := computeDigest(blobData) + chunkDigests := []digest.Digest{digest1, digest2} + + err := batchUploadBlob(ctx, casClient, chunk1, digest1) + require.NoError(t, err, "Unexpected error when uploading chunk1.") + + err = batchUploadBlob(ctx, casClient, chunk2, digest2) + require.NoError(t, err, "Unexpected error when uploading chunk2.") + + err = spliceBlob(ctx, casClient, blobDigest, chunkDigests) + require.NoError(t, err, "Unexpected error when splicing chunks.") + + digests, err := splitBlob(ctx, casClient, blobDigest) + require.NoError(t, err, "Unexpected error when splitting recently spliced blob.") + require.NotEqual(t, chunkDigests, digests, "Split should not return non standard split result.") + + chunks, err := batchDownloadBlobs(ctx, casClient, digests) + require.NoError(t, err, "Unexpected error when downloading chunks of split blob.") + + rebuiltBlob := make([]byte, 0, len(blobData)) + for _, chunk := range chunks { + rebuiltBlob = append(rebuiltBlob, chunk...) + } + require.Equal(t, blobData, rebuiltBlob, "Blob did not stitch back into expected result") + }) + + t.Run("SpliceAlreadyExistsOrNoop", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := []byte("This blob will be fully uploaded before we try to splice it.") + blobDigest := computeDigest(blobData) + err := batchUploadBlob(ctx, casClient, blobData, blobDigest) + require.NoError(t, err, "Unexpected error when uploading blob.") + + chunk1 := blobData[:1] + chunk2 := blobData[1:] + digest1 := computeDigest(chunk1) + digest2 := computeDigest(chunk2) + chunkDigests := []digest.Digest{digest1, digest2} + + err = batchUploadBlob(ctx, casClient, chunk1, digest1) + require.NoError(t, err, "Unexpected error when uploading chunk1.") + + err = batchUploadBlob(ctx, casClient, chunk2, digest2) + require.NoError(t, err, "Unexpected error when uploading chunk2.") + + err = spliceBlob(ctx, casClient, blobDigest, chunkDigests) + if err != nil { + require.Equal(t, codes.AlreadyExists, status.Code(err), "Expected OK or ALREADY_EXISTS") + } + }) + + t.Run("ValidationSpliceBlobRejections", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + validData := makeRandomData(t, 512, 0) + validDigest := computeDigest(validData) + err := batchUploadBlob(ctx, casClient, validData, validDigest) + require.NoError(t, err, "Unexpected error when uploading blob.") + + ghostDigest := computeDigest([]byte("I do not exist in storage")) + + tests := []struct { + name string + blobDigest digest.Digest + chunkDigests []digest.Digest + expectError codes.Code + }{ + { + name: "Missing Chunk", + blobDigest: ghostDigest, + chunkDigests: []digest.Digest{ghostDigest}, + expectError: codes.NotFound, + }, + { + name: "Digest Mismatch", + blobDigest: computeDigest([]byte("Fake target")), + chunkDigests: []digest.Digest{validDigest}, + expectError: codes.InvalidArgument, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := spliceBlob(ctx, casClient, tc.blobDigest, tc.chunkDigests) + require.Error(t, err) + require.Equal(t, tc.expectError, status.Code(err), "Not the expected error: %s", err.Error()) + }) + } + }) + + t.Run("SpliceSplicedBlob", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := makeRandomData(t, 2*minChunkSizeBytes+1, 0) + blobDigest := computeDigest(blobData) + chunk1 := blobData[:1] + chunk2 := blobData[1:] + digest1 := computeDigest(chunk1) + digest2 := computeDigest(chunk2) + chunkDigests := []digest.Digest{digest1, digest2} + + err := batchUploadBlob(ctx, casClient, chunk1, digest1) + require.NoError(t, err, "Unexpected error when uploading chunk1.") + + err = batchUploadBlob(ctx, casClient, chunk2, digest2) + require.NoError(t, err, "Unexpected error when uploading chunk2.") + + err = spliceBlob(ctx, casClient, blobDigest, chunkDigests) + require.NoError(t, err, "Unexpected error when splicing blob.") + }) +} + +func TestActionCacheAPI(t *testing.T) { + smallData := []byte("small file contents") + smallDigest := computeDigest(smallData) + + bigData := makeRandomData(t, int(minChunkSizeBytes*4), 0) + bigDigest := computeDigest(bigData) + + tree := &remoteexecution.Tree{ + Root: &remoteexecution.Directory{ + Files: []*remoteexecution.FileNode{ + {Name: "big.bin", Digest: bigDigest.GetProto(), IsExecutable: true}, + {Name: "small.txt", Digest: smallDigest.GetProto()}, + }, + }, + } + treeData, err := proto.Marshal(tree) + require.NoError(t, err) + treeDigest := computeDigest(treeData) + + actionResult := &remoteexecution.ActionResult{ + OutputDirectories: []*remoteexecution.OutputDirectory{ + {Path: "build_output", TreeDigest: treeDigest.GetProto()}, + }, + ExitCode: 0, + } + + actionDigest := computeDigest(makeRandomData(t, 128, 0)) + + t.Run("CompleteActionResult", func(t *testing.T) { + _, casClient, acClient, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + require.NoError(t, batchUploadBlob(ctx, casClient, smallData, smallDigest)) + require.NoError(t, batchUploadBlob(ctx, casClient, bigData, bigDigest)) + require.NoError(t, batchUploadBlob(ctx, casClient, treeData, treeDigest)) + + err := updateActionResult(ctx, acClient, actionDigest, actionResult) + require.NoError(t, err) + + getResp, err := getActionResult(ctx, acClient, actionDigest) + require.NoError(t, err) + require.Equal(t, actionResult.ExitCode, getResp.ExitCode) + require.Len(t, getResp.OutputDirectories, 1) + require.Equal(t, treeDigest.GetProto().Hash, getResp.OutputDirectories[0].TreeDigest.Hash) + }) + + t.Run("IncompleteActionResult", func(t *testing.T) { + _, casClient, acClient, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Big data has not been uploaded in this test + require.NoError(t, batchUploadBlob(ctx, casClient, smallData, smallDigest)) + require.NoError(t, batchUploadBlob(ctx, casClient, treeData, treeDigest)) + + err = updateActionResult(ctx, acClient, actionDigest, actionResult) + require.NoError(t, err, "Unexpected error updating action result") + + _, err := getActionResult(ctx, acClient, actionDigest) + require.Error(t, err, "Incomplete action result should fail") + require.Equal(t, codes.NotFound, status.Code(err)) + }) +} + +func TestAPIAuthorizationRejections(t *testing.T) { + capClient, casClient, acClient, bsClient := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + forbiddenInstance := "forbidden_instance" + data := []byte("top secret data") + forbiddenDigest := computeDigestWithInstanceName(data, forbiddenInstance) + chunkDigest1 := computeDigestWithInstanceName([]byte("top "), forbiddenInstance) + chunkDigest2 := computeDigestWithInstanceName([]byte("secret data"), forbiddenInstance) + + dummyActionResult := &remoteexecution.ActionResult{ + ExitCode: 0, + } + + tests := []struct { + name string + call func() error + }{ + { + name: "GetCapabilities", + call: func() error { + _, err := capClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: forbiddenInstance, + }) + return err + }, + }, + { + name: "FindMissingBlobs", + call: func() error { + _, err := findMissingBlobs(ctx, casClient, []digest.Digest{forbiddenDigest}) + return err + }, + }, + { + name: "BatchUpdateBlobs", + call: func() error { + return batchUploadBlob(ctx, casClient, data, forbiddenDigest) + }, + }, + { + name: "BatchReadBlobs", + call: func() error { + _, err := batchDownloadBlobs(ctx, casClient, []digest.Digest{forbiddenDigest}) + return err + }, + }, + { + name: "SplitBlob", + call: func() error { + _, err := splitBlob(ctx, casClient, forbiddenDigest) + return err + }, + }, + { + name: "SpliceBlob", + call: func() error { + return spliceBlob(ctx, casClient, forbiddenDigest, []digest.Digest{chunkDigest1, chunkDigest2}) + }, + }, + { + name: "ByteStream Write", + call: func() error { + return bytestreamWriteBlob(ctx, bsClient, data, forbiddenDigest, remoteexecution.Compressor_IDENTITY) + }, + }, + { + name: "ByteStream Read", + call: func() error { + _, err := bytestreamReadBlob(ctx, bsClient, forbiddenDigest, remoteexecution.Compressor_IDENTITY) + return err + }, + }, + { + name: "ActionCache UpdateActionResult", + call: func() error { + return updateActionResult(ctx, acClient, forbiddenDigest, dummyActionResult) + }, + }, + { + name: "ActionCache GetActionResult", + call: func() error { + _, err := getActionResult(ctx, acClient, forbiddenDigest) + return err + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.call() + require.Error(t, err, "Expected API call to fail for forbidden instance") + require.Equal(t, codes.PermissionDenied, status.Code(err), "Expected PermissionDenied, got %v", status.Code(err)) + }) + } +} diff --git a/pkg/blobstore/grpcservers/integration/utils_test.go b/pkg/blobstore/grpcservers/integration/utils_test.go new file mode 100644 index 000000000..178c7f547 --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/utils_test.go @@ -0,0 +1,618 @@ +package integration + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math/rand" + "os" + "os/exec" + "strings" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/bytestream" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +type storageAddresses struct { + storageShardSocketPaths []string + localCacheSocketPath string +} + +const ( + maximumMessageSizeBytes = 2 << 20 // 2MiB + minChunkSizeBytes = 256 << 10 // 256KiB + bytestreamWriteChunkSizeBytes = 1 << 20 // 1MiB +) + +func storageConfig(listenSocketPath string) string { + return fmt.Sprintf(` +local listenPath = '%s'; +local maximumMessageSizeBytes = %d; +local minChunkSizeBytes = %d; + +local inMemoryStorage = { + keyLocationMapInMemory: { entries: 1024 * 1024 }, + keyLocationMapMaximumGetAttempts: 32, + keyLocationMapMaximumPutAttempts: 64, + oldBlocks: 1, + currentBlocks: 1, + newBlocks: 1, + blocksInMemory: { blockSizeBytes: 32 * 1024 * 1024 }, +}; + +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: maximumMessageSizeBytes, + contentAddressableStorageServer: { + contentAddressableStorage: { + chunkStorage: { + 'local': inMemoryStorage + { chunkingParameters: { minChunkSizeBytes: minChunkSizeBytes, horizonSizeBytes: 8*minChunkSizeBytes } }, + }, + chunkListStorage: { + 'local': inMemoryStorage, + }, + }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, + actionCache: { + backend: { 'local': inMemoryStorage }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + }, +} +`, escapeJSON(listenSocketPath), maximumMessageSizeBytes, minChunkSizeBytes) +} + +func replicatorConfig(listenSocketPath string, params storageAddresses) string { + upstreamsJSON, err := json.Marshal(params.storageShardSocketPaths) + if err != nil { + panic(err) + } + return fmt.Sprintf(` +local listenPath = '%s'; +local upstreamAddresses = %s; +local localCacheAddress = "unix:%s"; +local maximumMessageSizeBytes = %d; + +local topology = { + sharding: { + shards: { + ["shard" + i]: { backend: { grpc: { client: { address: "unix:" + upstreamAddresses[i] } } }, weight: 1 } + for i in std.range(0, std.length(upstreamAddresses) - 1) + } + } +}; + +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: maximumMessageSizeBytes, + source: topology, + sink: { grpc: { client: { address: localCacheAddress } } }, + replicator: { + deduplicating: { + concurrencyLimiting: { + base: { 'local': {} }, + maximumConcurrency: 1, + }, + }, + } +} +`, escapeJSON(listenSocketPath), upstreamsJSON, escapeJSON(params.localCacheSocketPath), maximumMessageSizeBytes) +} + +func frontendConfig(listenSocketPath string, params storageAddresses, replicatorSocketPath string) string { + upstreamsJSON, err := json.Marshal(params.storageShardSocketPaths) + if err != nil { + panic(err) + } + return fmt.Sprintf(` +local listenPath = '%s'; +local upstreamAddresses = %s; +local localCacheAddress = "unix:%s"; +local replicatorAddress = "unix:%s"; +local maximumMessageSizeBytes = %d; + +local topology = { + sharding: { + shards: { + ["shard" + i]: { + backend: { + grpc: { + client: { address: "unix:" + upstreamAddresses[i] }, + }, + }, + weight: 1, + } + for i in std.range(0, std.length(upstreamAddresses) - 1) + } + } +}; + +local readCaching(inner, replicator) = { + readCaching: { + fast: { grpc: { client: { address: localCacheAddress } } }, + slow: inner, + replicator: replicator, + }, +}; + +local authorizer = { + jmespathExpression: { + expression: 'contains(authenticationMetadata.private.mayAccess, instanceName)' + }, +}; + +local csReplicator = { remote: { address: replicatorAddress } }; +local simpleReplicator = { 'local': {} }; + +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { + allow: { + private: { + mayAccess: ['allowed_instance'], + }, + }, + }, + }], + supportedCompressors: ['ZSTD'], + zstdPool: { + maximumEncoders: 16, + maximumDecoders: 16, + encoderWindowSizeBytes: 8 * 1024 * 1024, + decoderWindowSizeBytes: 8 * 1024 * 1024, + }, + maximumMessageSizeBytes: maximumMessageSizeBytes, + contentAddressableStorageServer: { + contentAddressableStorage: { + chunkStorage: readCaching(topology, csReplicator), + chunkListStorage: readCaching({ chunkListValidating: { backend: topology } }, simpleReplicator), + cdcParameterCache: { + cacheSize: 1, + cacheDuration: '60s', + cacheReplacementPolicy: 'LEAST_RECENTLY_USED', + }, + }, + getAuthorizer: authorizer, + putAuthorizer: authorizer, + findMissingAuthorizer: authorizer, + }, + actionCache: { + backend: readCaching( + { + completenessChecking: { + backend: topology, + maximumTotalTreeSizeBytes: 64 * 1024 * 1024, + }, + }, + simpleReplicator, + ), + getAuthorizer: authorizer, + putAuthorizer: authorizer, + }, +} +`, escapeJSON(listenSocketPath), upstreamsJSON, escapeJSON(params.localCacheSocketPath), escapeJSON(replicatorSocketPath), maximumMessageSizeBytes) +} + +func writeConfigFile(name, content string) (file *os.File, err error) { + if file, err = os.CreateTemp("", name); err != nil { + return nil, err + } + if _, err = file.WriteString(content); err != nil { + return nil, err + } + if err = file.Close(); err != nil { + return nil, err + } + return file, nil +} + +func getBinaryPath(envVar string) (string, error) { + rf, err := runfiles.New() + if err != nil { + return "", util.StatusWrap(err, "Failed to initialize runfiles") + } + + runfilePath := os.Getenv(envVar) + if runfilePath == "" { + return "", util.StatusWrapf(err, "'%s' environment variable is not set", envVar) + } + + return rf.Rlocation(runfilePath) +} + +func setupServers(t *testing.T) (remoteexecution.CapabilitiesClient, remoteexecution.ContentAddressableStorageClient, remoteexecution.ActionCacheClient, bytestream.ByteStreamClient) { + storageBinaryPath, err := getBinaryPath("BB_STORAGE_RUNFILE_PATH") + require.NoError(t, err, "Could not get storage binary path") + replicatorBinaryPath, err := getBinaryPath("BB_REPLICATOR_RUNFILE_PATH") + require.NoError(t, err, "Could not get replicator binary path") + + storageSocketPaths := make([]string, 2) + for i := 0; i < 2; i++ { + storageSocketPaths[i] = createSocketPath(t) + setupServer(t, storageBinaryPath, fmt.Sprintf("storage%d", i), storageConfig(storageSocketPaths[i])) + require.Eventually(t, func() bool { + _, err := os.Stat(storageSocketPaths[i]) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Storage server %d did not start.", i) + } + + localCacheSocketPath := createSocketPath(t) + setupServer(t, storageBinaryPath, "localCache", storageConfig(localCacheSocketPath)) + require.Eventually(t, func() bool { + _, err := os.Stat(localCacheSocketPath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Local cache storage server did not start.") + + storageAddresses := storageAddresses{ + storageShardSocketPaths: storageSocketPaths, + localCacheSocketPath: localCacheSocketPath, + } + + replicatorSocketPath := createSocketPath(t) + setupServer(t, replicatorBinaryPath, "replicator", replicatorConfig(replicatorSocketPath, storageAddresses)) + require.Eventually(t, func() bool { + _, err := os.Stat(replicatorSocketPath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Replicator did not start.") + + frontendSocketPath := createSocketPath(t) + setupServer(t, storageBinaryPath, "frontend", frontendConfig(frontendSocketPath, storageAddresses, replicatorSocketPath)) + require.Eventually(t, func() bool { + _, err := os.Stat(frontendSocketPath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Frontend server did not start.") + + conn, err := grpc.NewClient(fmt.Sprintf("unix:%s", frontendSocketPath), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + + t.Cleanup(func() { + conn.Close() + }) + + return remoteexecution.NewCapabilitiesClient(conn), + remoteexecution.NewContentAddressableStorageClient(conn), + remoteexecution.NewActionCacheClient(conn), + bytestream.NewByteStreamClient(conn) +} + +func setupServer(t *testing.T, binaryPath, name, config string) { + configFile, err := writeConfigFile(name, config) + require.NoError(t, err, "Could not write config file") + + cmd := exec.Command(binaryPath, configFile.Name()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Start() + require.NoError(t, err) + + t.Cleanup(func() { + cmd.Process.Kill() + cmd.Wait() + os.Remove(configFile.Name()) + }) +} + +func createSocketPath(t *testing.T) string { + t.Helper() + socketFile, err := os.CreateTemp("", "bb_*.sock") + require.NoError(t, err) + socketPath := socketFile.Name() + socketFile.Close() + err = os.Remove(socketPath) + require.NoError(t, err) + return socketPath +} + +func computeDigestWithInstanceName(data []byte, instanceName string) digest.Digest { + hash := sha256.Sum256(data) + return digest.MustNewDigest( + instanceName, + remoteexecution.DigestFunction_SHA256, + hex.EncodeToString(hash[:]), + int64(len(data)), + ) +} + +func computeDigest(data []byte) digest.Digest { + return computeDigestWithInstanceName(data, "allowed_instance") +} + +func escapeJSON(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, "\"", "\\\"") + s = strings.ReplaceAll(s, "'", "\\'") + s = strings.ReplaceAll(s, "\n", "\\n") + return s +} + +func makeRandomData(t *testing.T, size int, seed int64) []byte { + t.Helper() + data := make([]byte, size) + r := rand.New(rand.NewSource(seed)) + _, err := r.Read(data) + require.NoError(t, err) + return data +} + +func zstdEncode(data []byte) []byte { + encoder, err := zstd.NewWriter(nil) + if err != nil { + panic(err) + } + defer encoder.Close() + return encoder.EncodeAll(data, nil) +} + +func zstdDecode(data []byte) ([]byte, error) { + decoder, err := zstd.NewReader(nil) + if err != nil { + panic(err) + } + defer decoder.Close() + ret, err := decoder.DecodeAll(data, nil) + if err != nil { + return nil, err + } + return ret, nil +} + +func batchUploadBlob(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, data []byte, d digest.Digest) error { + req := &remoteexecution.BatchUpdateBlobsRequest{ + InstanceName: d.GetInstanceName().String(), + Requests: []*remoteexecution.BatchUpdateBlobsRequest_Request{ + { + Digest: d.GetProto(), + Data: data, + }, + }, + } + + responses, err := client.BatchUpdateBlobs(ctx, req) + if err != nil { + return err + } + for _, response := range responses.Responses { + return status.ErrorProto(response.Status) + } + return err +} + +func batchDownloadBlobs(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, digests []digest.Digest) ([][]byte, error) { + if len(digests) == 0 { + return nil, nil + } + + dataMap := make(map[string][]byte, len(digests)) + + batchStart := 0 + for batchStart < len(digests) { + batchSize := int64(0) + batchEnd := batchStart + + // Slide the window forward until we hit our byte limit. We + // always include at least one digest, even if it exceeds the + // limit. + for batchEnd < len(digests) { + size := digests[batchEnd].GetSizeBytes() + if batchEnd > batchStart && batchSize+size > maximumMessageSizeBytes/2 { + break + } + batchSize += size + batchEnd++ + } + + // Prepare the batch request. + var batchProtos []*remoteexecution.Digest + for _, d := range digests[batchStart:batchEnd] { + batchProtos = append(batchProtos, d.GetProto()) + } + + req := &remoteexecution.BatchReadBlobsRequest{ + InstanceName: digests[0].GetInstanceName().String(), + Digests: batchProtos, + } + + // Execute the RPC for this specific batch. + res, err := client.BatchReadBlobs(ctx, req) + if err != nil { + return nil, err + } + + for _, r := range res.Responses { + if r.Status != nil && r.Status.Code != int32(codes.OK) { + return nil, status.ErrorProto(r.Status) + } + dataMap[r.Digest.Hash] = r.Data + } + + // Move the window forward for the next batch. + batchStart = batchEnd + } + + // Map the responses back to the requested order. + var downloadedData [][]byte + for _, d := range digests { + data, ok := dataMap[d.GetHashString()] + if !ok { + return nil, status.Errorf(codes.NotFound, "Digest %s was not returned in BatchReadBlobs response", d.GetHashString()) + } + downloadedData = append(downloadedData, data) + } + + return downloadedData, nil +} + +func findMissingBlobs(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, digests []digest.Digest) ([]digest.Digest, error) { + if len(digests) == 0 { + return nil, nil + } + + var digestProtos []*remoteexecution.Digest + for _, d := range digests { + digestProtos = append(digestProtos, d.GetProto()) + } + + req := &remoteexecution.FindMissingBlobsRequest{ + InstanceName: digests[0].GetInstanceName().String(), + BlobDigests: digestProtos, + } + + res, err := client.FindMissingBlobs(ctx, req) + if err != nil { + return nil, err + } + + var missingDigests []digest.Digest + digestFunction := digests[0].GetDigestFunction() + for _, p := range res.MissingBlobDigests { + d, err := digestFunction.NewDigestFromProto(p) + if err != nil { + return nil, util.StatusWrap(err, "Failed to parse missing digest from proto") + } + missingDigests = append(missingDigests, d) + } + + return missingDigests, nil +} + +func splitBlob(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, d digest.Digest) ([]digest.Digest, error) { + req := &remoteexecution.SplitBlobRequest{ + InstanceName: d.GetInstanceName().String(), + BlobDigest: d.GetProto(), + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + res, err := client.SplitBlob(ctx, req) + if err != nil { + return nil, err + } + + chunkDigests := make([]digest.Digest, 0, len(res.ChunkDigests)) + digestFunction := d.GetDigestFunction() + for _, chunkProto := range res.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) + if err != nil { + return nil, util.StatusWrap(err, "Failed to parse chunk digest from proto") + } + chunkDigests = append(chunkDigests, chunkDigest) + } + + return chunkDigests, nil +} + +func spliceBlob(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, blob digest.Digest, chunks []digest.Digest) error { + chunkDigests := make([]*remoteexecution.Digest, 0, len(chunks)) + for _, d := range chunks { + chunkDigests = append(chunkDigests, d.GetProto()) + } + + req := &remoteexecution.SpliceBlobRequest{ + InstanceName: blob.GetInstanceName().String(), + BlobDigest: blob.GetProto(), + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + _, err := client.SpliceBlob(ctx, req) + return err +} + +func bytestreamWriteBlob(ctx context.Context, client bytestream.ByteStreamClient, data []byte, digest digest.Digest, compressor remoteexecution.Compressor_Value) error { + writeStream, err := client.Write(ctx) + if err != nil { + return err + } + offset := int64(0) + dataSize := int64(len(data)) + + for offset < dataSize { + end := offset + bytestreamWriteChunkSizeBytes + if end > dataSize { + end = dataSize + } + + chunk := data[offset:end] + isLast := end == dataSize + + err := writeStream.Send(&bytestream.WriteRequest{ + ResourceName: digest.GetByteStreamWritePath(uuid.New(), compressor), + WriteOffset: offset, + FinishWrite: isLast, + Data: chunk, + }) + if err == io.EOF { + _, innerErr := writeStream.CloseAndRecv() + if innerErr != nil { + return util.StatusFromMultiple([]error{innerErr, err}) + } + } else if err != nil { + return err + } + offset = end + } + _, err = writeStream.CloseAndRecv() + return err +} + +func bytestreamReadBlob(ctx context.Context, client bytestream.ByteStreamClient, digest digest.Digest, compressor remoteexecution.Compressor_Value) ([]byte, error) { + readReq := &bytestream.ReadRequest{ + ResourceName: digest.GetByteStreamReadPath(compressor), + } + readStream, err := client.Read(ctx, readReq) + if err != nil { + return nil, err + } + defer readStream.CloseSend() + + receivedData := make([]byte, 0, digest.GetSizeBytes()) + for { + res, err := readStream.Recv() + if err == io.EOF { + return receivedData, nil + } else if err != nil { + return nil, err + } + receivedData = append(receivedData, res.Data...) + } +} + +func updateActionResult(ctx context.Context, acClient remoteexecution.ActionCacheClient, actionDigest digest.Digest, result *remoteexecution.ActionResult) error { + _, err := acClient.UpdateActionResult(ctx, &remoteexecution.UpdateActionResultRequest{ + InstanceName: actionDigest.GetInstanceName().String(), + ActionDigest: actionDigest.GetProto(), + ActionResult: result, + }) + return err +} + +func getActionResult(ctx context.Context, acClient remoteexecution.ActionCacheClient, actionDigest digest.Digest) (*remoteexecution.ActionResult, error) { + return acClient.GetActionResult(ctx, &remoteexecution.GetActionResultRequest{ + InstanceName: actionDigest.GetInstanceName().String(), + ActionDigest: actionDigest.GetProto(), + }) +} diff --git a/pkg/blobstore/hierarchical_instance_names_blob_access.go b/pkg/blobstore/hierarchical_instance_names_blob_access.go index 87fb5149c..87cadafd4 100644 --- a/pkg/blobstore/hierarchical_instance_names_blob_access.go +++ b/pkg/blobstore/hierarchical_instance_names_blob_access.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -42,21 +41,6 @@ func (ba *hierarchicalInstanceNamesBlobAccess) Get(ctx context.Context, digest d ) } -func (ba *hierarchicalInstanceNamesBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - parentDigests := parentDigest.GetDigestsWithParentInstanceNames() - childDigests := childDigest.GetDigestsWithParentInstanceNames() - return buffer.WithErrorHandler( - ba.BlobAccess.GetFromComposite(ctx, parentDigests[len(parentDigests)-1], childDigests[len(childDigests)-1], slicer), - &hierarchicalInstanceNamesGetFromCompositeErrorHandler{ - blobAccess: ba.BlobAccess, - context: ctx, - parentDigests: parentDigests, - childDigests: childDigests, - slicer: slicer, - }, - ) -} - func (ba *hierarchicalInstanceNamesBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { // Skip as much work as possible during the initial phase by // just requesting the original set of digests. This keeps the @@ -156,34 +140,3 @@ func (eh *hierarchicalInstanceNamesGetErrorHandler) OnError(err error) (buffer.B } func (hierarchicalInstanceNamesGetErrorHandler) Done() {} - -type hierarchicalInstanceNamesGetFromCompositeErrorHandler struct { - blobAccess BlobAccess - context context.Context - parentDigests []digest.Digest - childDigests []digest.Digest - slicer slicing.BlobSlicer -} - -func (eh *hierarchicalInstanceNamesGetFromCompositeErrorHandler) OnError(err error) (buffer.Buffer, error) { - if status.Code(err) != codes.NotFound { - // Serious error. Prepend the instance name, so that - // errors can be disambiguated. - return nil, util.StatusWrapf(err, "Instance name %#v", eh.parentDigests[len(eh.parentDigests)-1].GetInstanceName().String()) - } - if len(eh.parentDigests) == 1 { - // The object was found in none of the instance names. - // There is no need to prepend the instance name. - return nil, err - } - eh.parentDigests = eh.parentDigests[:len(eh.parentDigests)-1] - eh.childDigests = eh.childDigests[:len(eh.childDigests)-1] - return eh.blobAccess.GetFromComposite( - eh.context, - eh.parentDigests[len(eh.parentDigests)-1], - eh.childDigests[len(eh.childDigests)-1], - eh.slicer, - ), nil -} - -func (hierarchicalInstanceNamesGetFromCompositeErrorHandler) Done() {} diff --git a/pkg/blobstore/hierarchical_instance_names_blob_access_test.go b/pkg/blobstore/hierarchical_instance_names_blob_access_test.go index 493d359ba..8118d1a4d 100644 --- a/pkg/blobstore/hierarchical_instance_names_blob_access_test.go +++ b/pkg/blobstore/hierarchical_instance_names_blob_access_test.go @@ -73,65 +73,6 @@ func TestHierarchicalInstanceNamesBlobAccessGet(t *testing.T) { }) } -func TestHierarchicalInstanceNamesBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - blobAccess := blobstore.NewHierarchicalInstanceNamesBlobAccess(baseBlobAccess) - - helloDigest1 := digest.MustNewDigest("a/b", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - helloDigest2 := digest.MustNewDigest("a", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - helloDigest3 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - ellDigest1 := digest.MustNewDigest("a/b", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 5) - ellDigest2 := digest.MustNewDigest("a", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 5) - ellDigest3 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - t.Run("Failure", func(t *testing.T) { - // Errors from backends should be propagated. The - // instance name should be prepended to the error - // message, to disambiguate. - gomock.InOrder( - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest1, ellDigest1, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest2, ellDigest2, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Disk on fire"))), - ) - - _, err := blobAccess.GetFromComposite(ctx, helloDigest1, ellDigest1, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Instance name \"a\": Disk on fire"), err) - }) - - t.Run("NotFound", func(t *testing.T) { - // NotFound errors should cause requests to be retried - // against parent instance names. - gomock.InOrder( - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest1, ellDigest1, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest2, ellDigest2, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest3, ellDigest3, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - ) - - _, err := blobAccess.GetFromComposite(ctx, helloDigest1, ellDigest1, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Object not found"), err) - }) - - t.Run("Success", func(t *testing.T) { - gomock.InOrder( - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest1, ellDigest1, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest2, ellDigest2, slicer). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("ell"))), - ) - - data, err := blobAccess.GetFromComposite(ctx, helloDigest1, ellDigest1, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("ell"), data) - }) -} - func TestHierarchicalInstanceNamesBlobAccessFindMissing(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/local/BUILD.bazel b/pkg/blobstore/local/BUILD.bazel index b6f9ae1d7..e9c57aea0 100644 --- a/pkg/blobstore/local/BUILD.bazel +++ b/pkg/blobstore/local/BUILD.bazel @@ -72,7 +72,6 @@ go_test( "//internal/mock", "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/filesystem", "//pkg/filesystem/path", diff --git a/pkg/blobstore/local/flat_blob_access_test.go b/pkg/blobstore/local/flat_blob_access_test.go index a7d9a34d6..7b5f9b4a5 100644 --- a/pkg/blobstore/local/flat_blob_access_test.go +++ b/pkg/blobstore/local/flat_blob_access_test.go @@ -9,7 +9,6 @@ import ( "github.com/buildbarn/bb-storage/internal/mock" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/local" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" "github.com/stretchr/testify/require" @@ -191,284 +190,6 @@ func TestFlatBlobAccessGet(t *testing.T) { }) } -func TestFlatBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - keyLocationMap := mock.NewMockKeyLocationMap(ctrl) - locationBlobMap := mock.NewMockLocationBlobMap(ctrl) - capabilitiesProvider := mock.NewMockCapabilitiesProvider(ctrl) - blobAccess := local.NewFlatBlobAccess(keyLocationMap, locationBlobMap, digest.KeyWithoutInstance, &sync.RWMutex{}, "cas", capabilitiesProvider) - parentDigest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 11) - parentKey := local.NewKeyFromString("3-3e25960a79dbc69b674cd4ec67a72c62-11") - child1Digest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - child1Key := local.NewKeyFromString("3-8b1a9953c4611296a827abf8c47804d7-5") - child2Digest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "7d793037a0760186574b0282f2f435e7", 5) - child2Key := local.NewKeyFromString("3-7d793037a0760186574b0282f2f435e7-5") - slicer := mock.NewMockBlobSlicer(ctrl) - location1 := local.Location{ - BlockIndex: 7, - OffsetBytes: 42, - SizeBytes: 11, - } - location2 := local.Location{ - BlockIndex: 8, - OffsetBytes: 382, - SizeBytes: 5, - } - - t.Run("NoSlicingParentNotFound", func(t *testing.T) { - // Even if the requested child object were to exist, the - // existence of the parent object is the one that - // counts. If the parent object does not exist, we - // should report it as being absent, so that it may be - // replicated/regenerated. - keyLocationMap.EXPECT().Get(parentKey). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Blob not found"), err) - }) - - t.Run("NoSlicingChildFailure", func(t *testing.T) { - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.Internal, "I/O error")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "I/O error"), err) - }) - - t.Run("NoSlicingSuccess", func(t *testing.T) { - // If both the parent doesn't need to be refreshed and - // the child exists, we can return data immediately. - // Don't bother taking into account whether the child - // needs to be refreshed. We'll just slice the object - // again by the time the child gets lost. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(location2, nil) - childGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location2). - Return(childGetter.Call, true) - childGetter.EXPECT().Call(child1Digest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - t.Run("SlicingParentNotFound", func(t *testing.T) { - // An initial lookup on the parent returned success, but - // when retrying the lookup while holding an exclusive - // lock, we got NotFound. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - getter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(getter.Call, true) - keyLocationMap.EXPECT().Get(parentKey). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Blob not found"), err) - }) - - t.Run("SlicingChildFailure", func(t *testing.T) { - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.Internal, "I/O error")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "I/O error"), err) - }) - - t.Run("SlicingButNoLongerNeeded", func(t *testing.T) { - // An initial lookup indicated that the parent needed to - // be sliced, but a second lookup while holding an - // exclusive lock showed that this is no longer needed. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, true) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(location2, nil) - childGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location2). - Return(childGetter.Call, true) - childGetter.EXPECT().Call(child1Digest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - t.Run("SlicingPutFailure", func(t *testing.T) { - // Successfully sliced an object, but we failed to - // insert new key-location map entries for the resulting - // slices. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - parentGetter2.EXPECT().Call(parentDigest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - slicer.EXPECT().Slice(gomock.Any(), child1Digest).DoAndReturn(func(b buffer.Buffer, child1Digest digest.Digest) (buffer.Buffer, []slicing.BlobSlice) { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), []slicing.BlobSlice{ - {Digest: child1Digest, OffsetBytes: 0, SizeBytes: 5}, - {Digest: child2Digest, OffsetBytes: 6, SizeBytes: 5}, - } - }) - keyLocationMap.EXPECT().Put(child1Key, local.Location{ - BlockIndex: 7, - OffsetBytes: 42, - SizeBytes: 5, - }).Return(status.Error(codes.Internal, "I/O error")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Failed to create child blob \"3-8b1a9953c4611296a827abf8c47804d7-5-example\": I/O error"), err) - }) - - t.Run("SlicingSuccess", func(t *testing.T) { - // Successful instance where an object is sliced, and the - // contents of a single slice is returned. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - parentGetter2.EXPECT().Call(parentDigest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - slicer.EXPECT().Slice(gomock.Any(), child1Digest).DoAndReturn(func(b buffer.Buffer, child1Digest digest.Digest) (buffer.Buffer, []slicing.BlobSlice) { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), []slicing.BlobSlice{ - {Digest: child1Digest, OffsetBytes: 0, SizeBytes: 5}, - {Digest: child2Digest, OffsetBytes: 6, SizeBytes: 5}, - } - }) - keyLocationMap.EXPECT().Put(child1Key, local.Location{ - BlockIndex: 7, - OffsetBytes: 42, - SizeBytes: 5, - }) - keyLocationMap.EXPECT().Put(child2Key, local.Location{ - BlockIndex: 7, - OffsetBytes: 48, - SizeBytes: 5, - }) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - t.Run("RefreshSuccess", func(t *testing.T) { - // Successful instance where an object is both refreshed and - // sliced, and the contents of a single slice is returned. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, true) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, true) - parentGetter2.EXPECT().Call(parentDigest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - putWriter := mock.NewMockLocationBlobPutWriter(ctrl) - locationBlobMap.EXPECT().Put(int64(11)). - Return(putWriter.Call, nil) - putFinalizer := mock.NewMockLocationBlobPutFinalizer(ctrl) - putWriter.EXPECT().Call(gomock.Any()).DoAndReturn(func(b buffer.Buffer) local.LocationBlobPutFinalizer { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return putFinalizer.Call - }) - putFinalizer.EXPECT().Call(). - Return(location2, nil) - keyLocationMap.EXPECT().Put(parentKey, location2) - slicer.EXPECT().Slice(gomock.Any(), child1Digest).DoAndReturn(func(b buffer.Buffer, child1Digest digest.Digest) (buffer.Buffer, []slicing.BlobSlice) { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), []slicing.BlobSlice{ - {Digest: child1Digest, OffsetBytes: 0, SizeBytes: 5}, - {Digest: child2Digest, OffsetBytes: 6, SizeBytes: 5}, - } - }) - keyLocationMap.EXPECT().Put(child1Key, local.Location{ - BlockIndex: 8, - OffsetBytes: 382, - SizeBytes: 5, - }) - keyLocationMap.EXPECT().Put(child2Key, local.Location{ - BlockIndex: 8, - OffsetBytes: 388, - SizeBytes: 5, - }) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - // TODO: Some testing coverage may be missing for the cases - // where refreshing fails. -} - func TestFlatBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/metrics_blob_access.go b/pkg/blobstore/metrics_blob_access.go index 0941156bc..dc1f4a2e8 100644 --- a/pkg/blobstore/metrics_blob_access.go +++ b/pkg/blobstore/metrics_blob_access.go @@ -7,7 +7,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -113,23 +112,6 @@ func (ba *metricsBlobAccess) Get(ctx context.Context, digest digest.Digest) buff return b } -func (ba *metricsBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - timeStart := ba.clock.Now() - b := buffer.WithErrorHandler( - ba.blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer), - &metricsErrorHandler{ - blobAccess: ba, - timeStart: timeStart, - errorCode: codes.OK, - durationSeconds: ba.getFromCompositeDurationSeconds, - }, - ) - if sizeBytes, err := b.GetSizeBytes(); err == nil { - ba.getFromCompositeBlobSizeBytes.Observe(float64(sizeBytes)) - } - return b -} - func (ba *metricsBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { // If the Buffer is in a known error state, return the error // here instead of propagating the error to the underlying diff --git a/pkg/blobstore/mirrored/BUILD.bazel b/pkg/blobstore/mirrored/BUILD.bazel index b037f64cc..b9da8b276 100644 --- a/pkg/blobstore/mirrored/BUILD.bazel +++ b/pkg/blobstore/mirrored/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//pkg/blobstore", "//pkg/blobstore/buffer", "//pkg/blobstore/replication", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/util", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/mirrored/mirrored_blob_access.go b/pkg/blobstore/mirrored/mirrored_blob_access.go index 377aa31d7..24a91e5c7 100644 --- a/pkg/blobstore/mirrored/mirrored_blob_access.go +++ b/pkg/blobstore/mirrored/mirrored_blob_access.go @@ -9,7 +9,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/replication" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" "github.com/prometheus/client_golang/prometheus" @@ -107,11 +106,6 @@ func (ba *mirroredBlobAccess) Get(ctx context.Context, digest digest.Digest) buf return replication.GetWithBlobReplicator(ctx, digest, firstBackend, successiveBackends) } -func (ba *mirroredBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - firstBackend, successiveBackends := ba.getBlobReplicatorSelector() - return replication.GetFromCompositeWithBlobReplicator(ctx, parentDigest, childDigest, slicer, firstBackend, successiveBackends) -} - func (ba *mirroredBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { // Store object in both storage backends. b1, b2 := b.CloneStream() diff --git a/pkg/blobstore/mirrored/mirrored_blob_access_test.go b/pkg/blobstore/mirrored/mirrored_blob_access_test.go index db67d60aa..89683b4ed 100644 --- a/pkg/blobstore/mirrored/mirrored_blob_access_test.go +++ b/pkg/blobstore/mirrored/mirrored_blob_access_test.go @@ -88,31 +88,6 @@ func TestMirroredBlobAccessGet(t *testing.T) { }) } -func TestMirroredBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - backendA := mock.NewMockBlobAccess(ctrl) - backendB := mock.NewMockBlobAccess(ctrl) - replicatorAToB := mock.NewMockBlobReplicator(ctrl) - replicatorBToA := mock.NewMockBlobReplicator(ctrl) - parentDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "834c514174f3a7d5952dfa68d4b657f3c4cf78b3973dcf2721731c3861559828", 100) - childDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c", 11) - slicer := mock.NewMockBlobSlicer(ctrl) - - // We assume that tests for Get() provides coverage for other - // scenarios. - - t.Run("RepairSuccess", func(t *testing.T) { - backendA.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found"))) - replicatorBToA.EXPECT().ReplicateComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - - blobAccess := mirrored.NewMirroredBlobAccess(backendA, backendB, replicatorAToB, replicatorBToA) - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - }) -} - func TestMirroredBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/read_canarying_blob_access.go b/pkg/blobstore/read_canarying_blob_access.go index f01361b07..895366d6e 100644 --- a/pkg/blobstore/read_canarying_blob_access.go +++ b/pkg/blobstore/read_canarying_blob_access.go @@ -6,7 +6,6 @@ import ( "time" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/eviction" @@ -139,26 +138,6 @@ func (ba *readCanaryingBlobAccess) Get(ctx context.Context, d digest.Digest) buf ) } -func (ba *readCanaryingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - instanceNameStr := parentDigest.GetInstanceName().String() - if ba.shouldSendToReplica(instanceNameStr) { - return buffer.WithErrorHandler( - ba.replica.GetFromComposite(ctx, parentDigest, childDigest, slicer), - &readCanaryingReplicaGetFromCompositeErrorHandler{ - blobAccess: ba, - context: ctx, - parentDigest: parentDigest, - childDigest: childDigest, - slicer: slicer, - }, - ) - } - return buffer.WithErrorHandler( - ba.BlobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer), - readCanaryingSourceErrorHandler{}, - ) -} - func (ba *readCanaryingBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { // The backend may behave differently based on the REv2 instance // name that is, for example if AuthorizingBlobAccess and @@ -226,39 +205,6 @@ func (eh *readCanaryingReplicaGetErrorHandler) Done() { } } -// readCanaryingReplicaGetFromCompositeErrorHandler is the ErrorHandler -// that is attached to all buffers read from the replica backend through -// the GetFromComposite() operation. -type readCanaryingReplicaGetFromCompositeErrorHandler struct { - blobAccess *readCanaryingBlobAccess - context context.Context - parentDigest digest.Digest - childDigest digest.Digest - slicer slicing.BlobSlicer -} - -func (eh *readCanaryingReplicaGetFromCompositeErrorHandler) OnError(err error) (buffer.Buffer, error) { - ba := eh.blobAccess - if ba == nil { - // Already retried the operation against the source backend. - return nil, util.StatusWrap(err, "Source") - } - eh.blobAccess = nil - if ba.recordReplicaResponse(eh.parentDigest.GetInstanceName().String(), err) { - // Request against the replica failed with an - // infrastructure error. Retry it against the source - // backend. - return ba.BlobAccess.GetFromComposite(eh.context, eh.parentDigest, eh.childDigest, eh.slicer), nil - } - return nil, util.StatusWrap(err, "Replica") -} - -func (eh *readCanaryingReplicaGetFromCompositeErrorHandler) Done() { - if ba := eh.blobAccess; ba != nil { - ba.recordReplicaResponse(eh.parentDigest.GetInstanceName().String(), nil) - } -} - // readCanaryingSourceErrorHandler is the ErrorHandler that is attached // to all buffers read from the source backend. type readCanaryingSourceErrorHandler struct{} diff --git a/pkg/blobstore/read_canarying_blob_access_test.go b/pkg/blobstore/read_canarying_blob_access_test.go index 6e89ac037..9795743e8 100644 --- a/pkg/blobstore/read_canarying_blob_access_test.go +++ b/pkg/blobstore/read_canarying_blob_access_test.go @@ -113,34 +113,6 @@ func TestReadCanaryingBlobAccess(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.Unavailable, "Source: Server is offline"), err) }) - t.Run("GetFromComposite", func(t *testing.T) { - // Don't provide exhaustive testing coverage for - // GetFromComposite(), as most of the logic is shared - // with Get(). Just test that traffic is capable of - // going to both backends. - parentDigest := digest.MustNewDigest("get-from-composite", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 100) - childDigest := digest.MustNewDigest("get-from-composite", remoteexecution.DigestFunction_MD5, "80cd354fb9a929ffad1b059d909b3b69", 10) - slicer := mock.NewMockBlobSlicer(ctrl) - - clock.EXPECT().Now().Return(time.Unix(15000, 0)) - replicaBackend.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Unavailable, "Server is offline"))) - replicaErrorLogger.EXPECT().Log(status.Error(codes.Unavailable, "Server is offline")) - clock.EXPECT().Now().Return(time.Unix(15000, 100000000)) - sourceBackend.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Unavailable, "Server is also offline"))) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Unavailable, "Source: Server is also offline"), err) - - clock.EXPECT().Now().Return(time.Unix(15001, 0)) - sourceBackend.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Unavailable, "Server is offline"))) - - _, err = blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Unavailable, "Source: Server is offline"), err) - }) - t.Run("FindMissing", func(t *testing.T) { // Let a FindMissing() call against the replica fail for a // given REv2 instance name. This should cause it to be diff --git a/pkg/blobstore/readcaching/BUILD.bazel b/pkg/blobstore/readcaching/BUILD.bazel index be443ce6c..b9f07c688 100644 --- a/pkg/blobstore/readcaching/BUILD.bazel +++ b/pkg/blobstore/readcaching/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//pkg/blobstore", "//pkg/blobstore/buffer", "//pkg/blobstore/replication", - "//pkg/blobstore/slicing", "//pkg/digest", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", diff --git a/pkg/blobstore/readcaching/read_caching_blob_access.go b/pkg/blobstore/readcaching/read_caching_blob_access.go index 7f4ff14bb..1be201e05 100644 --- a/pkg/blobstore/readcaching/read_caching_blob_access.go +++ b/pkg/blobstore/readcaching/read_caching_blob_access.go @@ -6,7 +6,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/replication" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "google.golang.org/grpc/codes" @@ -52,14 +51,3 @@ func (ba *readCachingBlobAccess) Get(ctx context.Context, digest digest.Digest) ba.getBlobReplicatorSelector(), ) } - -func (ba *readCachingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return replication.GetFromCompositeWithBlobReplicator( - ctx, - parentDigest, - childDigest, - slicer, - ba.fast, - ba.getBlobReplicatorSelector(), - ) -} diff --git a/pkg/blobstore/readcaching/read_caching_blob_access_test.go b/pkg/blobstore/readcaching/read_caching_blob_access_test.go index 17f1e2c2c..bd01fd879 100644 --- a/pkg/blobstore/readcaching/read_caching_blob_access_test.go +++ b/pkg/blobstore/readcaching/read_caching_blob_access_test.go @@ -66,30 +66,6 @@ func TestReadCachingBlobAccessGet(t *testing.T) { }) } -func TestReadCachingBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - slowBlobAccess := mock.NewMockBlobAccess(ctrl) - fastBlobAccess := mock.NewMockBlobAccess(ctrl) - blobReplicator := mock.NewMockBlobReplicator(ctrl) - blobAccess := readcaching.NewReadCachingBlobAccess(slowBlobAccess, fastBlobAccess, blobReplicator) - parentDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "834c514174f3a7d5952dfa68d4b657f3c4cf78b3973dcf2721731c3861559828", 100) - childDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c", 11) - slicer := mock.NewMockBlobSlicer(ctrl) - - // We assume that tests for Get() provides coverage for other - // scenarios. - - t.Run("Slow", func(t *testing.T) { - fastBlobAccess.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found"))) - blobReplicator.EXPECT().ReplicateComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - }) -} - func TestReadCachingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/readfallback/BUILD.bazel b/pkg/blobstore/readfallback/BUILD.bazel index 3a111a54d..7ded42327 100644 --- a/pkg/blobstore/readfallback/BUILD.bazel +++ b/pkg/blobstore/readfallback/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//pkg/blobstore", "//pkg/blobstore/buffer", "//pkg/blobstore/replication", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/util", "@org_golang_google_grpc//codes", diff --git a/pkg/blobstore/readfallback/read_fallback_blob_access.go b/pkg/blobstore/readfallback/read_fallback_blob_access.go index 39f4bac89..ff81f793f 100644 --- a/pkg/blobstore/readfallback/read_fallback_blob_access.go +++ b/pkg/blobstore/readfallback/read_fallback_blob_access.go @@ -6,7 +6,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/replication" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -67,17 +66,6 @@ func (ba *readFallbackBlobAccess) Get(ctx context.Context, digest digest.Digest) ) } -func (ba *readFallbackBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return replication.GetFromCompositeWithBlobReplicator( - ctx, - parentDigest, - childDigest, - slicer, - ba.BlobAccess, - ba.getBlobReplicatorSelector(), - ) -} - func (ba *readFallbackBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { // Call FindMissing() on the backends sequentially, as opposed // to calling them concurrently and merging the results. In the diff --git a/pkg/blobstore/readfallback/read_fallback_blob_access_test.go b/pkg/blobstore/readfallback/read_fallback_blob_access_test.go index 63717a51c..4e7642197 100644 --- a/pkg/blobstore/readfallback/read_fallback_blob_access_test.go +++ b/pkg/blobstore/readfallback/read_fallback_blob_access_test.go @@ -88,32 +88,6 @@ func TestReadFallbackBlobAccessGet(t *testing.T) { }) } -func TestReadFallbackBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - primary := mock.NewMockBlobAccess(ctrl) - secondary := mock.NewMockBlobAccess(ctrl) - replicator := mock.NewMockBlobReplicator(ctrl) - blobAccess := readfallback.NewReadFallbackBlobAccess(primary, secondary, replicator) - parentDigest := digest.MustNewDigest("instance", remoteexecution.DigestFunction_MD5, "d20fb8dfa347cf895b38649410aeb3f8", 100) - childDigest := digest.MustNewDigest("instance", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - // We assume that tests for Get() provides coverage for other - // scenarios. - - t.Run("SecondarySuccess", func(t *testing.T) { - primary.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))) - replicator.EXPECT().ReplicateComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) -} - func TestReadFallbackBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/referenceexpanding/BUILD.bazel b/pkg/blobstore/referenceexpanding/BUILD.bazel new file mode 100644 index 000000000..1c3c22586 --- /dev/null +++ b/pkg/blobstore/referenceexpanding/BUILD.bazel @@ -0,0 +1,49 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "referenceexpanding", + srcs = ["reference_expanding_blob_access.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/referenceexpanding", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/blobstore/slicing", + "//pkg/cas", + "//pkg/cloud/aws", + "//pkg/cloud/gcp", + "//pkg/digest", + "//pkg/proto/icas", + "//pkg/util", + "//pkg/zstd", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_aws_aws_sdk_go_v2//aws", + "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) + +go_test( + name = "referenceexpanding_test", + srcs = ["reference_expanding_blob_access_test.go"], + deps = [ + ":referenceexpanding", + "//internal/mock", + "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/digest", + "//pkg/proto/icas", + "//pkg/testutil", + "//pkg/zstd", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_aws_aws_sdk_go_v2//aws", + "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", + "@com_github_aws_aws_sdk_go_v2_service_s3//types", + "@com_github_klauspost_compress//zstd", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_uber_go_mock//gomock", + ], +) diff --git a/pkg/blobstore/reference_expanding_blob_access.go b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go similarity index 91% rename from pkg/blobstore/reference_expanding_blob_access.go rename to pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go index 340073a6b..c66d0b787 100644 --- a/pkg/blobstore/reference_expanding_blob_access.go +++ b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go @@ -1,4 +1,4 @@ -package blobstore +package referenceexpanding import ( "compress/flate" @@ -11,8 +11,10 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" + "github.com/buildbarn/bb-storage/pkg/cas" cloud_aws "github.com/buildbarn/bb-storage/pkg/cloud/aws" cloud_gcp "github.com/buildbarn/bb-storage/pkg/cloud/gcp" "github.com/buildbarn/bb-storage/pkg/digest" @@ -25,8 +27,8 @@ import ( ) type referenceExpandingBlobAccess struct { - indirectContentAddressableStorage BlobAccess - contentAddressableStorage BlobAccess + indirectContentAddressableStorage blobstore.BlobAccess + contentAddressableStorage cas.ContentAddressableStorage httpClient *http.Client s3Client cloud_aws.S3Client gcsClient cloud_gcp.StorageClient @@ -48,7 +50,7 @@ func getHTTPRangeHeader(reference *icas.Reference) string { // Storage (CAS) backend. Any object requested through this BlobAccess // will cause its reference to be loaded from the ICAS, followed by // fetching its data from the referenced location. -func NewReferenceExpandingBlobAccess(indirectContentAddressableStorage, contentAddressableStorage BlobAccess, httpClient *http.Client, s3Client cloud_aws.S3Client, gcsClient cloud_gcp.StorageClient, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) BlobAccess { +func NewReferenceExpandingBlobAccess(indirectContentAddressableStorage blobstore.BlobAccess, contentAddressableStorage cas.ContentAddressableStorage, httpClient *http.Client, s3Client cloud_aws.S3Client, gcsClient cloud_gcp.StorageClient, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { return &referenceExpandingBlobAccess{ indirectContentAddressableStorage: indirectContentAddressableStorage, contentAddressableStorage: contentAddressableStorage, @@ -135,17 +137,10 @@ func (ba *referenceExpandingBlobAccess) Get(ctx context.Context, blobDigest dige return buffer.NewBufferFromError(util.StatusWrapWithCode(err, codes.Internal, "Invalid digest")) } - b := ba.contentAddressableStorage.Get(ctx, referenceDigest) - if reference.Decompressor == remoteexecution.Compressor_IDENTITY { - // Optimize the fast path: if no transformations are - // being performed and the digests are identical, we - // can pass through the underlying buffer directly. - instanceNamePatcher := digest.NewInstanceNamePatcher(referenceDigest.GetInstanceName(), blobDigest.GetInstanceName()) - if blobDigest == instanceNamePatcher.PatchDigest(referenceDigest) { - return b - } + r, err = cas.GetReadCloser(ctx, ba.contentAddressableStorage, referenceDigest) + if err != nil { + return buffer.NewBufferFromError(err) } - r = b.ToReader() default: return buffer.NewBufferFromError(status.Error(codes.Unimplemented, "Reference uses an unsupported medium")) } diff --git a/pkg/blobstore/reference_expanding_blob_access_test.go b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access_test.go similarity index 93% rename from pkg/blobstore/reference_expanding_blob_access_test.go rename to pkg/blobstore/referenceexpanding/reference_expanding_blob_access_test.go index 7728da6a5..11cc47888 100644 --- a/pkg/blobstore/reference_expanding_blob_access_test.go +++ b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access_test.go @@ -1,4 +1,4 @@ -package blobstore_test +package referenceexpanding_test import ( "bytes" @@ -13,8 +13,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" - "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/referenceexpanding" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/icas" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -32,11 +33,11 @@ func TestReferenceExpandingBlobAccessGet(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) indirectContentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) roundTripper := mock.NewMockRoundTripper(ctrl) s3Client := mock.NewMockS3Client(ctrl) gcsClient := mock.NewMockStorageClient(ctrl) - blobAccess := blobstore.NewReferenceExpandingBlobAccess( + blobAccess := referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage, contentAddressableStorage, &http.Client{Transport: roundTripper}, @@ -467,10 +468,14 @@ func TestReferenceExpandingBlobAccessGet(t *testing.T) { buffer.BackendProvided(buffer.Irreparable(helloDigest)), ), ) - contentAddressableStorage.EXPECT().Get( + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("instance/name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + contentAddressableStorage.EXPECT().FetchChunk( ctx, digest.MustNewDigest("instance/name", remoteexecution.DigestFunction_SHA256, "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", 5), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) + ).Return([]byte("Hello"), nil) data, err := blobAccess.Get(ctx, helloDigest).ToByteSlice(10) require.NoError(t, err) @@ -497,16 +502,21 @@ func TestReferenceExpandingBlobAccessGet(t *testing.T) { buffer.BackendProvided(buffer.Irreparable(aaaDigest)), ), ) - contentAddressableStorage.EXPECT().Get( - ctx, - digest.MustNewDigest("instance/name", remoteexecution.DigestFunction_SHA1, "a6cf72f4c7f42afde230ac461d5c7b9e25838530", 21), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte{ + data := []byte{ // 21 bytes of Zstandard compressed data // that decompress to fifty 'a' bytes. 0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x58, 0x45, 0x00, 0x00, 0x10, 0x61, 0x61, 0x01, 0x00, 0x45, 0x00, 0x0b, 0x23, 0x9f, 0x0f, 0x9a, - })) + } + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("instance/name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + contentAddressableStorage.EXPECT().FetchChunk( + ctx, + digest.MustNewDigest("instance/name", remoteexecution.DigestFunction_SHA1, "a6cf72f4c7f42afde230ac461d5c7b9e25838530", 21), + ).Return(data, nil) data, err := blobAccess.Get(ctx, aaaDigest).ToByteSlice(100) require.NoError(t, err) @@ -518,11 +528,11 @@ func TestReferenceExpandingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) indirectContentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) roundTripper := mock.NewMockRoundTripper(ctrl) s3Client := mock.NewMockS3Client(ctrl) gcsClient := mock.NewMockStorageClient(ctrl) - blobAccess := blobstore.NewReferenceExpandingBlobAccess( + blobAccess := referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage, contentAddressableStorage, &http.Client{Transport: roundTripper}, @@ -557,11 +567,11 @@ func TestReferenceExpandingBlobAccessFindMissing(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) indirectContentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) roundTripper := mock.NewMockRoundTripper(ctrl) s3Client := mock.NewMockS3Client(ctrl) gcsClient := mock.NewMockStorageClient(ctrl) - blobAccess := blobstore.NewReferenceExpandingBlobAccess( + blobAccess := referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage, contentAddressableStorage, &http.Client{Transport: roundTripper}, @@ -598,3 +608,11 @@ func TestReferenceExpandingBlobAccessFindMissing(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Network error"), err) }) } + +func mustNewInstanceName(name string) digest.InstanceName { + instanceName, err := digest.NewInstanceName(name) + if err != nil { + panic(err) + } + return instanceName +} diff --git a/pkg/blobstore/replication/BUILD.bazel b/pkg/blobstore/replication/BUILD.bazel index 6c362b07b..3020382d6 100644 --- a/pkg/blobstore/replication/BUILD.bazel +++ b/pkg/blobstore/replication/BUILD.bazel @@ -20,7 +20,6 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/slicing", "//pkg/clock", "//pkg/digest", "//pkg/proto/replicator", diff --git a/pkg/blobstore/replication/blob_replicator.go b/pkg/blobstore/replication/blob_replicator.go index e302be13e..950f8d03f 100644 --- a/pkg/blobstore/replication/blob_replicator.go +++ b/pkg/blobstore/replication/blob_replicator.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -20,9 +19,6 @@ type BlobReplicator interface { // Replicate a single object between backends, while at the same // time giving a handle back to it. ReplicateSingle(ctx context.Context, digest digest.Digest) buffer.Buffer - // Replicate a single composite object between backends, while - // at the same time giving a handle back to one of its children. - ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer // Replicate a set of objects between backends. ReplicateMultiple(ctx context.Context, digests digest.Set) error } diff --git a/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go b/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go index ce8bfdf96..096da5bc3 100644 --- a/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go +++ b/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -47,16 +46,6 @@ func (br *concurrencyLimitingBlobReplicator) ReplicateSingle(ctx context.Context ) } -func (br *concurrencyLimitingBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - br.sink.GetFromComposite(ctx, parentDigest, childDigest, slicer), - notFoundToInternalErrorHandler{}, - ) -} - func (br *concurrencyLimitingBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { if err := util.AcquireSemaphore(ctx, br.semaphore, 1); err != nil { return err diff --git a/pkg/blobstore/replication/deduplicating_blob_replicator.go b/pkg/blobstore/replication/deduplicating_blob_replicator.go index bd219c114..cdd41b9cc 100644 --- a/pkg/blobstore/replication/deduplicating_blob_replicator.go +++ b/pkg/blobstore/replication/deduplicating_blob_replicator.go @@ -6,7 +6,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -64,16 +63,6 @@ func (br *deduplicatingBlobReplicator) ReplicateSingle(ctx context.Context, blob ) } -func (br *deduplicatingBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - br.sink.GetFromComposite(ctx, parentDigest, childDigest, slicer), - notFoundToInternalErrorHandler{}, - ) -} - func (br *deduplicatingBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { NextDigest: for _, digest := range digests.Items() { diff --git a/pkg/blobstore/replication/deduplicating_blob_replicator_test.go b/pkg/blobstore/replication/deduplicating_blob_replicator_test.go index ca747d602..99806e2d5 100644 --- a/pkg/blobstore/replication/deduplicating_blob_replicator_test.go +++ b/pkg/blobstore/replication/deduplicating_blob_replicator_test.go @@ -146,35 +146,6 @@ func TestDeduplicatingBlobReplicatorReplicateSingle(t *testing.T) { }) } -func TestDeduplicatingBlobReplicatorReplicateComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - base := mock.NewMockBlobReplicator(ctrl) - sink := mock.NewMockBlobAccess(ctrl) - replicator := replication.NewDeduplicatingBlobReplicator(base, sink, digest.KeyWithoutInstance) - - parentDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 11) - parentDigestSet := parentDigest.ToSingletonSet() - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - // Only a single test for the success case is provided, as the - // tests for ReplicateSingle() provide enough coverage. - - t.Run("SuccessReplication", func(t *testing.T) { - // If the sink reports the parent as absent, we should - // see a replication take place before reading the child - // blob from the sink. - sink.EXPECT().FindMissing(ctx, parentDigestSet).Return(parentDigestSet, nil) - base.EXPECT().ReplicateMultiple(ctx, parentDigestSet) - sink.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) -} - func TestDeduplicatingBlobReplicatorReplicateMultiple(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/replication/local_blob_replicator.go b/pkg/blobstore/replication/local_blob_replicator.go index c7611b6df..7d33b7827 100644 --- a/pkg/blobstore/replication/local_blob_replicator.go +++ b/pkg/blobstore/replication/local_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -42,20 +41,6 @@ func (br *localBlobReplicator) ReplicateSingle(ctx context.Context, digest diges }) } -func (br *localBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - // First replicate the object to the sink, and then read it back - // again. Even though cloning like in ReplicateSingle() is more - // optimal in terms of bandwidth, it would require us to do the - // slicing here. This may cause slicing information to get lost. - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - br.sink.GetFromComposite(ctx, parentDigest, childDigest, slicer), - notFoundToInternalErrorHandler{}, - ) -} - func (br *localBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { for _, blobDigest := range digests.Items() { if err := br.sink.Put(ctx, blobDigest, br.source.Get(ctx, blobDigest)); err != nil { diff --git a/pkg/blobstore/replication/metrics_blob_replicator.go b/pkg/blobstore/replication/metrics_blob_replicator.go index bd6a9fb27..1f5bac952 100644 --- a/pkg/blobstore/replication/metrics_blob_replicator.go +++ b/pkg/blobstore/replication/metrics_blob_replicator.go @@ -6,7 +6,6 @@ import ( "time" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -120,23 +119,6 @@ func (r *metricsBlobReplicator) ReplicateSingle(ctx context.Context, blobDigest return b } -func (r *metricsBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - timeStart := r.clock.Now() - b := buffer.WithErrorHandler( - r.replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer), - &metricsErrorHandler{ - replicator: r, - timeStart: timeStart, - errorCode: codes.OK, - durationSeconds: r.compositeDurationSeconds, - }, - ) - if sizeBytes, err := b.GetSizeBytes(); err == nil { - r.compositeBlobSizeBytes.Observe(float64(sizeBytes)) - } - return b -} - func (r *metricsBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { if digests.Empty() { return nil diff --git a/pkg/blobstore/replication/noop_blob_replicator.go b/pkg/blobstore/replication/noop_blob_replicator.go index 5e5951da1..11f2df240 100644 --- a/pkg/blobstore/replication/noop_blob_replicator.go +++ b/pkg/blobstore/replication/noop_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -27,10 +26,6 @@ func (br noopBlobReplicator) ReplicateSingle(ctx context.Context, digest digest. return br.source.Get(ctx, digest) } -func (br noopBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return br.source.GetFromComposite(ctx, parentDigest, childDigest, slicer) -} - func (noopBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { return nil } diff --git a/pkg/blobstore/replication/queued_blob_replicator.go b/pkg/blobstore/replication/queued_blob_replicator.go index 4b9431a5b..0e099a67b 100644 --- a/pkg/blobstore/replication/queued_blob_replicator.go +++ b/pkg/blobstore/replication/queued_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -51,15 +50,6 @@ func (br *queuedBlobReplicator) ReplicateSingle(ctx context.Context, blobDigest }) } -func (br *queuedBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return br.source.GetFromComposite(ctx, parentDigest, childDigest, slicer).WithTask(func() error { - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return util.StatusWrap(err, "Replication failed") - } - return nil - }) -} - func (br *queuedBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { // Don't queue requests for objects that have already been // replicated. diff --git a/pkg/blobstore/replication/queued_blob_replicator_test.go b/pkg/blobstore/replication/queued_blob_replicator_test.go index 0594edfe3..49dc4cb7d 100644 --- a/pkg/blobstore/replication/queued_blob_replicator_test.go +++ b/pkg/blobstore/replication/queued_blob_replicator_test.go @@ -127,68 +127,6 @@ func TestQueuedBlobReplicatorReplicateSingle(t *testing.T) { }) } -func TestQueuedBlobReplicatorReplicateComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - source := mock.NewMockBlobAccess(ctrl) - baseReplicator := mock.NewMockBlobReplicator(ctrl) - clock := mock.NewMockClock(ctrl) - replicator := replication.NewQueuedBlobReplicator( - source, - baseReplicator, - digest.NewExistenceCache(clock, digest.KeyWithoutInstance, 10, time.Minute, eviction.NewLRUSet[string]()), - ) - - parentDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 11) - parentDigests := parentDigest.ToSingletonSet() - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - // Only a single test for the success case is provided, as the - // tests for ReplicateSingle() provide enough coverage. - - t.Run("Success", func(t *testing.T) { - // The first time the object is requested, it should be - // replicated in the background. - source.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), - ) - clock.EXPECT().Now().Return(time.Unix(1000, 0)).Times(3) - baseReplicator.EXPECT().ReplicateMultiple(ctx, parentDigests).Return(nil) - - b := replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer) - data, err := b.ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - - // The fact that the object has been replicated should - // be cached. We should only perform the load during the - // second attempt. - source.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), - ) - clock.EXPECT().Now().Return(time.Unix(1060, 0)) - - b = replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer) - data, err = b.ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - - // A third request after the cache entry has expired - // should trigger a replication once again. - source.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), - ) - clock.EXPECT().Now().Return(time.Unix(1060, 1)).Times(3) - baseReplicator.EXPECT().ReplicateMultiple(ctx, parentDigests).Return(nil) - - b = replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer) - data, err = b.ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) -} - func TestQueuedBlobReplicatorReplicateMultiple(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/replication/remote_blob_replicator.go b/pkg/blobstore/replication/remote_blob_replicator.go index 7ce854e9a..c935cd716 100644 --- a/pkg/blobstore/replication/remote_blob_replicator.go +++ b/pkg/blobstore/replication/remote_blob_replicator.go @@ -6,7 +6,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/replicator" @@ -44,20 +43,6 @@ func (br *remoteBlobReplicator) ReplicateSingle(ctx context.Context, digest dige }) } -func (br *remoteBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return br.source.GetFromComposite(ctx, parentDigest, childDigest, slicer).WithTask(func() error { - digestFunction := parentDigest.GetDigestFunction() - _, err := br.replicatorClient.ReplicateBlobs(ctx, &replicator.ReplicateBlobsRequest{ - InstanceName: digestFunction.GetInstanceName().String(), - DigestFunction: digestFunction.GetEnumValue(), - BlobDigests: []*remoteexecution.Digest{ - parentDigest.GetProto(), - }, - }) - return err - }) -} - func (br *remoteBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { // Partition all digests by digest function, as the // ReplicateBlobs() RPC can only process digests for a single diff --git a/pkg/blobstore/replication/with_blob_replicator.go b/pkg/blobstore/replication/with_blob_replicator.go index c92e72268..23365c117 100644 --- a/pkg/blobstore/replication/with_blob_replicator.go +++ b/pkg/blobstore/replication/with_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -45,37 +44,3 @@ func (eh *getReplicatingErrorHandler) OnError(observedErr error) (buffer.Buffer, } func (getReplicatingErrorHandler) Done() {} - -// GetFromCompositeWithBlobReplicator is equivalent to -// GetWithBlobReplicator, except that it's a common implementation of -// BlobAccess.GetFromComposite(). -func GetFromCompositeWithBlobReplicator(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer, initialBackend blobstore.BlobAccess, successiveBackends BlobReplicatorSelector) buffer.Buffer { - return buffer.WithErrorHandler( - initialBackend.GetFromComposite(ctx, parentDigest, childDigest, slicer), - &getFromCompositeReplicatingErrorHandler{ - selector: successiveBackends, - context: ctx, - parentDigest: parentDigest, - childDigest: childDigest, - slicer: slicer, - }, - ) -} - -type getFromCompositeReplicatingErrorHandler struct { - selector BlobReplicatorSelector - context context.Context - parentDigest digest.Digest - childDigest digest.Digest - slicer slicing.BlobSlicer -} - -func (eh *getFromCompositeReplicatingErrorHandler) OnError(observedErr error) (buffer.Buffer, error) { - replicator, err := eh.selector(observedErr) - if err != nil { - return nil, err - } - return replicator.ReplicateComposite(eh.context, eh.parentDigest, eh.childDigest, eh.slicer), nil -} - -func (getFromCompositeReplicatingErrorHandler) Done() {} diff --git a/pkg/blobstore/sharding/BUILD.bazel b/pkg/blobstore/sharding/BUILD.bazel index c5794b2ad..4db190e21 100644 --- a/pkg/blobstore/sharding/BUILD.bazel +++ b/pkg/blobstore/sharding/BUILD.bazel @@ -12,7 +12,6 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/util", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/sharding/sharding_blob_access.go b/pkg/blobstore/sharding/sharding_blob_access.go index 7d22974ec..62e73bc0b 100644 --- a/pkg/blobstore/sharding/sharding_blob_access.go +++ b/pkg/blobstore/sharding/sharding_blob_access.go @@ -8,7 +8,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -53,14 +52,6 @@ func (ba *shardingBlobAccess) Get(ctx context.Context, digest digest.Digest) buf ) } -func (ba *shardingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - index := ba.getBackendIndexByDigest(parentDigest) - return buffer.WithErrorHandler( - ba.backends[index].Backend.GetFromComposite(ctx, parentDigest, childDigest, slicer), - shardKeyAddingErrorHandler{key: ba.backends[index].Key}, - ) -} - func (ba *shardingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { index := ba.getBackendIndexByDigest(digest) if err := ba.backends[index].Backend.Put(ctx, digest, b); err != nil { diff --git a/pkg/blobstore/sharding/sharding_blob_access_test.go b/pkg/blobstore/sharding/sharding_blob_access_test.go index 62d3d7a0c..a20195e0d 100644 --- a/pkg/blobstore/sharding/sharding_blob_access_test.go +++ b/pkg/blobstore/sharding/sharding_blob_access_test.go @@ -40,7 +40,6 @@ func TestShardingBlobAccess(t *testing.T) { ) helloDigest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - llDigest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "5b54c0a045f179bcbbbc9abcb8b5cd4c", 2) t.Run("GetFailure", func(t *testing.T) { // Errors should be prefixed with the shard key. @@ -62,20 +61,6 @@ func TestShardingBlobAccess(t *testing.T) { require.Equal(t, []byte("Hello"), data) }) - t.Run("GetFromCompositeSuccess", func(t *testing.T) { - // For reads from composite objects, the sharding needs - // to be based on the parent digest. That digest was - // used to upload the object to storage. - shardSelector.EXPECT().GetShard(uint64(0x8b1a9953c4611296)).Return(0) - slicer := mock.NewMockBlobSlicer(ctrl) - shard0.EXPECT().GetFromComposite(ctx, helloDigest, llDigest, slicer). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("ll"))) - - data, err := blobAccess.GetFromComposite(ctx, helloDigest, llDigest, slicer).ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("ll"), data) - }) - t.Run("PutFailure", func(t *testing.T) { // Errors should be prefixed with a shard key. shardSelector.EXPECT().GetShard(uint64(0x8b1a9953c4611296)).Return(1) diff --git a/pkg/cas/BUILD.bazel b/pkg/cas/BUILD.bazel new file mode 100644 index 000000000..9762efda6 --- /dev/null +++ b/pkg/cas/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "cas", + srcs = [ + "content_addressable_storage.go", + "operations.go", + "put_proto.go", + "read_blob_at.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/cas", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", + "//pkg/digest", + "//pkg/util", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/pkg/cas/content_addressable_storage.go b/pkg/cas/content_addressable_storage.go new file mode 100644 index 000000000..ce5c4d758 --- /dev/null +++ b/pkg/cas/content_addressable_storage.go @@ -0,0 +1,130 @@ +package cas + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" +) + +type contentAddressableStorage struct { + chunkStorage blobstore.BlobAccess + chunkListFetcher chunklist.Fetcher + cdcParametersFetcher cdc.ParametersFetcher + chunkListStorage blobstore.BlobAccess + digestKeyFormat digest.KeyFormat +} + +// ContentAddressableStorage is an interface which describes +// interactions with a Content Addressable Storage (CAS). +type ContentAddressableStorage interface { + // GetDigestKeyFormat returns the combined digest key format of the + // CS and CLS. + GetDigestKeyFormat() digest.KeyFormat + // FetchCDCParameters fetches the CDC parameters of the CAS. + FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (cdc.Parameters, error) + // FindMissing returns the set of digests that are not present in + // the CAS. + FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) + // FetchChunk fetches the contents of a specific chunk. + FetchChunk(ctx context.Context, d digest.Digest) ([]byte, error) + // PutChunk stores the contents of a specific chunk. + PutChunk(ctx context.Context, d digest.Digest, data []byte) error + // GetManifest returns the chunk list which constitutes a specific + // blob. + GetManifest(ctx context.Context, d digest.Digest) (chunklist.ChunkList, error) + // PutManifest stores the chunk list which constitutes a specific + // blob. + PutManifest(ctx context.Context, d digest.Digest, manifest chunklist.ChunkList) error +} + +// ContentAddressableStorage implements the chunklist.ChunkFetcher +// interface +var _ chunklist.ChunkFetcher = ContentAddressableStorage(nil) + +// ContentAddressableStorage implements the ParametersFetcher interface. +var _ cdc.ParametersFetcher = ContentAddressableStorage(nil) + +// NewContentAddressableStorage creates an interface towards the Content +// Addressable Storage (CAS) that allows access to arbitrary sized +// objects from the chunks described in the chunk list. +func NewContentAddressableStorage(chunkStorage, chunkListStorage blobstore.BlobAccess, chunkListFetcher chunklist.Fetcher, cdcParametersFetcher cdc.ParametersFetcher, digestKeyFormat digest.KeyFormat) ContentAddressableStorage { + return &contentAddressableStorage{ + chunkStorage: chunkStorage, + chunkListStorage: chunkListStorage, + chunkListFetcher: chunkListFetcher, + cdcParametersFetcher: cdcParametersFetcher, + digestKeyFormat: digestKeyFormat, + } +} + +func (cas *contentAddressableStorage) GetDigestKeyFormat() digest.KeyFormat { + return cas.digestKeyFormat +} + +func (cas *contentAddressableStorage) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (cdc.Parameters, error) { + return cas.cdcParametersFetcher.FetchCDCParameters(ctx, instanceName) +} + +func (cas *contentAddressableStorage) FetchChunk(ctx context.Context, d digest.Digest) ([]byte, error) { + return cas.chunkStorage.Get(ctx, d).ToByteSlice(int(d.GetSizeBytes())) +} + +func (cas *contentAddressableStorage) PutChunk(ctx context.Context, d digest.Digest, data []byte) error { + return cas.chunkStorage.Put(ctx, d, buffer.NewCASBufferFromByteSlice(d, data, buffer.UserProvided)) +} + +func (cas *contentAddressableStorage) GetManifest(ctx context.Context, d digest.Digest) (chunklist.ChunkList, error) { + return cas.chunkListFetcher.FetchChunkList(ctx, d) +} + +func (cas *contentAddressableStorage) PutManifest(ctx context.Context, d digest.Digest, manifest chunklist.ChunkList) error { + b := buffer.NewProtoBufferFromProto(chunklist.ToProto(manifest), buffer.UserProvided) + if err := cas.chunkListStorage.Put(ctx, d, b); err != nil { + return util.StatusWrap(err, "Could not save chunk list for blob") + } + return nil +} + +func (cas *contentAddressableStorage) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + digestSets := digests.PartitionByInstanceName() + missings := make([]digest.Set, 0, len(digestSets)) + for _, digestSet := range digestSets { + // PartitionByInstanceNames guarantees non empty sets. + missing, err := cas.findMissingFromInstance(ctx, digestSet.Items()[0].GetInstanceName(), digestSet) + if err != nil { + return digest.EmptySet, err + } + missings = append(missings, missing) + } + return digest.GetUnion(missings), nil +} + +func (cas *contentAddressableStorage) findMissingFromInstance(ctx context.Context, instanceName digest.InstanceName, digests digest.Set) (digest.Set, error) { + params, err := cas.cdcParametersFetcher.FetchCDCParameters(ctx, instanceName) + if err != nil { + return digest.EmptySet, err + } + smallDigests := digest.NewSetBuilder(digests.Length()) + largeDigests := digest.NewSetBuilder(digests.Length()) + for _, d := range digests.Items() { + if IsSingleChunk(params, d) { + smallDigests.Add(d) + } else { + largeDigests.Add(d) + } + } + smallMissing, err := cas.chunkStorage.FindMissing(ctx, smallDigests.Build()) + if err != nil { + return digest.EmptySet, err + } + largeMissing, err := cas.chunkListStorage.FindMissing(ctx, largeDigests.Build()) + if err != nil { + return digest.EmptySet, err + } + return digest.GetUnion([]digest.Set{smallMissing, largeMissing}), nil +} diff --git a/pkg/cas/operations.go b/pkg/cas/operations.go new file mode 100644 index 000000000..c2978ec89 --- /dev/null +++ b/pkg/cas/operations.go @@ -0,0 +1,212 @@ +package cas + +import ( + "bytes" + "context" + "io" + + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// IsSingleChunk checks if a digest is represented by a single chunk. +func IsSingleChunk(params cdc.Parameters, d digest.Digest) bool { + return d.GetSizeBytes() < 2*params.MinChunkSizeBytes +} + +// GetBytes returns the bytes of a digest from the CAS as a byteslice. +func GetBytes(ctx context.Context, cas ContentAddressableStorage, d digest.Digest) ([]byte, error) { + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + return cas.FetchChunk(ctx, d) + } + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch chunk list") + } + ret := make([]byte, d.GetSizeBytes()) + for _, chunk := range manifest { + chunkBytes, err := cas.FetchChunk(ctx, chunk.Digest) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch chunk") + } + copy(ret[chunk.Offset:], chunkBytes) + } + return ret, nil +} + +// GetReadCloser returns an io.ReadCloser that reads contents of a blob +// in CAS. +func GetReadCloser(ctx context.Context, cas ContentAddressableStorage, d digest.Digest) (io.ReadCloser, error) { + return GetReadCloserAt(ctx, cas, d, 0) +} + +// GetReadCloserAt returns an io.ReadCloser that reads contents of a +// blob in seeked to a specific offset. +func GetReadCloserAt(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, offset int64) (io.ReadCloser, error) { + if offset < 0 || offset > d.GetSizeBytes() { + return nil, status.Errorf(codes.InvalidArgument, "Offset %d is outside of blob %s", offset, d) + } + + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + chunk, err := cas.FetchChunk(ctx, d) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(chunk[offset:])), nil + } + + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch chunk list") + } + index, chunkOffset := chunklist.FindChunkOffset(manifest, uint64(offset)) + if index >= len(manifest) { + // Offset at the end of the blob. + return io.NopCloser(bytes.NewReader(nil)), nil + } + r := chunklist.NewChunkConcatenatingReader(ctx, manifest[index:], cas) + if chunkOffset > 0 { + if _, err := io.CopyN(io.Discard, r, chunkOffset); err != nil { + return nil, util.StatusWrap(err, "Failed to skip to read offset") + } + } + return r, nil +} + +// PutBytes inserts all chunks for a digest from it's raw bytes. +func PutBytes(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, data []byte) error { + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + return cas.PutChunk(ctx, d, data) + } + return PutReader(ctx, cas, d, bytes.NewReader(data)) +} + +// PutReader inserts all chunks for a digest from an io.Reader. +func PutReader(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, r io.Reader) error { + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return util.StatusWrap(err, "Could not fetch CDC parameters") + } + + digestFunction := d.GetDigestFunction() + chunker := cdc.NewReaderChunker(digestFunction, r, params.MinChunkSizeBytes, params.HorizonSizeBytes) + wholeGen := digestFunction.NewGenerator(d.GetSizeBytes()) + + chunkList := make(chunklist.ChunkList, 0, d.GetSizeBytes()/params.MinChunkSizeBytes+1) + var offset uint64 + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + if err != nil { + return util.StatusWrap(err, "Failed to chunk write stream") + } + + if _, err := wholeGen.Write(chunk.Data); err != nil { + return status.Error(codes.Internal, "Could not compute digest of blob") + } + + if err := cas.PutChunk(ctx, chunk.Digest, chunk.Data); err != nil { + return util.StatusWrap(err, "Failed to save chunk") + } + + chunkList = append(chunkList, chunklist.Entry{Offset: offset, Digest: chunk.Digest}) + offset += uint64(chunk.Digest.GetSizeBytes()) + } + + // Verify the whole blob against the advertised digest. + if actual := wholeGen.Sum(); actual != d { + return status.Errorf(codes.InvalidArgument, "Blob digest mismatch: advertised %s, actual %s", d, actual) + } + + // A single chunk is the trivial case: it already lives in the + // chunk storage and needs no chunk list. + if len(chunkList) <= 1 { + return nil + } + + // We generated and validated this chunk list ourselves. + ctx = cdc.NewContextWithChunkListValidationBypass(ctx) + if err := cas.PutManifest(ctx, d, chunkList); err != nil { + return util.StatusWrap(err, "Could not save chunk list for blob") + } + return nil +} + +// GetProto retrieves a proto message from the CAS unmarshalling it into +// the supplied message object +func GetProto[T proto.Message](ctx context.Context, cas ContentAddressableStorage, d digest.Digest, message T) (T, error) { + var zero T + bytes, err := GetBytes(ctx, cas, d) + if err != nil { + return zero, err + } + err = proto.Unmarshal(bytes, message) + if err != nil { + return zero, util.StatusWrapWithCode(err, codes.InvalidArgument, "Failed to unmarshal message") + } + return message, nil +} + +// IntoWriter streams the chunks of a blob directly into the provided +// io.Writer from the specified offset. +func IntoWriter(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, offset int64, w io.Writer) error { + if offset < 0 || offset > d.GetSizeBytes() { + return status.Errorf(codes.InvalidArgument, "Invalid offset %d for digest %s", offset, d) + } + + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return util.StatusWrapf(err, "Failed to fetch CDC parameters for instance %s", d.GetInstanceName()) + } + + if IsSingleChunk(params, d) { + chunkData, err := cas.FetchChunk(ctx, d) + if err != nil { + return err + } + if _, err := w.Write(chunkData[offset:]); err != nil { + return err + } + return nil + } + + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return err + } + index, chunkOffset := chunklist.FindChunkOffset(manifest, uint64(offset)) + for ; index < len(manifest); index++ { + chunk := manifest[index] + chunkData, err := cas.FetchChunk(ctx, chunk.Digest) + if err != nil { + return err + } + + if _, err := w.Write(chunkData[chunkOffset:]); err != nil { + return err + } + chunkOffset = 0 + } + + return nil +} diff --git a/pkg/cas/put_proto.go b/pkg/cas/put_proto.go new file mode 100644 index 000000000..659d52788 --- /dev/null +++ b/pkg/cas/put_proto.go @@ -0,0 +1,29 @@ +package cas + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/protobuf/proto" +) + +// PutProto is a helper function for storing Protobuf messages in the +// Content Addressable Storage (CAS). It computes the digest of the +// message and stores it under that key. The digest is then returned, so +// that the object may be referenced. +func PutProto(ctx context.Context, contentAddressableStorage ContentAddressableStorage, message proto.Message, digestFunction digest.Function) (digest.Digest, error) { + bytes, err := proto.Marshal(message) + if err != nil { + return digest.BadDigest, err + } + digestGenerator := digestFunction.NewGenerator(int64(len(bytes))) + if _, err := digestGenerator.Write(bytes); err != nil { + return digest.BadDigest, err + } + blobDigest := digestGenerator.Sum() + if err := PutBytes(ctx, contentAddressableStorage, blobDigest, bytes); err != nil { + return digest.BadDigest, err + } + return blobDigest, nil +} diff --git a/pkg/cas/read_blob_at.go b/pkg/cas/read_blob_at.go new file mode 100644 index 000000000..6f9ab1de5 --- /dev/null +++ b/pkg/cas/read_blob_at.go @@ -0,0 +1,68 @@ +package cas + +import ( + "context" + "io" + + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ReadBlobAt fills buf with the contents of a blob starting at the +// given offset, following the semantics of io.ReaderAt: it returns +// the number of bytes read and io.EOF when the read reaches the end +// of the blob before filling buf completely. +// +// Blobs stored as a single chunk are read with a single Chunk Storage +// fetch. For larger blobs only the chunks covering the requested +// range are fetched, so random access reads do not stream the full +// blob. +func ReadBlobAt(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, buf []byte, offset int64) (int, error) { + if offset < 0 || offset > d.GetSizeBytes() { + return 0, status.Errorf(codes.InvalidArgument, "Offset %d is outside of blob %s", offset, d) + } + if len(buf) == 0 { + return 0, nil + } + + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return 0, util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + chunk, err := cas.FetchChunk(ctx, d) + if err != nil { + return 0, err + } + n := copy(buf, chunk[offset:]) + if n < len(buf) { + return n, io.EOF + } + return n, nil + } + + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return 0, util.StatusWrap(err, "Could not fetch chunk list") + } + index, chunkOffset := chunklist.FindChunkOffset(manifest, uint64(offset)) + + n := 0 + for n < len(buf) && index < len(manifest) { + chunkBytes, err := cas.FetchChunk(ctx, manifest[index].Digest) + if err != nil { + return n, util.StatusWrap(err, "Could not fetch chunk") + } + n += copy(buf[n:], chunkBytes[chunkOffset:]) + chunkOffset = 0 + index++ + } + if n < len(buf) { + return n, io.EOF + } + return n, nil +} diff --git a/pkg/digest/function.go b/pkg/digest/function.go index 5f3149864..29c25bf07 100644 --- a/pkg/digest/function.go +++ b/pkg/digest/function.go @@ -113,6 +113,9 @@ func (f Function) NewDigestFromProto(digest *remoteexecution.Digest) (Digest, er if digest == nil { return BadDigest, status.Error(codes.InvalidArgument, "No digest provided") } + if digest.SizeBytes < 0 { + return BadDigest, status.Error(codes.InvalidArgument, "Negative size digest provided") + } return f.NewDigest(digest.Hash, digest.SizeBytes) } diff --git a/pkg/proto/blobstore/chunklist/BUILD.bazel b/pkg/proto/blobstore/chunklist/BUILD.bazel new file mode 100644 index 000000000..9b741fb25 --- /dev/null +++ b/pkg/proto/blobstore/chunklist/BUILD.bazel @@ -0,0 +1,24 @@ +load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//proto:def.bzl", "go_proto_library") + +proto_library( + name = "chunklist_proto", + srcs = ["chunk_list.proto"], + import_prefix = "github.com/buildbarn/bb-storage", + visibility = ["//visibility:public"], +) + +go_proto_library( + name = "chunklist_go_proto", + importpath = "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist", + proto = ":chunklist_proto", + visibility = ["//visibility:public"], +) + +go_library( + name = "chunklist", + embed = [":chunklist_go_proto"], + importpath = "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist", + visibility = ["//visibility:public"], +) diff --git a/pkg/proto/blobstore/chunklist/chunk_list.pb.go b/pkg/proto/blobstore/chunklist/chunk_list.pb.go new file mode 100644 index 000000000..5835010a9 --- /dev/null +++ b/pkg/proto/blobstore/chunklist/chunk_list.pb.go @@ -0,0 +1,124 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11-devel +// protoc v7.35.0 +// source: github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist/chunk_list.proto + +package chunklist + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ChunkList struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChunkList) Reset() { + *x = ChunkList{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkList) ProtoMessage() {} + +func (x *ChunkList) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkList.ProtoReflect.Descriptor instead. +func (*ChunkList) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescGZIP(), []int{0} +} + +func (x *ChunkList) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto protoreflect.FileDescriptor + +const file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc = "" + + "\n" + + "Ngithub.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist/chunk_list.proto\x12\x1dbuildbarn.blobstore.chunklist\"\x1f\n" + + "\tChunkList\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04dataB?Z=github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklistb\x06proto3" + +var ( + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescOnce sync.Once + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescData []byte +) + +func file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescGZIP() []byte { + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescOnce.Do(func() { + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc))) + }) + return file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescData +} + +var file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_goTypes = []any{ + (*ChunkList)(nil), // 0: buildbarn.blobstore.chunklist.ChunkList +} +var file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_init() +} +func file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_init() { + if File_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_goTypes, + DependencyIndexes: file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_depIdxs, + MessageInfos: file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes, + }.Build() + File_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto = out.File + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_goTypes = nil + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_depIdxs = nil +} diff --git a/pkg/proto/blobstore/chunklist/chunk_list.proto b/pkg/proto/blobstore/chunklist/chunk_list.proto new file mode 100644 index 000000000..7b8c6fdb4 --- /dev/null +++ b/pkg/proto/blobstore/chunklist/chunk_list.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package buildbarn.blobstore.chunklist; + +option go_package = "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist"; + +// ChunkList is the internal storage format for chunk lists in the +// Chunk List Storage (CLS). The data field contains a sequence of +// compact binary digest encodings as produced by +// digest.Digest.GetCompactBinary(). +message ChunkList { + bytes data = 1; +} diff --git a/pkg/proto/configuration/bb_replicator/bb_replicator.proto b/pkg/proto/configuration/bb_replicator/bb_replicator.proto index 363383881..cda2231e9 100644 --- a/pkg/proto/configuration/bb_replicator/bb_replicator.proto +++ b/pkg/proto/configuration/bb_replicator/bb_replicator.proto @@ -16,10 +16,10 @@ message ApplicationConfiguration { // gRPC servers to spawn to listen for requests from clients. repeated buildbarn.configuration.grpc.ServerConfiguration grpc_servers = 2; - // Content Addressable Storage where data needs to be read. + // Chunk Storage where data needs to be read. buildbarn.configuration.blobstore.BlobAccessConfiguration source = 3; - // Content Addressable Storage where data needs to be written. + // Chunk Storage where data needs to be written. buildbarn.configuration.blobstore.BlobAccessConfiguration sink = 4; // Configuration for replication. diff --git a/pkg/proto/configuration/bb_storage/bb_storage.pb.go b/pkg/proto/configuration/bb_storage/bb_storage.pb.go index fa4fb021e..c45e752eb 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.pb.go +++ b/pkg/proto/configuration/bb_storage/bb_storage.pb.go @@ -29,19 +29,19 @@ const ( ) type ApplicationConfiguration struct { - state protoimpl.MessageState `protogen:"open.v1"` - GrpcServers []*grpc.ServerConfiguration `protobuf:"bytes,4,rep,name=grpc_servers,json=grpcServers,proto3" json:"grpc_servers,omitempty"` - Schedulers map[string]*builder.SchedulerConfiguration `protobuf:"bytes,5,rep,name=schedulers,proto3" json:"schedulers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - MaximumMessageSizeBytes int64 `protobuf:"varint,8,opt,name=maximum_message_size_bytes,json=maximumMessageSizeBytes,proto3" json:"maximum_message_size_bytes,omitempty"` - Global *global.Configuration `protobuf:"bytes,9,opt,name=global,proto3" json:"global,omitempty"` - ContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,17,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` - ActionCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,18,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` - IndirectContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,10,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` - InitialSizeClassCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,11,opt,name=initial_size_class_cache,json=initialSizeClassCache,proto3" json:"initial_size_class_cache,omitempty"` - FileSystemAccessCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,19,opt,name=file_system_access_cache,json=fileSystemAccessCache,proto3" json:"file_system_access_cache,omitempty"` - ExecuteAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,16,opt,name=execute_authorizer,json=executeAuthorizer,proto3" json:"execute_authorizer,omitempty"` - SupportedCompressors []v2.Compressor_Value `protobuf:"varint,20,rep,packed,name=supported_compressors,json=supportedCompressors,proto3,enum=build.bazel.remote.execution.v2.Compressor_Value" json:"supported_compressors,omitempty"` - ZstdPool *zstd.PoolConfiguration `protobuf:"bytes,21,opt,name=zstd_pool,json=zstdPool,proto3" json:"zstd_pool,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + GrpcServers []*grpc.ServerConfiguration `protobuf:"bytes,4,rep,name=grpc_servers,json=grpcServers,proto3" json:"grpc_servers,omitempty"` + Schedulers map[string]*builder.SchedulerConfiguration `protobuf:"bytes,5,rep,name=schedulers,proto3" json:"schedulers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + MaximumMessageSizeBytes int64 `protobuf:"varint,8,opt,name=maximum_message_size_bytes,json=maximumMessageSizeBytes,proto3" json:"maximum_message_size_bytes,omitempty"` + Global *global.Configuration `protobuf:"bytes,9,opt,name=global,proto3" json:"global,omitempty"` + ContentAddressableStorageServer *ContentAddressableStorageServerConfiguration `protobuf:"bytes,17,opt,name=content_addressable_storage_server,json=contentAddressableStorageServer,proto3" json:"content_addressable_storage_server,omitempty"` + ActionCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,18,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` + IndirectContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,10,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` + InitialSizeClassCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,11,opt,name=initial_size_class_cache,json=initialSizeClassCache,proto3" json:"initial_size_class_cache,omitempty"` + FileSystemAccessCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,19,opt,name=file_system_access_cache,json=fileSystemAccessCache,proto3" json:"file_system_access_cache,omitempty"` + ExecuteAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,16,opt,name=execute_authorizer,json=executeAuthorizer,proto3" json:"execute_authorizer,omitempty"` + SupportedCompressors []v2.Compressor_Value `protobuf:"varint,20,rep,packed,name=supported_compressors,json=supportedCompressors,proto3,enum=build.bazel.remote.execution.v2.Compressor_Value" json:"supported_compressors,omitempty"` + ZstdPool *zstd.PoolConfiguration `protobuf:"bytes,21,opt,name=zstd_pool,json=zstdPool,proto3" json:"zstd_pool,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -104,9 +104,9 @@ func (x *ApplicationConfiguration) GetGlobal() *global.Configuration { return nil } -func (x *ApplicationConfiguration) GetContentAddressableStorage() *ScannableBlobAccessConfiguration { +func (x *ApplicationConfiguration) GetContentAddressableStorageServer() *ContentAddressableStorageServerConfiguration { if x != nil { - return x.ContentAddressableStorage + return x.ContentAddressableStorageServer } return nil } @@ -160,6 +160,74 @@ func (x *ApplicationConfiguration) GetZstdPool() *zstd.PoolConfiguration { return nil } +type ContentAddressableStorageServerConfiguration struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContentAddressableStorage *blobstore.ContentAddressableStorageConfiguration `protobuf:"bytes,1,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + GetAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,2,opt,name=get_authorizer,json=getAuthorizer,proto3" json:"get_authorizer,omitempty"` + PutAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,3,opt,name=put_authorizer,json=putAuthorizer,proto3" json:"put_authorizer,omitempty"` + FindMissingAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,4,opt,name=find_missing_authorizer,json=findMissingAuthorizer,proto3" json:"find_missing_authorizer,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContentAddressableStorageServerConfiguration) Reset() { + *x = ContentAddressableStorageServerConfiguration{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContentAddressableStorageServerConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContentAddressableStorageServerConfiguration) ProtoMessage() {} + +func (x *ContentAddressableStorageServerConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContentAddressableStorageServerConfiguration.ProtoReflect.Descriptor instead. +func (*ContentAddressableStorageServerConfiguration) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{1} +} + +func (x *ContentAddressableStorageServerConfiguration) GetContentAddressableStorage() *blobstore.ContentAddressableStorageConfiguration { + if x != nil { + return x.ContentAddressableStorage + } + return nil +} + +func (x *ContentAddressableStorageServerConfiguration) GetGetAuthorizer() *auth.AuthorizerConfiguration { + if x != nil { + return x.GetAuthorizer + } + return nil +} + +func (x *ContentAddressableStorageServerConfiguration) GetPutAuthorizer() *auth.AuthorizerConfiguration { + if x != nil { + return x.PutAuthorizer + } + return nil +} + +func (x *ContentAddressableStorageServerConfiguration) GetFindMissingAuthorizer() *auth.AuthorizerConfiguration { + if x != nil { + return x.FindMissingAuthorizer + } + return nil +} + type NonScannableBlobAccessConfiguration struct { state protoimpl.MessageState `protogen:"open.v1"` Backend *blobstore.BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` @@ -171,7 +239,7 @@ type NonScannableBlobAccessConfiguration struct { func (x *NonScannableBlobAccessConfiguration) Reset() { *x = NonScannableBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -183,7 +251,7 @@ func (x *NonScannableBlobAccessConfiguration) String() string { func (*NonScannableBlobAccessConfiguration) ProtoMessage() {} func (x *NonScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -196,7 +264,7 @@ func (x *NonScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Messag // Deprecated: Use NonScannableBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*NonScannableBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{1} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{2} } func (x *NonScannableBlobAccessConfiguration) GetBackend() *blobstore.BlobAccessConfiguration { @@ -232,7 +300,7 @@ type ScannableBlobAccessConfiguration struct { func (x *ScannableBlobAccessConfiguration) Reset() { *x = ScannableBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -244,7 +312,7 @@ func (x *ScannableBlobAccessConfiguration) String() string { func (*ScannableBlobAccessConfiguration) ProtoMessage() {} func (x *ScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -257,7 +325,7 @@ func (x *ScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ScannableBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ScannableBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{2} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{3} } func (x *ScannableBlobAccessConfiguration) GetBackend() *blobstore.BlobAccessConfiguration { @@ -292,15 +360,15 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_s const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc = "" + "\n" + - "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xbd\v\n" + + "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xd6\v\n" + "\x18ApplicationConfiguration\x12T\n" + "\fgrpc_servers\x18\x04 \x03(\v21.buildbarn.configuration.grpc.ServerConfigurationR\vgrpcServers\x12l\n" + "\n" + "schedulers\x18\x05 \x03(\v2L.buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntryR\n" + "schedulers\x12;\n" + "\x1amaximum_message_size_bytes\x18\b \x01(\x03R\x17maximumMessageSizeBytes\x12E\n" + - "\x06global\x18\t \x01(\v2-.buildbarn.configuration.global.ConfigurationR\x06global\x12\x84\x01\n" + - "\x1bcontent_addressable_storage\x18\x11 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x19contentAddressableStorage\x12j\n" + + "\x06global\x18\t \x01(\v2-.buildbarn.configuration.global.ConfigurationR\x06global\x12\x9d\x01\n" + + "\"content_addressable_storage_server\x18\x11 \x01(\v2P.buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfigurationR\x1fcontentAddressableStorageServer\x12j\n" + "\faction_cache\x18\x12 \x01(\v2G.buildbarn.configuration.bb_storage.NonScannableBlobAccessConfigurationR\vactionCache\x12\x95\x01\n" + "$indirect_content_addressable_storage\x18\n" + " \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR!indirectContentAddressableStorage\x12\x80\x01\n" + @@ -311,7 +379,12 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb "\tzstd_pool\x18\x15 \x01(\v2/.buildbarn.configuration.zstd.PoolConfigurationR\bzstdPool\x1av\n" + "\x0fSchedulersEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12M\n" + - "\x05value\x18\x02 \x01(\v27.buildbarn.configuration.builder.SchedulerConfigurationR\x05value:\x028\x01J\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\f\x10\rJ\x04\b\r\x10\x0eJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10\"\xb7\x02\n" + + "\x05value\x18\x02 \x01(\v27.buildbarn.configuration.builder.SchedulerConfigurationR\x05value:\x028\x01J\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\f\x10\rJ\x04\b\r\x10\x0eJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10\"\xe5\x03\n" + + ",ContentAddressableStorageServerConfiguration\x12\x89\x01\n" + + "\x1bcontent_addressable_storage\x18\x01 \x01(\v2I.buildbarn.configuration.blobstore.ContentAddressableStorageConfigurationR\x19contentAddressableStorage\x12\\\n" + + "\x0eget_authorizer\x18\x02 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\rgetAuthorizer\x12\\\n" + + "\x0eput_authorizer\x18\x03 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\rputAuthorizer\x12m\n" + + "\x17find_missing_authorizer\x18\x04 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\x15findMissingAuthorizer\"\xb7\x02\n" + "#NonScannableBlobAccessConfiguration\x12T\n" + "\abackend\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackend\x12\\\n" + "\x0eget_authorizer\x18\x02 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\rgetAuthorizer\x12\\\n" + @@ -334,45 +407,51 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_ return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescData } -var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_goTypes = []any{ - (*ApplicationConfiguration)(nil), // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration - (*NonScannableBlobAccessConfiguration)(nil), // 1: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - (*ScannableBlobAccessConfiguration)(nil), // 2: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - nil, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry - (*grpc.ServerConfiguration)(nil), // 4: buildbarn.configuration.grpc.ServerConfiguration - (*global.Configuration)(nil), // 5: buildbarn.configuration.global.Configuration - (*auth.AuthorizerConfiguration)(nil), // 6: buildbarn.configuration.auth.AuthorizerConfiguration - (v2.Compressor_Value)(0), // 7: build.bazel.remote.execution.v2.Compressor.Value - (*zstd.PoolConfiguration)(nil), // 8: buildbarn.configuration.zstd.PoolConfiguration - (*blobstore.BlobAccessConfiguration)(nil), // 9: buildbarn.configuration.blobstore.BlobAccessConfiguration - (*builder.SchedulerConfiguration)(nil), // 10: buildbarn.configuration.builder.SchedulerConfiguration + (*ApplicationConfiguration)(nil), // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration + (*ContentAddressableStorageServerConfiguration)(nil), // 1: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration + (*NonScannableBlobAccessConfiguration)(nil), // 2: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + (*ScannableBlobAccessConfiguration)(nil), // 3: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + nil, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry + (*grpc.ServerConfiguration)(nil), // 5: buildbarn.configuration.grpc.ServerConfiguration + (*global.Configuration)(nil), // 6: buildbarn.configuration.global.Configuration + (*auth.AuthorizerConfiguration)(nil), // 7: buildbarn.configuration.auth.AuthorizerConfiguration + (v2.Compressor_Value)(0), // 8: build.bazel.remote.execution.v2.Compressor.Value + (*zstd.PoolConfiguration)(nil), // 9: buildbarn.configuration.zstd.PoolConfiguration + (*blobstore.ContentAddressableStorageConfiguration)(nil), // 10: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + (*blobstore.BlobAccessConfiguration)(nil), // 11: buildbarn.configuration.blobstore.BlobAccessConfiguration + (*builder.SchedulerConfiguration)(nil), // 12: buildbarn.configuration.builder.SchedulerConfiguration } var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_depIdxs = []int32{ - 4, // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration.grpc_servers:type_name -> buildbarn.configuration.grpc.ServerConfiguration - 3, // 1: buildbarn.configuration.bb_storage.ApplicationConfiguration.schedulers:type_name -> buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry - 5, // 2: buildbarn.configuration.bb_storage.ApplicationConfiguration.global:type_name -> buildbarn.configuration.global.Configuration - 2, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 2, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 1, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 6, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 7, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value - 8, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration - 9, // 11: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 12: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 13: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 9, // 14: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 15: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 16: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 17: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 10, // 18: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration - 19, // [19:19] is the sub-list for method output_type - 19, // [19:19] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 5, // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration.grpc_servers:type_name -> buildbarn.configuration.grpc.ServerConfiguration + 4, // 1: buildbarn.configuration.bb_storage.ApplicationConfiguration.schedulers:type_name -> buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry + 6, // 2: buildbarn.configuration.bb_storage.ApplicationConfiguration.global:type_name -> buildbarn.configuration.global.Configuration + 1, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.content_addressable_storage_server:type_name -> buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration + 2, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 3, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + 2, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 2, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 7, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 8, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value + 9, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration + 10, // 11: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + 7, // 12: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 13: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 14: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 11, // 15: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 7, // 16: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 17: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 11, // 18: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 7, // 19: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 20: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 21: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 12, // 22: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration + 23, // [23:23] is the sub-list for method output_type + 23, // [23:23] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { @@ -388,7 +467,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc)), NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/configuration/bb_storage/bb_storage.proto b/pkg/proto/configuration/bb_storage/bb_storage.proto index 365c75527..ee27ee097 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.proto +++ b/pkg/proto/configuration/bb_storage/bb_storage.proto @@ -57,9 +57,10 @@ message ApplicationConfiguration { // Common configuration options that apply to all Buildbarn binaries. buildbarn.configuration.global.Configuration global = 9; - // Optional: Blobstore configuration for the Content Addressable - // Storage (CAS). - ScannableBlobAccessConfiguration content_addressable_storage = 17; + // Optional: Configurations for the Content Addressable Storage server + // (CAS). + ContentAddressableStorageServerConfiguration + content_addressable_storage_server = 17; // Optional: Blobstore configuration for the Action Cache (AC). NonScannableBlobAccessConfiguration action_cache = 18; @@ -111,6 +112,29 @@ message ApplicationConfiguration { buildbarn.configuration.zstd.PoolConfiguration zstd_pool = 21; } +// Storage configuration for a content addressable storage. +message ContentAddressableStorageServerConfiguration { + // Configurations for the constituent Blob Accesses that make up the + // ContentAddressableStorageServer. + buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + content_addressable_storage = 1; + + // The authorizer for determining whether a client may read from + // storage. Access to this authorizer pertains to the + // ByteStream.Read(), BatchReadBlobs() and SplitBlob() operations. + buildbarn.configuration.auth.AuthorizerConfiguration get_authorizer = 2; + + // The authorizer for determining whether a client may write to + // storage. Access to this configuration pertains to the + // ByteStream.Write(), BatchUpdateBlobs() and SpliceBlob() operations. + buildbarn.configuration.auth.AuthorizerConfiguration put_authorizer = 3; + + // The authorizer for determining whether a client may scan storage + // for the existence of a batch of digests. + buildbarn.configuration.auth.AuthorizerConfiguration find_missing_authorizer = + 4; +} + // Storage configuration for backends which don't allow batch digest // scanning. message NonScannableBlobAccessConfiguration { diff --git a/pkg/proto/configuration/blobstore/BUILD.bazel b/pkg/proto/configuration/blobstore/BUILD.bazel index a64c36145..83d0b07cc 100644 --- a/pkg/proto/configuration/blobstore/BUILD.bazel +++ b/pkg/proto/configuration/blobstore/BUILD.bazel @@ -14,6 +14,7 @@ proto_library( "//pkg/proto/configuration/digest:digest_proto", "//pkg/proto/configuration/grpc:grpc_proto", "//pkg/proto/configuration/http/client:client_proto", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto", "@googleapis//google/rpc:status_proto", "@protobuf//:duration_proto", "@protobuf//:empty_proto", @@ -33,6 +34,7 @@ go_proto_library( "//pkg/proto/configuration/digest", "//pkg/proto/configuration/grpc", "//pkg/proto/configuration/http/client", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", "@org_golang_google_genproto_googleapis_rpc//status", ], ) diff --git a/pkg/proto/configuration/blobstore/blobstore.pb.go b/pkg/proto/configuration/blobstore/blobstore.pb.go index dc3f8a4e0..b3e408af5 100644 --- a/pkg/proto/configuration/blobstore/blobstore.pb.go +++ b/pkg/proto/configuration/blobstore/blobstore.pb.go @@ -7,6 +7,7 @@ package blobstore import ( + v2 "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" blockdevice "github.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice" aws "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws" gcp "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp" @@ -31,17 +32,85 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ContentAddressableStorageConfiguration struct { + state protoimpl.MessageState `protogen:"open.v1"` + ChunkStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=chunk_storage,json=chunkStorage,proto3" json:"chunk_storage,omitempty"` + ChunkListStorage *BlobAccessConfiguration `protobuf:"bytes,2,opt,name=chunk_list_storage,json=chunkListStorage,proto3" json:"chunk_list_storage,omitempty"` + ChunkListCache *digest.ExistenceCacheConfiguration `protobuf:"bytes,3,opt,name=chunk_list_cache,json=chunkListCache,proto3" json:"chunk_list_cache,omitempty"` + CdcParameterCache *digest.ExistenceCacheConfiguration `protobuf:"bytes,4,opt,name=cdc_parameter_cache,json=cdcParameterCache,proto3" json:"cdc_parameter_cache,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContentAddressableStorageConfiguration) Reset() { + *x = ContentAddressableStorageConfiguration{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContentAddressableStorageConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContentAddressableStorageConfiguration) ProtoMessage() {} + +func (x *ContentAddressableStorageConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContentAddressableStorageConfiguration.ProtoReflect.Descriptor instead. +func (*ContentAddressableStorageConfiguration) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{0} +} + +func (x *ContentAddressableStorageConfiguration) GetChunkStorage() *BlobAccessConfiguration { + if x != nil { + return x.ChunkStorage + } + return nil +} + +func (x *ContentAddressableStorageConfiguration) GetChunkListStorage() *BlobAccessConfiguration { + if x != nil { + return x.ChunkListStorage + } + return nil +} + +func (x *ContentAddressableStorageConfiguration) GetChunkListCache() *digest.ExistenceCacheConfiguration { + if x != nil { + return x.ChunkListCache + } + return nil +} + +func (x *ContentAddressableStorageConfiguration) GetCdcParameterCache() *digest.ExistenceCacheConfiguration { + if x != nil { + return x.CdcParameterCache + } + return nil +} + type BlobstoreConfiguration struct { - state protoimpl.MessageState `protogen:"open.v1"` - ContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` - ActionCache *BlobAccessConfiguration `protobuf:"bytes,2,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + ContentAddressableStorage *ContentAddressableStorageConfiguration `protobuf:"bytes,1,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + ActionCache *BlobAccessConfiguration `protobuf:"bytes,2,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *BlobstoreConfiguration) Reset() { *x = BlobstoreConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -53,7 +122,7 @@ func (x *BlobstoreConfiguration) String() string { func (*BlobstoreConfiguration) ProtoMessage() {} func (x *BlobstoreConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -66,10 +135,10 @@ func (x *BlobstoreConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobstoreConfiguration.ProtoReflect.Descriptor instead. func (*BlobstoreConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{0} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{1} } -func (x *BlobstoreConfiguration) GetContentAddressableStorage() *BlobAccessConfiguration { +func (x *BlobstoreConfiguration) GetContentAddressableStorage() *ContentAddressableStorageConfiguration { if x != nil { return x.ContentAddressableStorage } @@ -106,6 +175,7 @@ type BlobAccessConfiguration struct { // *BlobAccessConfiguration_WithLabels // *BlobAccessConfiguration_Label // *BlobAccessConfiguration_DeadlineEnforcing + // *BlobAccessConfiguration_ChunkListValidating Backend isBlobAccessConfiguration_Backend `protobuf_oneof:"backend"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -113,7 +183,7 @@ type BlobAccessConfiguration struct { func (x *BlobAccessConfiguration) Reset() { *x = BlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -125,7 +195,7 @@ func (x *BlobAccessConfiguration) String() string { func (*BlobAccessConfiguration) ProtoMessage() {} func (x *BlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -138,7 +208,7 @@ func (x *BlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*BlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{1} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{2} } func (x *BlobAccessConfiguration) GetBackend() isBlobAccessConfiguration_Backend { @@ -319,6 +389,15 @@ func (x *BlobAccessConfiguration) GetDeadlineEnforcing() *DeadlineEnforcingBlobA return nil } +func (x *BlobAccessConfiguration) GetChunkListValidating() *ChunkListValidatingBlobAccessConfiguration { + if x != nil { + if x, ok := x.Backend.(*BlobAccessConfiguration_ChunkListValidating); ok { + return x.ChunkListValidating + } + } + return nil +} + type isBlobAccessConfiguration_Backend interface { isBlobAccessConfiguration_Backend() } @@ -399,6 +478,10 @@ type BlobAccessConfiguration_DeadlineEnforcing struct { DeadlineEnforcing *DeadlineEnforcingBlobAccess `protobuf:"bytes,28,opt,name=deadline_enforcing,json=deadlineEnforcing,proto3,oneof"` } +type BlobAccessConfiguration_ChunkListValidating struct { + ChunkListValidating *ChunkListValidatingBlobAccessConfiguration `protobuf:"bytes,29,opt,name=chunk_list_validating,json=chunkListValidating,proto3,oneof"` +} + func (*BlobAccessConfiguration_ReadCaching) isBlobAccessConfiguration_Backend() {} func (*BlobAccessConfiguration_Grpc) isBlobAccessConfiguration_Backend() {} @@ -437,6 +520,8 @@ func (*BlobAccessConfiguration_Label) isBlobAccessConfiguration_Backend() {} func (*BlobAccessConfiguration_DeadlineEnforcing) isBlobAccessConfiguration_Backend() {} +func (*BlobAccessConfiguration_ChunkListValidating) isBlobAccessConfiguration_Backend() {} + type ReadCachingBlobAccessConfiguration struct { state protoimpl.MessageState `protogen:"open.v1"` Slow *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=slow,proto3" json:"slow,omitempty"` @@ -448,7 +533,7 @@ type ReadCachingBlobAccessConfiguration struct { func (x *ReadCachingBlobAccessConfiguration) Reset() { *x = ReadCachingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -460,7 +545,7 @@ func (x *ReadCachingBlobAccessConfiguration) String() string { func (*ReadCachingBlobAccessConfiguration) ProtoMessage() {} func (x *ReadCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -473,7 +558,7 @@ func (x *ReadCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Message // Deprecated: Use ReadCachingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReadCachingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{2} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{3} } func (x *ReadCachingBlobAccessConfiguration) GetSlow() *BlobAccessConfiguration { @@ -506,7 +591,7 @@ type ShardingBlobAccessConfiguration struct { func (x *ShardingBlobAccessConfiguration) Reset() { *x = ShardingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -518,7 +603,7 @@ func (x *ShardingBlobAccessConfiguration) String() string { func (*ShardingBlobAccessConfiguration) ProtoMessage() {} func (x *ShardingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -531,7 +616,7 @@ func (x *ShardingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ShardingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{3} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{4} } func (x *ShardingBlobAccessConfiguration) GetShards() map[string]*ShardingBlobAccessConfiguration_Shard { @@ -553,7 +638,7 @@ type MirroredBlobAccessConfiguration struct { func (x *MirroredBlobAccessConfiguration) Reset() { *x = MirroredBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -565,7 +650,7 @@ func (x *MirroredBlobAccessConfiguration) String() string { func (*MirroredBlobAccessConfiguration) ProtoMessage() {} func (x *MirroredBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -578,7 +663,7 @@ func (x *MirroredBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use MirroredBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*MirroredBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{4} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5} } func (x *MirroredBlobAccessConfiguration) GetBackendA() *BlobAccessConfiguration { @@ -628,13 +713,14 @@ type LocalBlobAccessConfiguration struct { BlocksBackend isLocalBlobAccessConfiguration_BlocksBackend `protobuf_oneof:"blocks_backend"` Persistent *LocalBlobAccessConfiguration_Persistent `protobuf:"bytes,13,opt,name=persistent,proto3" json:"persistent,omitempty"` HierarchicalInstanceNames bool `protobuf:"varint,14,opt,name=hierarchical_instance_names,json=hierarchicalInstanceNames,proto3" json:"hierarchical_instance_names,omitempty"` + ChunkingParameters *v2.RepMaxCdcParams `protobuf:"bytes,15,opt,name=chunking_parameters,json=chunkingParameters,proto3" json:"chunking_parameters,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *LocalBlobAccessConfiguration) Reset() { *x = LocalBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -646,7 +732,7 @@ func (x *LocalBlobAccessConfiguration) String() string { func (*LocalBlobAccessConfiguration) ProtoMessage() {} func (x *LocalBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -659,7 +745,7 @@ func (x *LocalBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6} } func (x *LocalBlobAccessConfiguration) GetKeyLocationMapBackend() isLocalBlobAccessConfiguration_KeyLocationMapBackend { @@ -761,6 +847,13 @@ func (x *LocalBlobAccessConfiguration) GetHierarchicalInstanceNames() bool { return false } +func (x *LocalBlobAccessConfiguration) GetChunkingParameters() *v2.RepMaxCdcParams { + if x != nil { + return x.ChunkingParameters + } + return nil +} + type isLocalBlobAccessConfiguration_KeyLocationMapBackend interface { isLocalBlobAccessConfiguration_KeyLocationMapBackend() } @@ -806,7 +899,7 @@ type ExistenceCachingBlobAccessConfiguration struct { func (x *ExistenceCachingBlobAccessConfiguration) Reset() { *x = ExistenceCachingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -818,7 +911,7 @@ func (x *ExistenceCachingBlobAccessConfiguration) String() string { func (*ExistenceCachingBlobAccessConfiguration) ProtoMessage() {} func (x *ExistenceCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -831,7 +924,7 @@ func (x *ExistenceCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Me // Deprecated: Use ExistenceCachingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ExistenceCachingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{7} } func (x *ExistenceCachingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -858,7 +951,7 @@ type CompletenessCheckingBlobAccessConfiguration struct { func (x *CompletenessCheckingBlobAccessConfiguration) Reset() { *x = CompletenessCheckingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -870,7 +963,7 @@ func (x *CompletenessCheckingBlobAccessConfiguration) String() string { func (*CompletenessCheckingBlobAccessConfiguration) ProtoMessage() {} func (x *CompletenessCheckingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -883,7 +976,7 @@ func (x *CompletenessCheckingBlobAccessConfiguration) ProtoReflect() protoreflec // Deprecated: Use CompletenessCheckingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*CompletenessCheckingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{7} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{8} } func (x *CompletenessCheckingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -911,7 +1004,7 @@ type ReadFallbackBlobAccessConfiguration struct { func (x *ReadFallbackBlobAccessConfiguration) Reset() { *x = ReadFallbackBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -923,7 +1016,7 @@ func (x *ReadFallbackBlobAccessConfiguration) String() string { func (*ReadFallbackBlobAccessConfiguration) ProtoMessage() {} func (x *ReadFallbackBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -936,7 +1029,7 @@ func (x *ReadFallbackBlobAccessConfiguration) ProtoReflect() protoreflect.Messag // Deprecated: Use ReadFallbackBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReadFallbackBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{8} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{9} } func (x *ReadFallbackBlobAccessConfiguration) GetPrimary() *BlobAccessConfiguration { @@ -961,19 +1054,19 @@ func (x *ReadFallbackBlobAccessConfiguration) GetReplicator() *BlobReplicatorCon } type ReferenceExpandingBlobAccessConfiguration struct { - state protoimpl.MessageState `protogen:"open.v1"` - IndirectContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` - AwsSession *aws.SessionConfiguration `protobuf:"bytes,2,opt,name=aws_session,json=awsSession,proto3" json:"aws_session,omitempty"` - HttpClient *client.Configuration `protobuf:"bytes,3,opt,name=http_client,json=httpClient,proto3" json:"http_client,omitempty"` - GcpClientOptions *gcp.ClientOptionsConfiguration `protobuf:"bytes,4,opt,name=gcp_client_options,json=gcpClientOptions,proto3" json:"gcp_client_options,omitempty"` - ContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,5,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + IndirectContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` + AwsSession *aws.SessionConfiguration `protobuf:"bytes,2,opt,name=aws_session,json=awsSession,proto3" json:"aws_session,omitempty"` + HttpClient *client.Configuration `protobuf:"bytes,3,opt,name=http_client,json=httpClient,proto3" json:"http_client,omitempty"` + GcpClientOptions *gcp.ClientOptionsConfiguration `protobuf:"bytes,4,opt,name=gcp_client_options,json=gcpClientOptions,proto3" json:"gcp_client_options,omitempty"` + ContentAddressableStorage *ContentAddressableStorageConfiguration `protobuf:"bytes,5,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ReferenceExpandingBlobAccessConfiguration) Reset() { *x = ReferenceExpandingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -985,7 +1078,7 @@ func (x *ReferenceExpandingBlobAccessConfiguration) String() string { func (*ReferenceExpandingBlobAccessConfiguration) ProtoMessage() {} func (x *ReferenceExpandingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -998,7 +1091,7 @@ func (x *ReferenceExpandingBlobAccessConfiguration) ProtoReflect() protoreflect. // Deprecated: Use ReferenceExpandingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReferenceExpandingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{9} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{10} } func (x *ReferenceExpandingBlobAccessConfiguration) GetIndirectContentAddressableStorage() *BlobAccessConfiguration { @@ -1029,7 +1122,7 @@ func (x *ReferenceExpandingBlobAccessConfiguration) GetGcpClientOptions() *gcp.C return nil } -func (x *ReferenceExpandingBlobAccessConfiguration) GetContentAddressableStorage() *BlobAccessConfiguration { +func (x *ReferenceExpandingBlobAccessConfiguration) GetContentAddressableStorage() *ContentAddressableStorageConfiguration { if x != nil { return x.ContentAddressableStorage } @@ -1053,7 +1146,7 @@ type BlobReplicatorConfiguration struct { func (x *BlobReplicatorConfiguration) Reset() { *x = BlobReplicatorConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1065,7 +1158,7 @@ func (x *BlobReplicatorConfiguration) String() string { func (*BlobReplicatorConfiguration) ProtoMessage() {} func (x *BlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1078,7 +1171,7 @@ func (x *BlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobReplicatorConfiguration.ProtoReflect.Descriptor instead. func (*BlobReplicatorConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{10} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{11} } func (x *BlobReplicatorConfiguration) GetMode() isBlobReplicatorConfiguration_Mode { @@ -1192,7 +1285,7 @@ type QueuedBlobReplicatorConfiguration struct { func (x *QueuedBlobReplicatorConfiguration) Reset() { *x = QueuedBlobReplicatorConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1204,7 +1297,7 @@ func (x *QueuedBlobReplicatorConfiguration) String() string { func (*QueuedBlobReplicatorConfiguration) ProtoMessage() {} func (x *QueuedBlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1217,7 +1310,7 @@ func (x *QueuedBlobReplicatorConfiguration) ProtoReflect() protoreflect.Message // Deprecated: Use QueuedBlobReplicatorConfiguration.ProtoReflect.Descriptor instead. func (*QueuedBlobReplicatorConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{11} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{12} } func (x *QueuedBlobReplicatorConfiguration) GetBase() *BlobReplicatorConfiguration { @@ -1244,7 +1337,7 @@ type ConcurrencyLimitingBlobReplicatorConfiguration struct { func (x *ConcurrencyLimitingBlobReplicatorConfiguration) Reset() { *x = ConcurrencyLimitingBlobReplicatorConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1256,7 +1349,7 @@ func (x *ConcurrencyLimitingBlobReplicatorConfiguration) String() string { func (*ConcurrencyLimitingBlobReplicatorConfiguration) ProtoMessage() {} func (x *ConcurrencyLimitingBlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1269,7 +1362,7 @@ func (x *ConcurrencyLimitingBlobReplicatorConfiguration) ProtoReflect() protoref // Deprecated: Use ConcurrencyLimitingBlobReplicatorConfiguration.ProtoReflect.Descriptor instead. func (*ConcurrencyLimitingBlobReplicatorConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{12} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{13} } func (x *ConcurrencyLimitingBlobReplicatorConfiguration) GetBase() *BlobReplicatorConfiguration { @@ -1295,7 +1388,7 @@ type DemultiplexingBlobAccessConfiguration struct { func (x *DemultiplexingBlobAccessConfiguration) Reset() { *x = DemultiplexingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1307,7 +1400,7 @@ func (x *DemultiplexingBlobAccessConfiguration) String() string { func (*DemultiplexingBlobAccessConfiguration) ProtoMessage() {} func (x *DemultiplexingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1320,7 +1413,7 @@ func (x *DemultiplexingBlobAccessConfiguration) ProtoReflect() protoreflect.Mess // Deprecated: Use DemultiplexingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*DemultiplexingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{13} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{14} } func (x *DemultiplexingBlobAccessConfiguration) GetInstanceNamePrefixes() map[string]*DemultiplexedBlobAccessConfiguration { @@ -1340,7 +1433,7 @@ type DemultiplexedBlobAccessConfiguration struct { func (x *DemultiplexedBlobAccessConfiguration) Reset() { *x = DemultiplexedBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1352,7 +1445,7 @@ func (x *DemultiplexedBlobAccessConfiguration) String() string { func (*DemultiplexedBlobAccessConfiguration) ProtoMessage() {} func (x *DemultiplexedBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1365,7 +1458,7 @@ func (x *DemultiplexedBlobAccessConfiguration) ProtoReflect() protoreflect.Messa // Deprecated: Use DemultiplexedBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*DemultiplexedBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{14} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{15} } func (x *DemultiplexedBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -1394,7 +1487,7 @@ type ActionResultExpiringBlobAccessConfiguration struct { func (x *ActionResultExpiringBlobAccessConfiguration) Reset() { *x = ActionResultExpiringBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1406,7 +1499,7 @@ func (x *ActionResultExpiringBlobAccessConfiguration) String() string { func (*ActionResultExpiringBlobAccessConfiguration) ProtoMessage() {} func (x *ActionResultExpiringBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1419,7 +1512,7 @@ func (x *ActionResultExpiringBlobAccessConfiguration) ProtoReflect() protoreflec // Deprecated: Use ActionResultExpiringBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ActionResultExpiringBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{15} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{16} } func (x *ActionResultExpiringBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -1462,7 +1555,7 @@ type ReadCanaryingBlobAccessConfiguration struct { func (x *ReadCanaryingBlobAccessConfiguration) Reset() { *x = ReadCanaryingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1474,7 +1567,7 @@ func (x *ReadCanaryingBlobAccessConfiguration) String() string { func (*ReadCanaryingBlobAccessConfiguration) ProtoMessage() {} func (x *ReadCanaryingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1487,7 +1580,7 @@ func (x *ReadCanaryingBlobAccessConfiguration) ProtoReflect() protoreflect.Messa // Deprecated: Use ReadCanaryingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReadCanaryingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{16} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{17} } func (x *ReadCanaryingBlobAccessConfiguration) GetSource() *BlobAccessConfiguration { @@ -1528,7 +1621,7 @@ type ZIPBlobAccessConfiguration struct { func (x *ZIPBlobAccessConfiguration) Reset() { *x = ZIPBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1540,7 +1633,7 @@ func (x *ZIPBlobAccessConfiguration) String() string { func (*ZIPBlobAccessConfiguration) ProtoMessage() {} func (x *ZIPBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1553,7 +1646,7 @@ func (x *ZIPBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ZIPBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ZIPBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{17} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{18} } func (x *ZIPBlobAccessConfiguration) GetPath() string { @@ -1580,7 +1673,7 @@ type WithLabelsBlobAccessConfiguration struct { func (x *WithLabelsBlobAccessConfiguration) Reset() { *x = WithLabelsBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1592,7 +1685,7 @@ func (x *WithLabelsBlobAccessConfiguration) String() string { func (*WithLabelsBlobAccessConfiguration) ProtoMessage() {} func (x *WithLabelsBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1605,7 +1698,7 @@ func (x *WithLabelsBlobAccessConfiguration) ProtoReflect() protoreflect.Message // Deprecated: Use WithLabelsBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*WithLabelsBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{18} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{19} } func (x *WithLabelsBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -1632,7 +1725,7 @@ type DeadlineEnforcingBlobAccess struct { func (x *DeadlineEnforcingBlobAccess) Reset() { *x = DeadlineEnforcingBlobAccess{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1644,7 +1737,7 @@ func (x *DeadlineEnforcingBlobAccess) String() string { func (*DeadlineEnforcingBlobAccess) ProtoMessage() {} func (x *DeadlineEnforcingBlobAccess) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1657,7 +1750,7 @@ func (x *DeadlineEnforcingBlobAccess) ProtoReflect() protoreflect.Message { // Deprecated: Use DeadlineEnforcingBlobAccess.ProtoReflect.Descriptor instead. func (*DeadlineEnforcingBlobAccess) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{19} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{20} } func (x *DeadlineEnforcingBlobAccess) GetTimeout() *durationpb.Duration { @@ -1684,7 +1777,7 @@ type GrpcBlobAccessConfiguration struct { func (x *GrpcBlobAccessConfiguration) Reset() { *x = GrpcBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1696,7 +1789,7 @@ func (x *GrpcBlobAccessConfiguration) String() string { func (*GrpcBlobAccessConfiguration) ProtoMessage() {} func (x *GrpcBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1709,7 +1802,7 @@ func (x *GrpcBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use GrpcBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*GrpcBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{20} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{21} } func (x *GrpcBlobAccessConfiguration) GetClient() *grpc.ClientConfiguration { @@ -1726,6 +1819,50 @@ func (x *GrpcBlobAccessConfiguration) GetEnableCompression() bool { return false } +type ChunkListValidatingBlobAccessConfiguration struct { + state protoimpl.MessageState `protogen:"open.v1"` + Backend *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChunkListValidatingBlobAccessConfiguration) Reset() { + *x = ChunkListValidatingBlobAccessConfiguration{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkListValidatingBlobAccessConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkListValidatingBlobAccessConfiguration) ProtoMessage() {} + +func (x *ChunkListValidatingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkListValidatingBlobAccessConfiguration.ProtoReflect.Descriptor instead. +func (*ChunkListValidatingBlobAccessConfiguration) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{22} +} + +func (x *ChunkListValidatingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { + if x != nil { + return x.Backend + } + return nil +} + type ShardingBlobAccessConfiguration_Shard struct { state protoimpl.MessageState `protogen:"open.v1"` Backend *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` @@ -1736,7 +1873,7 @@ type ShardingBlobAccessConfiguration_Shard struct { func (x *ShardingBlobAccessConfiguration_Shard) Reset() { *x = ShardingBlobAccessConfiguration_Shard{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1748,7 +1885,7 @@ func (x *ShardingBlobAccessConfiguration_Shard) String() string { func (*ShardingBlobAccessConfiguration_Shard) ProtoMessage() {} func (x *ShardingBlobAccessConfiguration_Shard) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1761,7 +1898,7 @@ func (x *ShardingBlobAccessConfiguration_Shard) ProtoReflect() protoreflect.Mess // Deprecated: Use ShardingBlobAccessConfiguration_Shard.ProtoReflect.Descriptor instead. func (*ShardingBlobAccessConfiguration_Shard) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{3, 0} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{4, 0} } func (x *ShardingBlobAccessConfiguration_Shard) GetBackend() *BlobAccessConfiguration { @@ -1787,7 +1924,7 @@ type LocalBlobAccessConfiguration_KeyLocationMapInMemory struct { func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) Reset() { *x = LocalBlobAccessConfiguration_KeyLocationMapInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1799,7 +1936,7 @@ func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) String() string { func (*LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1812,7 +1949,7 @@ func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoReflect() pro // Deprecated: Use LocalBlobAccessConfiguration_KeyLocationMapInMemory.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_KeyLocationMapInMemory) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 0} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 0} } func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) GetEntries() int64 { @@ -1831,7 +1968,7 @@ type LocalBlobAccessConfiguration_BlocksInMemory struct { func (x *LocalBlobAccessConfiguration_BlocksInMemory) Reset() { *x = LocalBlobAccessConfiguration_BlocksInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1843,7 +1980,7 @@ func (x *LocalBlobAccessConfiguration_BlocksInMemory) String() string { func (*LocalBlobAccessConfiguration_BlocksInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1856,7 +1993,7 @@ func (x *LocalBlobAccessConfiguration_BlocksInMemory) ProtoReflect() protoreflec // Deprecated: Use LocalBlobAccessConfiguration_BlocksInMemory.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_BlocksInMemory) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 1} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 1} } func (x *LocalBlobAccessConfiguration_BlocksInMemory) GetBlockSizeBytes() int64 { @@ -1877,7 +2014,7 @@ type LocalBlobAccessConfiguration_BlocksOnBlockDevice struct { func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) Reset() { *x = LocalBlobAccessConfiguration_BlocksOnBlockDevice{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1889,7 +2026,7 @@ func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) String() string { func (*LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1902,7 +2039,7 @@ func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoReflect() protor // Deprecated: Use LocalBlobAccessConfiguration_BlocksOnBlockDevice.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_BlocksOnBlockDevice) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 2} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 2} } func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) GetSource() *blockdevice.Configuration { @@ -1936,7 +2073,7 @@ type LocalBlobAccessConfiguration_Persistent struct { func (x *LocalBlobAccessConfiguration_Persistent) Reset() { *x = LocalBlobAccessConfiguration_Persistent{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1948,7 +2085,7 @@ func (x *LocalBlobAccessConfiguration_Persistent) String() string { func (*LocalBlobAccessConfiguration_Persistent) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_Persistent) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1961,7 +2098,7 @@ func (x *LocalBlobAccessConfiguration_Persistent) ProtoReflect() protoreflect.Me // Deprecated: Use LocalBlobAccessConfiguration_Persistent.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_Persistent) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 3} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 3} } func (x *LocalBlobAccessConfiguration_Persistent) GetStateDirectoryPath() string { @@ -1982,10 +2119,15 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc = "" + "\n" + - "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xf3\x01\n" + - "\x16BlobstoreConfiguration\x12z\n" + - "\x1bcontent_addressable_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x19contentAddressableStorage\x12]\n" + - "\faction_cache\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\vactionCache\"\xe3\x0f\n" + + "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xc7\x03\n" + + "&ContentAddressableStorageConfiguration\x12_\n" + + "\rchunk_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\fchunkStorage\x12h\n" + + "\x12chunk_list_storage\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x10chunkListStorage\x12e\n" + + "\x10chunk_list_cache\x18\x03 \x01(\v2;.buildbarn.configuration.digest.ExistenceCacheConfigurationR\x0echunkListCache\x12k\n" + + "\x13cdc_parameter_cache\x18\x04 \x01(\v2;.buildbarn.configuration.digest.ExistenceCacheConfigurationR\x11cdcParameterCache\"\x83\x02\n" + + "\x16BlobstoreConfiguration\x12\x89\x01\n" + + "\x1bcontent_addressable_storage\x18\x01 \x01(\v2I.buildbarn.configuration.blobstore.ContentAddressableStorageConfigurationR\x19contentAddressableStorage\x12]\n" + + "\faction_cache\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\vactionCache\"\xe9\x10\n" + "\x17BlobAccessConfiguration\x12j\n" + "\fread_caching\x18\x04 \x01(\v2E.buildbarn.configuration.blobstore.ReadCachingBlobAccessConfigurationH\x00R\vreadCaching\x12T\n" + "\x04grpc\x18\a \x01(\v2>.buildbarn.configuration.blobstore.GrpcBlobAccessConfigurationH\x00R\x04grpc\x12*\n" + @@ -2008,7 +2150,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\vwith_labels\x18\x1a \x01(\v2D.buildbarn.configuration.blobstore.WithLabelsBlobAccessConfigurationH\x00R\n" + "withLabels\x12\x16\n" + "\x05label\x18\x1b \x01(\tH\x00R\x05label\x12o\n" + - "\x12deadline_enforcing\x18\x1c \x01(\v2>.buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccessH\x00R\x11deadlineEnforcingB\t\n" + + "\x12deadline_enforcing\x18\x1c \x01(\v2>.buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccessH\x00R\x11deadlineEnforcing\x12\x83\x01\n" + + "\x15chunk_list_validating\x18\x1d \x01(\v2M.buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfigurationH\x00R\x13chunkListValidatingB\t\n" + "\abackendJ\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x05\x10\x06J\x04\b\x06\x10\aJ\x04\b\n" + "\x10\v\"\xa4\x02\n" + "\"ReadCachingBlobAccessConfiguration\x12N\n" + @@ -2029,7 +2172,7 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\tbackend_a\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\bbackendA\x12W\n" + "\tbackend_b\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\bbackendB\x12i\n" + "\x11replicator_a_to_b\x18\x03 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorAToB\x12i\n" + - "\x11replicator_b_to_a\x18\x04 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorBToA\"\xb6\f\n" + + "\x11replicator_b_to_a\x18\x04 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorBToA\"\x99\r\n" + "\x1cLocalBlobAccessConfiguration\x12\x94\x01\n" + "\x1akey_location_map_in_memory\x18\v \x01(\v2V.buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemoryH\x00R\x16keyLocationMapInMemory\x12{\n" + " key_location_map_on_block_device\x18\f \x01(\v22.buildbarn.configuration.blockdevice.ConfigurationH\x00R\x1bkeyLocationMapOnBlockDevice\x12O\n" + @@ -2046,7 +2189,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\n" + "persistent\x18\r \x01(\v2J.buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.PersistentR\n" + "persistent\x12>\n" + - "\x1bhierarchical_instance_names\x18\x0e \x01(\bR\x19hierarchicalInstanceNames\x1a2\n" + + "\x1bhierarchical_instance_names\x18\x0e \x01(\bR\x19hierarchicalInstanceNames\x12a\n" + + "\x13chunking_parameters\x18\x0f \x01(\v20.build.bazel.remote.execution.v2.RepMaxCdcParamsR\x12chunkingParameters\x1a2\n" + "\x16KeyLocationMapInMemory\x12\x18\n" + "\aentries\x18\x01 \x01(\x03R\aentries\x1a:\n" + "\x0eBlocksInMemory\x12(\n" + @@ -2072,15 +2216,15 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\tsecondary\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\tsecondary\x12^\n" + "\n" + "replicator\x18\x03 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\n" + - "replicator\"\xd1\x04\n" + + "replicator\"\xe1\x04\n" + ")ReferenceExpandingBlobAccessConfiguration\x12\x8b\x01\n" + "$indirect_content_addressable_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR!indirectContentAddressableStorage\x12X\n" + "\vaws_session\x18\x02 \x01(\v27.buildbarn.configuration.cloud.aws.SessionConfigurationR\n" + "awsSession\x12S\n" + "\vhttp_client\x18\x03 \x01(\v22.buildbarn.configuration.http.client.ConfigurationR\n" + "httpClient\x12k\n" + - "\x12gcp_client_options\x18\x04 \x01(\v2=.buildbarn.configuration.cloud.gcp.ClientOptionsConfigurationR\x10gcpClientOptions\x12z\n" + - "\x1bcontent_addressable_storage\x18\x05 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x19contentAddressableStorage\"\xa1\x04\n" + + "\x12gcp_client_options\x18\x04 \x01(\v2=.buildbarn.configuration.cloud.gcp.ClientOptionsConfigurationR\x10gcpClientOptions\x12\x89\x01\n" + + "\x1bcontent_addressable_storage\x18\x05 \x01(\v2I.buildbarn.configuration.blobstore.ContentAddressableStorageConfigurationR\x19contentAddressableStorage\"\xa1\x04\n" + "\x1bBlobReplicatorConfiguration\x12.\n" + "\x05local\x18\x01 \x01(\v2\x16.google.protobuf.EmptyH\x00R\x05local\x12K\n" + "\x06remote\x18\x02 \x01(\v21.buildbarn.configuration.grpc.ClientConfigurationH\x00R\x06remote\x12^\n" + @@ -2127,7 +2271,9 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\abackend\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackend\"\x97\x01\n" + "\x1bGrpcBlobAccessConfiguration\x12I\n" + "\x06client\x18\x01 \x01(\v21.buildbarn.configuration.grpc.ClientConfigurationR\x06client\x12-\n" + - "\x12enable_compression\x18\x02 \x01(\bR\x11enableCompressionBCZAgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstoreb\x06proto3" + "\x12enable_compression\x18\x02 \x01(\bR\x11enableCompression\"\x82\x01\n" + + "*ChunkListValidatingBlobAccessConfiguration\x12T\n" + + "\abackend\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackendBCZAgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstoreb\x06proto3" var ( file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescOnce sync.Once @@ -2141,129 +2287,139 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescData } -var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 31) var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_goTypes = []any{ - (*BlobstoreConfiguration)(nil), // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration - (*BlobAccessConfiguration)(nil), // 1: buildbarn.configuration.blobstore.BlobAccessConfiguration - (*ReadCachingBlobAccessConfiguration)(nil), // 2: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration - (*ShardingBlobAccessConfiguration)(nil), // 3: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration - (*MirroredBlobAccessConfiguration)(nil), // 4: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration - (*LocalBlobAccessConfiguration)(nil), // 5: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration - (*ExistenceCachingBlobAccessConfiguration)(nil), // 6: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration - (*CompletenessCheckingBlobAccessConfiguration)(nil), // 7: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration - (*ReadFallbackBlobAccessConfiguration)(nil), // 8: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration - (*ReferenceExpandingBlobAccessConfiguration)(nil), // 9: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration - (*BlobReplicatorConfiguration)(nil), // 10: buildbarn.configuration.blobstore.BlobReplicatorConfiguration - (*QueuedBlobReplicatorConfiguration)(nil), // 11: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration - (*ConcurrencyLimitingBlobReplicatorConfiguration)(nil), // 12: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration - (*DemultiplexingBlobAccessConfiguration)(nil), // 13: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration - (*DemultiplexedBlobAccessConfiguration)(nil), // 14: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration - (*ActionResultExpiringBlobAccessConfiguration)(nil), // 15: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration - (*ReadCanaryingBlobAccessConfiguration)(nil), // 16: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration - (*ZIPBlobAccessConfiguration)(nil), // 17: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration - (*WithLabelsBlobAccessConfiguration)(nil), // 18: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration - (*DeadlineEnforcingBlobAccess)(nil), // 19: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess - (*GrpcBlobAccessConfiguration)(nil), // 20: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - (*ShardingBlobAccessConfiguration_Shard)(nil), // 21: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - nil, // 22: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 23: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 24: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - (*LocalBlobAccessConfiguration_Persistent)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - nil, // 27: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - nil, // 28: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - (*status.Status)(nil), // 29: google.rpc.Status - (*blockdevice.Configuration)(nil), // 30: buildbarn.configuration.blockdevice.Configuration + (*ContentAddressableStorageConfiguration)(nil), // 0: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + (*BlobstoreConfiguration)(nil), // 1: buildbarn.configuration.blobstore.BlobstoreConfiguration + (*BlobAccessConfiguration)(nil), // 2: buildbarn.configuration.blobstore.BlobAccessConfiguration + (*ReadCachingBlobAccessConfiguration)(nil), // 3: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration + (*ShardingBlobAccessConfiguration)(nil), // 4: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration + (*MirroredBlobAccessConfiguration)(nil), // 5: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration + (*LocalBlobAccessConfiguration)(nil), // 6: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration + (*ExistenceCachingBlobAccessConfiguration)(nil), // 7: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration + (*CompletenessCheckingBlobAccessConfiguration)(nil), // 8: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration + (*ReadFallbackBlobAccessConfiguration)(nil), // 9: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration + (*ReferenceExpandingBlobAccessConfiguration)(nil), // 10: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration + (*BlobReplicatorConfiguration)(nil), // 11: buildbarn.configuration.blobstore.BlobReplicatorConfiguration + (*QueuedBlobReplicatorConfiguration)(nil), // 12: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration + (*ConcurrencyLimitingBlobReplicatorConfiguration)(nil), // 13: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration + (*DemultiplexingBlobAccessConfiguration)(nil), // 14: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration + (*DemultiplexedBlobAccessConfiguration)(nil), // 15: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration + (*ActionResultExpiringBlobAccessConfiguration)(nil), // 16: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration + (*ReadCanaryingBlobAccessConfiguration)(nil), // 17: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration + (*ZIPBlobAccessConfiguration)(nil), // 18: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration + (*WithLabelsBlobAccessConfiguration)(nil), // 19: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration + (*DeadlineEnforcingBlobAccess)(nil), // 20: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess + (*GrpcBlobAccessConfiguration)(nil), // 21: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration + (*ChunkListValidatingBlobAccessConfiguration)(nil), // 22: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + (*ShardingBlobAccessConfiguration_Shard)(nil), // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + nil, // 24: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 27: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + (*LocalBlobAccessConfiguration_Persistent)(nil), // 28: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + nil, // 29: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + nil, // 30: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry (*digest.ExistenceCacheConfiguration)(nil), // 31: buildbarn.configuration.digest.ExistenceCacheConfiguration - (*aws.SessionConfiguration)(nil), // 32: buildbarn.configuration.cloud.aws.SessionConfiguration - (*client.Configuration)(nil), // 33: buildbarn.configuration.http.client.Configuration - (*gcp.ClientOptionsConfiguration)(nil), // 34: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - (*emptypb.Empty)(nil), // 35: google.protobuf.Empty - (*grpc.ClientConfiguration)(nil), // 36: buildbarn.configuration.grpc.ClientConfiguration - (*durationpb.Duration)(nil), // 37: google.protobuf.Duration - (*timestamppb.Timestamp)(nil), // 38: google.protobuf.Timestamp + (*status.Status)(nil), // 32: google.rpc.Status + (*blockdevice.Configuration)(nil), // 33: buildbarn.configuration.blockdevice.Configuration + (*v2.RepMaxCdcParams)(nil), // 34: build.bazel.remote.execution.v2.RepMaxCdcParams + (*aws.SessionConfiguration)(nil), // 35: buildbarn.configuration.cloud.aws.SessionConfiguration + (*client.Configuration)(nil), // 36: buildbarn.configuration.http.client.Configuration + (*gcp.ClientOptionsConfiguration)(nil), // 37: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + (*emptypb.Empty)(nil), // 38: google.protobuf.Empty + (*grpc.ClientConfiguration)(nil), // 39: buildbarn.configuration.grpc.ClientConfiguration + (*durationpb.Duration)(nil), // 40: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp } var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_depIdxs = []int32{ - 1, // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 1: buildbarn.configuration.blobstore.BlobstoreConfiguration.action_cache:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 2, // 2: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_caching:type_name -> buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration - 20, // 3: buildbarn.configuration.blobstore.BlobAccessConfiguration.grpc:type_name -> buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - 29, // 4: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status - 3, // 5: buildbarn.configuration.blobstore.BlobAccessConfiguration.sharding:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration - 4, // 6: buildbarn.configuration.blobstore.BlobAccessConfiguration.mirrored:type_name -> buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration - 5, // 7: buildbarn.configuration.blobstore.BlobAccessConfiguration.local:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration - 6, // 8: buildbarn.configuration.blobstore.BlobAccessConfiguration.existence_caching:type_name -> buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration - 7, // 9: buildbarn.configuration.blobstore.BlobAccessConfiguration.completeness_checking:type_name -> buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration - 8, // 10: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_fallback:type_name -> buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration - 9, // 11: buildbarn.configuration.blobstore.BlobAccessConfiguration.reference_expanding:type_name -> buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration - 13, // 12: buildbarn.configuration.blobstore.BlobAccessConfiguration.demultiplexing:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration - 1, // 13: buildbarn.configuration.blobstore.BlobAccessConfiguration.hierarchical_instance_names:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 15, // 14: buildbarn.configuration.blobstore.BlobAccessConfiguration.action_result_expiring:type_name -> buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration - 16, // 15: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_canarying:type_name -> buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration - 17, // 16: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_reading:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration - 17, // 17: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_writing:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration - 18, // 18: buildbarn.configuration.blobstore.BlobAccessConfiguration.with_labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration - 19, // 19: buildbarn.configuration.blobstore.BlobAccessConfiguration.deadline_enforcing:type_name -> buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess - 1, // 20: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 21: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 22: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 22, // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - 1, // 24: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 25: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 26: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 10, // 27: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 23, // 28: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - 30, // 29: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration - 24, // 30: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - 25, // 31: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - 26, // 32: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - 1, // 33: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 31, // 34: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 35: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 36: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 37: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 38: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 1, // 39: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 32, // 40: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration - 33, // 41: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration - 34, // 42: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - 1, // 43: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 35, // 44: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty - 36, // 45: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 11, // 46: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration - 35, // 47: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty - 10, // 48: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 12, // 49: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration - 10, // 50: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 31, // 51: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 10, // 52: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 27, // 53: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - 1, // 54: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 55: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 37, // 56: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration - 37, // 57: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration - 38, // 58: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp - 1, // 59: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 60: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 37, // 61: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration - 31, // 62: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 63: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 28, // 64: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - 37, // 65: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration - 1, // 66: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 36, // 67: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 1, // 68: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 21, // 69: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - 30, // 70: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration - 31, // 71: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 37, // 72: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration - 14, // 73: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration - 1, // 74: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 75, // [75:75] is the sub-list for method output_type - 75, // [75:75] is the sub-list for method input_type - 75, // [75:75] is the sub-list for extension type_name - 75, // [75:75] is the sub-list for extension extendee - 0, // [0:75] is the sub-list for field type_name + 2, // 0: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.chunk_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 1: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.chunk_list_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 31, // 2: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.chunk_list_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 31, // 3: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.cdc_parameter_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 0, // 4: buildbarn.configuration.blobstore.BlobstoreConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + 2, // 5: buildbarn.configuration.blobstore.BlobstoreConfiguration.action_cache:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 3, // 6: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_caching:type_name -> buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration + 21, // 7: buildbarn.configuration.blobstore.BlobAccessConfiguration.grpc:type_name -> buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration + 32, // 8: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status + 4, // 9: buildbarn.configuration.blobstore.BlobAccessConfiguration.sharding:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration + 5, // 10: buildbarn.configuration.blobstore.BlobAccessConfiguration.mirrored:type_name -> buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration + 6, // 11: buildbarn.configuration.blobstore.BlobAccessConfiguration.local:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration + 7, // 12: buildbarn.configuration.blobstore.BlobAccessConfiguration.existence_caching:type_name -> buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration + 8, // 13: buildbarn.configuration.blobstore.BlobAccessConfiguration.completeness_checking:type_name -> buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration + 9, // 14: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_fallback:type_name -> buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration + 10, // 15: buildbarn.configuration.blobstore.BlobAccessConfiguration.reference_expanding:type_name -> buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration + 14, // 16: buildbarn.configuration.blobstore.BlobAccessConfiguration.demultiplexing:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration + 2, // 17: buildbarn.configuration.blobstore.BlobAccessConfiguration.hierarchical_instance_names:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 16, // 18: buildbarn.configuration.blobstore.BlobAccessConfiguration.action_result_expiring:type_name -> buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration + 17, // 19: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_canarying:type_name -> buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration + 18, // 20: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_reading:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration + 18, // 21: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_writing:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration + 19, // 22: buildbarn.configuration.blobstore.BlobAccessConfiguration.with_labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration + 20, // 23: buildbarn.configuration.blobstore.BlobAccessConfiguration.deadline_enforcing:type_name -> buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess + 22, // 24: buildbarn.configuration.blobstore.BlobAccessConfiguration.chunk_list_validating:type_name -> buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + 2, // 25: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 26: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 11, // 27: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 24, // 28: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + 2, // 29: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 30: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 11, // 31: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 11, // 32: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 25, // 33: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + 33, // 34: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration + 26, // 35: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + 27, // 36: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + 28, // 37: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + 34, // 38: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.chunking_parameters:type_name -> build.bazel.remote.execution.v2.RepMaxCdcParams + 2, // 39: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 31, // 40: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 2, // 41: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 42: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 43: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 11, // 44: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 2, // 45: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 35, // 46: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration + 36, // 47: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration + 37, // 48: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + 0, // 49: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + 38, // 50: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty + 39, // 51: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 12, // 52: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration + 38, // 53: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty + 11, // 54: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 13, // 55: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration + 11, // 56: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 31, // 57: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 11, // 58: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 29, // 59: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + 2, // 60: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 61: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 40, // 62: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration + 40, // 63: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration + 41, // 64: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp + 2, // 65: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 66: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 40, // 67: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration + 31, // 68: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 2, // 69: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 30, // 70: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry + 40, // 71: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration + 2, // 72: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 39, // 73: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 2, // 74: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 75: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 23, // 76: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + 33, // 77: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration + 31, // 78: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 40, // 79: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration + 15, // 80: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration + 2, // 81: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 82, // [82:82] is the sub-list for method output_type + 82, // [82:82] is the sub-list for method input_type + 82, // [82:82] is the sub-list for extension type_name + 82, // [82:82] is the sub-list for extension extendee + 0, // [0:82] is the sub-list for field type_name } func init() { @@ -2273,7 +2429,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob if File_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto != nil { return } - file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1].OneofWrappers = []any{ + file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2].OneofWrappers = []any{ (*BlobAccessConfiguration_ReadCaching)(nil), (*BlobAccessConfiguration_Grpc)(nil), (*BlobAccessConfiguration_Error)(nil), @@ -2293,14 +2449,15 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob (*BlobAccessConfiguration_WithLabels)(nil), (*BlobAccessConfiguration_Label)(nil), (*BlobAccessConfiguration_DeadlineEnforcing)(nil), + (*BlobAccessConfiguration_ChunkListValidating)(nil), } - file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5].OneofWrappers = []any{ + file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6].OneofWrappers = []any{ (*LocalBlobAccessConfiguration_KeyLocationMapInMemory_)(nil), (*LocalBlobAccessConfiguration_KeyLocationMapOnBlockDevice)(nil), (*LocalBlobAccessConfiguration_BlocksInMemory_)(nil), (*LocalBlobAccessConfiguration_BlocksOnBlockDevice_)(nil), } - file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10].OneofWrappers = []any{ + file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11].OneofWrappers = []any{ (*BlobReplicatorConfiguration_Local)(nil), (*BlobReplicatorConfiguration_Remote)(nil), (*BlobReplicatorConfiguration_Queued)(nil), @@ -2314,7 +2471,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc)), NumEnums: 0, - NumMessages: 29, + NumMessages: 31, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/configuration/blobstore/blobstore.proto b/pkg/proto/configuration/blobstore/blobstore.proto index 68b226711..734c05d1e 100644 --- a/pkg/proto/configuration/blobstore/blobstore.proto +++ b/pkg/proto/configuration/blobstore/blobstore.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package buildbarn.configuration.blobstore; +import "build/bazel/remote/execution/v2/remote_execution.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto"; @@ -15,10 +16,35 @@ import "google/rpc/status.proto"; option go_package = "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore"; +message ContentAddressableStorageConfiguration { + // Storage configuration for the Chunk Storage (CS). + BlobAccessConfiguration chunk_storage = 1; + + // Storage configuration for the Chunk List Storage (CLS). + BlobAccessConfiguration chunk_list_storage = 2; + + // Optional: In memory cache for parsed object from the CLS. This + // cache is particularly useful for consumers which need to random + // access reads to the same digest often such as bb-clientd and + // bb-worker. + buildbarn.configuration.digest.ExistenceCacheConfiguration chunk_list_cache = + 3; + + // Optional: In memory cache for CDC parameters per instance. When the + // chunk storage is remote this cache prevents each operation that + // interacts with CDC from having to fetch the capabilities from the + // upstream CS for every interaction with the CDC subsystem. It can be + // set to a relatively small cache with short retention as the number + // of instance names is typically small and the cost of a cache miss + // is relatively low. + buildbarn.configuration.digest.ExistenceCacheConfiguration + cdc_parameter_cache = 4; +} + // Storage configuration for Bazel Buildbarn. message BlobstoreConfiguration { // Storage configuration for the Content Addressable Storage (CAS). - BlobAccessConfiguration content_addressable_storage = 1; + ContentAddressableStorageConfiguration content_addressable_storage = 1; // Storage configuration for the Action Cache (AC). BlobAccessConfiguration action_cache = 2; @@ -199,6 +225,28 @@ message BlobAccessConfiguration { // value. When gRPC calls are timed out a `DEADLINE_EXCEEDED` error // code will be returned. DeadlineEnforcingBlobAccess deadline_enforcing = 28; + + // Validate that chunk list requests fulfill the API specification. + // Chunk List Storage (CLS) is used to implement the SplitBlob and + // SpliceBlob methods from the REv2 API. The protocol has several + // demands on the state of the Content Addressable Storage (CAS) + // after those methods have been called. + // + // SplitBlob requires that the blob as well as all chunks of the + // blob are available in the CAS, they have their lifetime renewed + // and that the resulting chunk list composes into the blob. This is + // validated by the Get path. + // + // SpliceBlob requires that the blob as well as all chunks of the + // blob are present in storage, they have their lifetime renewed and + // that the supplied chunk list composes into the blob. Notably it + // does not require the chunks to follow any particular chunking + // algorithm but our implementation ensures that after any call a + // proper RepMaxCDC chunk list is verified even if the caller + // supplied a different chunk list. + // + // This decorator is only supported for the CLS. + ChunkListValidatingBlobAccessConfiguration chunk_list_validating = 29; } // Was 'redis'. Instead of using Redis, one may run a separate @@ -642,6 +690,18 @@ message LocalBlobAccessConfiguration { // level, e.g., on top of CompletenessCheckingBlobAccess. This can be // achieved by using HierarchicalInstanceNamesBlobAccess. bool hierarchical_instance_names = 14; + + // The chunking parameters advertised via GetCapabilities, setting + // this announces support for the SplitBlob and SpliceBlob api calls. + // + // Adding the chunking parameters does not make the underlying + // blobstore enforce the chunking parameters or implement the split + // and splice blob. There must be a configured Chunk List Storage with + // an outer ChunkValidatingBlobAccess that has view of the entire + // Content Addressable Storage (CAS). + // + // This option is only supported for the CS. + build.bazel.remote.execution.v2.RepMaxCdcParams chunking_parameters = 15; } message ExistenceCachingBlobAccessConfiguration { @@ -705,7 +765,7 @@ message ReferenceExpandingBlobAccessConfiguration { // Optional: Storage backend to use when Reference objects refer to // objects stored in another Content Addressable Storage. - BlobAccessConfiguration content_addressable_storage = 5; + ContentAddressableStorageConfiguration content_addressable_storage = 5; } message BlobReplicatorConfiguration { @@ -937,3 +997,8 @@ message GrpcBlobAccessConfiguration { // types (AC, ICAS, etc.). bool enable_compression = 2; } + +message ChunkListValidatingBlobAccessConfiguration { + // The backend to which validated operations are delegated. + BlobAccessConfiguration backend = 1; +} diff --git a/pkg/storage/BUILD.bazel b/pkg/storage/BUILD.bazel new file mode 100644 index 000000000..0708f6cc6 --- /dev/null +++ b/pkg/storage/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +# Exported for mock generation purposes. +exports_files(["message_reader.go"]) + +go_library( + name = "storage", + srcs = ["message_reader.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/storage", + visibility = ["//visibility:public"], + deps = [ + "//pkg/digest", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/pkg/storage/message_reader.go b/pkg/storage/message_reader.go new file mode 100644 index 000000000..b5829da30 --- /dev/null +++ b/pkg/storage/message_reader.go @@ -0,0 +1,19 @@ +package storage + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/protobuf/proto" +) + +// MessageReader can be used to read a proto.Message from storage and +// return its contents. +type MessageReader[T proto.Message] interface { + // Return a parsed message from storage. + // + // Implementations may use the provided message to store the + // unmarshalled message. + ReadMessage(ctx context.Context, d digest.Digest, msg T) (T, error) +} diff --git a/pkg/testutil/BUILD.bazel b/pkg/testutil/BUILD.bazel index 733a24b27..c3564d7ba 100644 --- a/pkg/testutil/BUILD.bazel +++ b/pkg/testutil/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/buildbarn/bb-storage/pkg/testutil", visibility = ["//visibility:public"], deps = [ + "//pkg/filesystem", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//status", "@org_golang_google_protobuf//encoding/protojson", diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go index 6fc3172b0..bc9dbe4d8 100644 --- a/pkg/testutil/testutil.go +++ b/pkg/testutil/testutil.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/buildbarn/bb-storage/pkg/filesystem" "github.com/stretchr/testify/require" "google.golang.org/grpc/status" @@ -14,6 +15,22 @@ import ( "go.uber.org/mock/gomock" ) +// FileReaderIsProto is a helper function that requires a +// filesystem.FileReader to contain the binary representation of a +// specific proto object. The resulting content gets stored in the +// actual parameter. +func FileReaderIsProto[T proto.Message](t *testing.T, r filesystem.FileReader, expected, actual T) { + t.Helper() + len, err := r.Len() + require.NoError(t, err) + bytes := make([]byte, len) + n, err := r.ReadAt(bytes, 0) + require.NoError(t, err) + require.Equal(t, len, int64(n)) + require.NoError(t, proto.Unmarshal(bytes, actual)) + RequireEqualProto(t, expected, actual) +} + // RequireEqualProto asserts that the two passed protocol buffer // messages are equal. // diff --git a/pkg/ttlcache/BUILD.bazel b/pkg/ttlcache/BUILD.bazel new file mode 100644 index 000000000..ce11025da --- /dev/null +++ b/pkg/ttlcache/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "ttlcache", + srcs = ["ttl_cache.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/ttlcache", + visibility = ["//visibility:public"], + deps = [ + "//pkg/clock", + "//pkg/eviction", + "//pkg/proto/configuration/digest", + "//pkg/util", + ], +) diff --git a/pkg/ttlcache/ttl_cache.go b/pkg/ttlcache/ttl_cache.go new file mode 100644 index 000000000..2e2039a49 --- /dev/null +++ b/pkg/ttlcache/ttl_cache.go @@ -0,0 +1,97 @@ +package ttlcache + +import ( + "sync" + "time" + + "github.com/buildbarn/bb-storage/pkg/clock" + "github.com/buildbarn/bb-storage/pkg/eviction" + digest_pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/digest" + "github.com/buildbarn/bb-storage/pkg/util" +) + +type cachedItem[V any] struct { + value V + expiration time.Time +} + +// TTLCache provides a generic, concurrency-safe cache with TTL and +// eviction. +type TTLCache[K comparable, V any] struct { + clock clock.Clock + evictionSet eviction.Set[K] + maxItems int + cacheDuration time.Duration + + lock sync.Mutex + items map[K]cachedItem[V] +} + +// NewTTLCache instantiates a reusable TTLCache for any key-value pair. +func NewTTLCache[K comparable, V any](clock clock.Clock, evictionSet eviction.Set[K], maxItems int, cacheDuration time.Duration) *TTLCache[K, V] { + return &TTLCache[K, V]{ + clock: clock, + evictionSet: evictionSet, + maxItems: maxItems, + cacheDuration: cacheDuration, + items: make(map[K]cachedItem[V]), + } +} + +// NewTTLCacheFromConfiguration wraps NewTTLCache with the parameters +// specified in a configuration message. +func NewTTLCacheFromConfiguration[K comparable, V any](configuration *digest_pb.ExistenceCacheConfiguration, clock clock.Clock, name string) (*TTLCache[K, V], error) { + cacheDuration := configuration.CacheDuration + if err := cacheDuration.CheckValid(); err != nil { + return nil, util.StatusWrap(err, "Invalid cache duration") + } + evictionSet, err := eviction.NewSetFromConfiguration[K](configuration.CacheReplacementPolicy) + if err != nil { + return nil, util.StatusWrap(err, "Failed to create eviction set") + } + return NewTTLCache[K, V]( + clock, + eviction.NewMetricsSet(evictionSet, name), + int(configuration.CacheSize), + cacheDuration.AsDuration(), + ), nil +} + +// Get retrieves an item if it exists and hasn't expired. +func (c *TTLCache[K, V]) Get(key K) (V, bool) { + c.lock.Lock() + defer c.lock.Unlock() + + if cached, ok := c.items[key]; ok { + if !c.clock.Now().After(cached.expiration) { + c.evictionSet.Touch(key) + return cached.value, true + } + } + + var zero V + return zero, false +} + +// Put inserts or updates an item in the cache, handling eviction if at +// capacity. +func (c *TTLCache[K, V]) Put(key K, value V) { + c.lock.Lock() + defer c.lock.Unlock() + + expiration := c.clock.Now().Add(c.cacheDuration) + + if _, ok := c.items[key]; ok { + c.items[key] = cachedItem[V]{value: value, expiration: expiration} + c.evictionSet.Touch(key) + return + } + + if len(c.items) >= c.maxItems { + delete(c.items, c.evictionSet.Peek()) + c.evictionSet.Remove() + } + + c.items[key] = cachedItem[V]{value: value, expiration: expiration} + c.evictionSet.Insert(key) +}