diff --git a/pbm/config/config.go b/pbm/config/config.go index 8574d385a..13ce1cee0 100644 --- a/pbm/config/config.go +++ b/pbm/config/config.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "maps" - "os" "reflect" "strconv" "strings" @@ -629,12 +628,6 @@ func SetConfig(ctx context.Context, m connect.Client, cfg *Config) error { } sanitizeStoragePaths(&cfg.Storage) - if cfg.Storage.Type == storage.S3 { - // call the function for notification purpose. - // warning about unsupported levels will be printed - s3.SDKLogLevel(cfg.Storage.S3.DebugLogLevels, os.Stderr) - } - if cfg.PITR != nil { if c := string(cfg.PITR.Compression); c != "" && !compress.IsValidCompressionType(c) { return errors.Errorf("unsupported compression type: %q", c) @@ -725,7 +718,9 @@ func SetConfigVar(ctx context.Context, m connect.Client, key, val string) error return errors.New("storage.filesystem.path can't be empty") } case "storage.s3.debugLogLevels": - s3.SDKLogLevel(v.(string), os.Stderr) + if err := s3.ValidateDebugLogLevels(v.(string)); err != nil { + return errors.Wrap(err, "set s3 debug log") + } } _, err = m.ConfigCollection().UpdateOne(ctx, diff --git a/pbm/config/config_test.go b/pbm/config/config_test.go index 3fd8871cd..93f2ddf9c 100644 --- a/pbm/config/config_test.go +++ b/pbm/config/config_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/mongodb" + "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/mongo/options" "go.mongodb.org/mongo-driver/v2/mongo/readpref" @@ -620,6 +621,52 @@ func TestConfig(t *testing.T) { }) } +func TestS3DebugLogLevelValidation(t *testing.T) { + ctx := context.Background() + newConfig := func(levels string) *Config { + return &Config{ + Storage: StorageConf{ + Type: storage.S3, + S3: &s3.Config{ + Bucket: "bucket", + DebugLogLevels: levels, + }, + }, + } + } + + const validLevels = "Signing,Retries" + require.NoError(t, SetConfig(ctx, connClient, newConfig(validLevels))) + require.NoError(t, SetConfigVar(ctx, connClient, "storage.s3.debugLogLevels", "Request,Response")) + + err := SetConfigVar(ctx, connClient, "storage.s3.debugLogLevels", "RequestEventMessage") + require.ErrorContains(t, err, "set s3 debug log") + + err = SetConfig(ctx, connClient, newConfig("LogDebug")) + require.Error(t, err) + + profile := newConfig("Unknown") + profile.Name = "invalid-debug-log-level" + profile.IsProfile = true + err = AddProfile(ctx, connClient, profile) + require.Error(t, err) + + _, err = connClient.ConfigCollection().UpdateOne(ctx, + bson.D{{"profile", nil}}, + bson.M{"$set": bson.M{"storage.s3.debugLogLevels": "Unknown"}}, + ) + require.NoError(t, err) + + persisted, err := GetConfig(ctx, connClient) + require.NoError(t, err) + require.ErrorContains(t, persisted.Storage.Cast(), "validate s3 debug log") + + require.NoError(t, SetConfigVar(ctx, connClient, "storage.s3.debugLogLevels", "Signing")) + got, err := GetConfigVar(ctx, connClient, "storage.s3.debugLogLevels") + require.NoError(t, err) + assert.Equal(t, "Signing", got) +} + func TestRestoreConfGetIndexCommitQuorum(t *testing.T) { tests := []struct { name string diff --git a/pbm/config/profile.go b/pbm/config/profile.go index 4d927205d..b4da9f515 100644 --- a/pbm/config/profile.go +++ b/pbm/config/profile.go @@ -2,7 +2,6 @@ package config import ( "context" - "os" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" @@ -10,8 +9,6 @@ import ( "github.com/percona/percona-backup-mongodb/pbm/connect" "github.com/percona/percona-backup-mongodb/pbm/errors" - "github.com/percona/percona-backup-mongodb/pbm/storage" - "github.com/percona/percona-backup-mongodb/pbm/storage/s3" ) func ListProfiles(ctx context.Context, m connect.Client) ([]Config, error) { @@ -66,12 +63,6 @@ func AddProfile(ctx context.Context, m connect.Client, profile *Config) error { } sanitizeStoragePaths(&profile.Storage) - if profile.Storage.Type == storage.S3 { - // call the function for notification purpose. - // warning about unsupported levels will be printed - s3.SDKLogLevel(profile.Storage.S3.DebugLogLevels, os.Stderr) - } - _, err := m.ConfigCollection().ReplaceOne(ctx, bson.D{ {"profile", true}, diff --git a/pbm/storage/s3/s3.go b/pbm/storage/s3/s3.go index 4d82ed480..2391d9454 100644 --- a/pbm/storage/s3/s3.go +++ b/pbm/storage/s3/s3.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "crypto/tls" "encoding/base64" - "fmt" "io" "maps" "net/http" @@ -61,11 +60,10 @@ type Config struct { // certificate chain and host name InsecureSkipTLSVerify bool `bson:"insecureSkipTLSVerify" json:"insecureSkipTLSVerify" yaml:"insecureSkipTLSVerify"` - // DebugLogLevels enables AWS SDK debug logging (sub)levels. Available options: - // LogDebug, Signing, HTTPBody, RequestRetries, RequestErrors, EventStreamBody - // - // Any sub levels will enable LogDebug level accordingly to AWS SDK Go module behavior - // https://pkg.go.dev/github.com/aws/aws-sdk-go@v1.40.7/aws#LogLevelType + // DebugLogLevels enables AWS SDK v2 debug logging modes. Available options: + // Signing, Retries, Request, RequestWithBody, Response, ResponseWithBody, + // DeprecatedUsage. + // https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/aws#ClientLogMode DebugLogLevels string `bson:"debugLogLevels,omitempty" json:"debugLogLevels,omitempty" yaml:"debugLogLevels,omitempty"` // Retryer is configuration for client.DefaultRetryer @@ -90,21 +88,13 @@ type Retryer struct { type SDKDebugLogLevel string const ( - Signing SDKDebugLogLevel = "Signing" - Retries SDKDebugLogLevel = "Retries" - Request SDKDebugLogLevel = "Request" - RequestWithBody SDKDebugLogLevel = "RequestWithBody" - Response SDKDebugLogLevel = "Response" - ResponseWithBody SDKDebugLogLevel = "ResponseWithBody" - DeprecatedUsage SDKDebugLogLevel = "DeprecatedUsage" - RequestEventMessage SDKDebugLogLevel = "RequestEventMessage" - ResponseEventMessage SDKDebugLogLevel = "ResponseEventMessage" - - LogDebug SDKDebugLogLevel = "LogDebug" - HTTPBody SDKDebugLogLevel = "HTTPBody" - RequestRetries SDKDebugLogLevel = "RequestRetries" - RequestErrors SDKDebugLogLevel = "RequestErrors" - EventStreamBody SDKDebugLogLevel = "EventStreamBody" + Signing SDKDebugLogLevel = "Signing" + Retries SDKDebugLogLevel = "Retries" + Request SDKDebugLogLevel = "Request" + RequestWithBody SDKDebugLogLevel = "RequestWithBody" + Response SDKDebugLogLevel = "Response" + ResponseWithBody SDKDebugLogLevel = "ResponseWithBody" + DeprecatedUsage SDKDebugLogLevel = "DeprecatedUsage" ) type AWSsse struct { @@ -209,6 +199,9 @@ func (cfg *Config) Cast() error { if cfg == nil { return errors.New("missing S3 configuration with S3 storage type") } + if err := ValidateDebugLogLevels(cfg.DebugLogLevels); err != nil { + return errors.Wrap(err, "validate s3 debug log") + } if cfg.Region == "" { cfg.Region = defaultS3Region } @@ -252,38 +245,16 @@ func (cfg *Config) GetMaxObjSizeGB() float64 { return defaultMaxObjSizeGB } -// SDKLogLevel returns AWS SDK log level value from comma-separated -// SDKDebugLogLevel values string. If the string does not contain a valid value, -// returns 0 (logging is disabled). -// -// If the string is incorrect formatted, prints warnings to the io.Writer. -// Passing nil as the io.Writer will discard any warnings. -// -// Deprecated log level values from v1 are supported for backwards -// compatibility and are automatically mapped to their current equivalents. -func SDKLogLevel(levels string, out io.Writer) aws.ClientLogMode { - if out == nil { - out = io.Discard - } - - var logLevel aws.ClientLogMode - +// ValidateDebugLogLevels checks that all configured levels are supported. +func ValidateDebugLogLevels(levels string) error { for _, lvl := range strings.Split(levels, ",") { lvl = strings.TrimSpace(lvl) - if lvl == "" { - continue + if lvl != "" && toClientLogMode(lvl) == 0 { + return errors.Errorf("unsupported S3 client debug log level %q", lvl) } - - l := toClientLogMode(lvl) - if l == 0 { - fmt.Fprintf(out, "Warning: S3 client debug log level: unsupported %q\n", lvl) - continue - } - - logLevel |= l } - return logLevel + return nil } //nolint:lll @@ -718,55 +689,27 @@ func toClientLogMode(levels string) aws.ClientLogMode { for _, item := range items { flag := strings.TrimSpace(item) - switch flag { - case "Signing": - // v1 had "LogDebugWithSigning" + switch SDKDebugLogLevel(flag) { + case Signing: mode |= aws.LogSigning - case "Retries": + case Retries: mode |= aws.LogRetries - case "Request": + case Request: mode |= aws.LogRequest - case "RequestWithBody": + case RequestWithBody: mode |= aws.LogRequestWithBody - case "Response": + case Response: mode |= aws.LogResponse - case "ResponseWithBody": + case ResponseWithBody: mode |= aws.LogResponseWithBody - case "DeprecatedUsage": + case DeprecatedUsage: mode |= aws.LogDeprecatedUsage - - case "RequestEventMessage": - mode |= aws.LogRequestEventMessage - - case "ResponseEventMessage": - mode |= aws.LogResponseEventMessage - - // Mapping deprecated flags from v1 for backwards compatibility - case "LogDebug": - // v1 had "LogDebug" - mode |= aws.LogRequest | aws.LogResponse - - case "HTTPBody": - // v1 had "LogDebugWithHTTPBody" - mode |= aws.LogRequestWithBody | aws.LogResponseWithBody - - case "RequestRetries": - // v1 had "LogDebugWithRequestRetries" - mode |= aws.LogRetries - - case "RequestErrors": - // v1 had "LogDebugWithRequestErrors" - mode |= aws.LogResponse - - case "EventStreamBody": - // v1 had "LogDebugWithEventStreamBody" - mode |= aws.LogRequestWithBody | aws.LogResponseWithBody } } diff --git a/pbm/storage/s3/s3_test.go b/pbm/storage/s3/s3_test.go index 37aa598f5..82583a853 100644 --- a/pbm/storage/s3/s3_test.go +++ b/pbm/storage/s3/s3_test.go @@ -95,13 +95,6 @@ func TestS3(t *testing.T) { storage.RunStorageAPITests(t, stg) storage.RunSplitMergeMWTests(t, stg) }) - - t.Run("default SDKLogLevel for invalid value", func(t *testing.T) { - logLvl := SDKLogLevel("invalid", nil) - if logLvl != 0 { - t.Fatalf("expected SDKLogLevel to be 0, got %v", 0) - } - }) } func TestConfig(t *testing.T) { @@ -182,6 +175,17 @@ func TestConfig(t *testing.T) { } }) + t.Run("Cast rejects unsupported debug log levels", func(t *testing.T) { + for _, levels := range []string{"LogDebug", "RequestEventMessage", "Unknown"} { + t.Run(levels, func(t *testing.T) { + err := (&Config{DebugLogLevels: levels}).Cast() + if err == nil { + t.Fatal("expected error") + } + }) + } + }) + t.Run("GetMaxObjSizeGB", func(t *testing.T) { tests := []struct { name string @@ -247,30 +251,85 @@ func TestToClientLogMode(t *testing.T) { input: "Signing", expected: aws.LogSigning, }, + { + name: "Single flag: Retries", + input: "Retries", + expected: aws.LogRetries, + }, + { + name: "Single flag: Request", + input: "Request", + expected: aws.LogRequest, + }, + { + name: "Single flag: RequestWithBody", + input: "RequestWithBody", + expected: aws.LogRequestWithBody, + }, + { + name: "Single flag: Response", + input: "Response", + expected: aws.LogResponse, + }, + { + name: "Single flag: ResponseWithBody", + input: "ResponseWithBody", + expected: aws.LogResponseWithBody, + }, + { + name: "Single flag: DeprecatedUsage", + input: "DeprecatedUsage", + expected: aws.LogDeprecatedUsage, + }, + { + name: "Unsupported RequestEventMessage", + input: "RequestEventMessage", + expected: 0, + }, + { + name: "Unsupported ResponseEventMessage", + input: "ResponseEventMessage", + expected: 0, + }, { name: "Multiple flags with commas", input: "Retries, Request, Response", expected: aws.LogRetries | aws.LogRequest | aws.LogResponse, }, { - name: "Deprecated LogDebug", + name: "Unsupported LogDebug", input: "LogDebug", - expected: aws.LogRequest | aws.LogResponse, + expected: 0, }, { - name: "Deprecated HTTPBody", + name: "Unsupported HTTPBody", input: "HTTPBody", - expected: aws.LogRequestWithBody | aws.LogResponseWithBody, + expected: 0, + }, + { + name: "Unsupported RequestRetries", + input: "RequestRetries", + expected: 0, + }, + { + name: "Unsupported RequestErrors", + input: "RequestErrors", + expected: 0, + }, + { + name: "Unsupported EventStreamBody", + input: "EventStreamBody", + expected: 0, }, { name: "Flags with extra spaces", input: " Signing , RequestEventMessage,ResponseEventMessage ", - expected: aws.LogSigning | aws.LogRequestEventMessage | aws.LogResponseEventMessage, + expected: aws.LogSigning, }, { - name: "Multiple deprecated flags combined", + name: "Multiple unsupported flags combined", input: "LogDebug, HTTPBody, RequestRetries, RequestErrors, EventStreamBody", - expected: aws.LogRequest | aws.LogResponse | aws.LogRequestWithBody | aws.LogResponseWithBody | aws.LogRetries, + expected: 0, }, } @@ -284,6 +343,51 @@ func TestToClientLogMode(t *testing.T) { } } +func TestValidateDebugLogLevels(t *testing.T) { + tests := []struct { + name string + levels string + wantErr bool + }{ + { + name: "empty", + levels: "", + }, + { + name: "supported", + levels: "Signing,Retries,Request,RequestWithBody,Response,ResponseWithBody,DeprecatedUsage", + }, + { + name: "spaces and empty items", + levels: " Signing, , Retries,", + }, + { + name: "legacy value", + levels: "LogDebug", + wantErr: true, + }, + { + name: "event value", + levels: "RequestEventMessage", + wantErr: true, + }, + { + name: "unknown value after supported value", + levels: "Request,Unknown", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateDebugLogLevels(tt.levels) + if (err != nil) != tt.wantErr { + t.Fatalf("ValidateDebugLogLevels(%q) error = %v, wantErr %v", tt.levels, err, tt.wantErr) + } + }) + } +} + var ( fileSize = flag.Int64("file-size", 500, "file size that will be uploaded") partSize = flag.Int64("part-size", 10, "part size that will be used to upload file")