diff --git a/api/api.go b/api/api.go index 1e82930b1d..4e271c110d 100644 --- a/api/api.go +++ b/api/api.go @@ -663,6 +663,10 @@ func (a *ApplicationHandler) mountControlPlaneRoutes(router chi.Router, handler configRouter.Get("/auth", handler.GetAuthConfiguration) }) + uiRouter.Route("/backups", func(backupRouter chi.Router) { + backupRouter.Post("/trigger", handler.TriggerBackup) + }) + billingHandler := &handlers.BillingHandler{ Handler: handler, BillingClient: a.A.BillingClient, diff --git a/api/handlers/backup.go b/api/handlers/backup.go new file mode 100644 index 0000000000..5eb7cdf13a --- /dev/null +++ b/api/handlers/backup.go @@ -0,0 +1,92 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "time" + + "github.com/go-chi/render" + "github.com/oklog/ulid/v2" + + "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/internal/pkg/exporter" + "github.com/frain-dev/convoy/queue" + "github.com/frain-dev/convoy/util" +) + +type triggerBackupRequest struct { + Start *time.Time `json:"start"` + End *time.Time `json:"end"` +} + +type triggerBackupPayload struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` +} + +// TriggerBackup enqueues an asynchronous manual backup job. +// POST /ui/backups/trigger +func (h *Handler) TriggerBackup(w http.ResponseWriter, r *http.Request) { + if !h.isInstanceAdmin(r) { + _ = render.Render(w, r, util.NewErrorResponse("Unauthorized: instance admin access required", http.StatusForbidden)) + return + } + + cfg, err := config.Get() + if err != nil { + _ = render.Render(w, r, util.NewErrorResponse("failed to load config", http.StatusInternalServerError)) + return + } + + if !cfg.RetentionPolicy.IsRetentionPolicyEnabled { + _ = render.Render(w, r, util.NewErrorResponse("backup is not enabled in configuration", http.StatusUnprocessableEntity)) + return + } + + // Default time window: last backup interval to now + end := time.Now() + start := end.Add(-exporter.ParseBackupInterval(cfg.RetentionPolicy.BackupInterval)) + + // Parse optional overrides from request body (empty body is fine) + var req triggerBackupRequest + if r.ContentLength > 0 { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + _ = render.Render(w, r, util.NewErrorResponse("invalid request body", http.StatusBadRequest)) + return + } + if req.Start != nil { + start = *req.Start + } + if req.End != nil { + end = *req.End + } + } + + if !start.Before(end) { + _ = render.Render(w, r, util.NewErrorResponse("start must be before end", http.StatusBadRequest)) + return + } + + payload, err := json.Marshal(triggerBackupPayload{Start: start, End: end}) + if err != nil { + _ = render.Render(w, r, util.NewErrorResponse("failed to marshal payload", http.StatusInternalServerError)) + return + } + + job := &queue.Job{ + ID: ulid.Make().String(), + Payload: payload, + } + + if err := h.A.Queue.Write(convoy.ManualBackupJob, convoy.DefaultQueue, job); err != nil { + _ = render.Render(w, r, util.NewErrorResponse("failed to enqueue backup job", http.StatusInternalServerError)) + return + } + + _ = render.Render(w, r, util.NewServerResponse("backup job enqueued", map[string]interface{}{ + "job_id": job.ID, + "start": start.Format(time.RFC3339), + "end": end.Format(time.RFC3339), + }, http.StatusAccepted)) +} diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index 429e3de110..a69fe30714 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -15,6 +15,7 @@ import ( "github.com/frain-dev/convoy/auth" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/datastore/cached" "github.com/frain-dev/convoy/internal/organisation_members" "github.com/frain-dev/convoy/internal/organisations" "github.com/frain-dev/convoy/internal/pkg/middleware" @@ -56,7 +57,7 @@ func (h *Handler) retrieveProject(r *http.Request) (*datastore.Project, error) { var project *datastore.Project var err error - projectRepo := projects.New(h.A.Logger, h.A.DB) + projectRepo := cached.NewCachedProjectRepository(projects.New(h.A.Logger, h.A.DB), h.A.Cache, 5*time.Minute, h.A.Logger) switch { case h.IsReqWithJWT(authUser), h.IsReqWithPersonalAccessToken(authUser): @@ -81,29 +82,10 @@ func (h *Handler) retrieveProject(r *http.Request) (*datastore.Project, error) { projectID := apiKey.Role.Project - var p datastore.Project - - cacheKey := convoy.ProjectCacheKey.Get(projectID) - cacheErr := h.A.Cache.Get(r.Context(), cacheKey.String(), &p) - - // If cache hit with valid data, return it - if cacheErr == nil && p.UID != "" { - h.A.Logger.Info("found item in cache") - return &p, nil - } - - // Cache miss - fetch from database - pp, fetchErr := projectRepo.FetchProjectByID(r.Context(), projectID) - if fetchErr != nil { - return nil, fetchErr - } - - cacheErr = h.A.Cache.Set(r.Context(), cacheKey.String(), &pp, time.Hour) - if cacheErr != nil { - h.A.Logger.Error("failed to cache item", "error", cacheErr) + project, err = projectRepo.FetchProjectByID(r.Context(), projectID) + if err != nil { + return nil, err } - - return pp, nil case h.IsReqWithPortalLinkToken(authUser): if len(authUser.Credential.Token) > 0 { // this is the legacy static token type svc := portal_links.New(h.A.Logger, h.A.DB) diff --git a/api/handlers/middleware.go b/api/handlers/middleware.go index e3b9e4c414..0b825d7cad 100644 --- a/api/handlers/middleware.go +++ b/api/handlers/middleware.go @@ -4,12 +4,14 @@ import ( "context" "errors" "net/http" + "time" "github.com/go-chi/chi/v5" "github.com/go-chi/render" "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/datastore/cached" "github.com/frain-dev/convoy/internal/organisations" "github.com/frain-dev/convoy/util" ) @@ -61,7 +63,7 @@ func (h *Handler) RequireEnabledOrganisation() func(next http.Handler) http.Hand if cachedOrg := r.Context().Value(convoy.OrganisationCtx); cachedOrg != nil { org = cachedOrg.(*datastore.Organisation) } else { - orgRepo := organisations.New(h.A.Logger, h.A.DB) + orgRepo := cached.NewCachedOrganisationRepository(organisations.New(h.A.Logger, h.A.DB), h.A.Cache, 5*time.Minute, h.A.Logger) org, err = orgRepo.FetchOrganisationByID(r.Context(), project.OrganisationID) if err != nil { h.A.Logger.Error("Failed to fetch organisation for disabled check", "error", err) diff --git a/api/models/configuration.go b/api/models/configuration.go index a579575dd8..cc53dec36b 100644 --- a/api/models/configuration.go +++ b/api/models/configuration.go @@ -47,7 +47,7 @@ type ConfigurationResponse struct { } type StoragePolicyConfiguration struct { - // Storage policy type e.g on_prem or s3 + // Storage policy type e.g on_prem, s3, or azure_blob Type datastore.StorageType `json:"type,omitempty" valid:"supported_storage~please provide a valid storage type,required"` // S3 Bucket creds @@ -55,6 +55,9 @@ type StoragePolicyConfiguration struct { // On_Prem directory OnPrem *OnPremStorage `json:"on_prem"` + + // Azure Blob Storage creds + AzureBlob *AzureBlobStorage `json:"azure_blob"` } func (sc *StoragePolicyConfiguration) Transform() *datastore.StoragePolicyConfiguration { @@ -63,9 +66,10 @@ func (sc *StoragePolicyConfiguration) Transform() *datastore.StoragePolicyConfig } return &datastore.StoragePolicyConfiguration{ - Type: sc.Type, - S3: sc.S3.transform(), - OnPrem: sc.OnPrem.transform(), + Type: sc.Type, + S3: sc.S3.transform(), + OnPrem: sc.OnPrem.transform(), + AzureBlob: sc.AzureBlob.transform(), } } @@ -119,3 +123,25 @@ func (os *OnPremStorage) transform() *datastore.OnPremStorage { return &datastore.OnPremStorage{Path: os.Path} } + +type AzureBlobStorage struct { + AccountName null.String `json:"account_name"` + AccountKey null.String `json:"account_key,omitempty"` + ContainerName null.String `json:"container_name"` + Endpoint null.String `json:"endpoint,omitempty"` + Prefix null.String `json:"prefix,omitempty"` +} + +func (az *AzureBlobStorage) transform() *datastore.AzureBlobStorage { + if az == nil { + return nil + } + + return &datastore.AzureBlobStorage{ + AccountName: az.AccountName, + AccountKey: az.AccountKey, + ContainerName: az.ContainerName, + Endpoint: az.Endpoint, + Prefix: az.Prefix, + } +} diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index be083c242c..2265a9f1a6 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -2,6 +2,7 @@ package agent import ( "context" + "errors" "fmt" "os" "os/signal" @@ -48,8 +49,8 @@ func AddAgentCommand(a *cli.App) *cobra.Command { return err } - if err := config.Override(cliConfig); err != nil { - return err + if oErr := config.Override(cliConfig); oErr != nil { + return oErr } cfg, err := config.Get() @@ -71,9 +72,9 @@ func AddAgentCommand(a *cli.App) *cobra.Command { case <-quit: cancel() return nil - case err := <-runtimeErr: - if err != nil && err != context.Canceled { - return err + case eRrr := <-runtimeErr: + if eRrr != nil && !errors.Is(eRrr, context.Canceled) { + return eRrr } return nil case <-ctx.Done(): diff --git a/cmd/backup/backup.go b/cmd/backup/backup.go new file mode 100644 index 0000000000..4991182dc3 --- /dev/null +++ b/cmd/backup/backup.go @@ -0,0 +1,108 @@ +package backup + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/internal/configuration" + "github.com/frain-dev/convoy/internal/delivery_attempts" + "github.com/frain-dev/convoy/internal/event_deliveries" + "github.com/frain-dev/convoy/internal/events" + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" + "github.com/frain-dev/convoy/internal/pkg/cli" + "github.com/frain-dev/convoy/internal/pkg/exporter" +) + +func AddBackupCommand(a *cli.App) *cobra.Command { + var startFlag, endFlag string + + cmd := &cobra.Command{ + Use: "backup", + Short: "Run a one-time backup of events, deliveries, and delivery attempts", + Annotations: map[string]string{ + "ShouldBootstrap": "false", + }, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + cfg, err := config.Get() + if err != nil { + return fmt.Errorf("failed to get config: %w", err) + } + + // Default time window: last backup interval to now + end := time.Now() + start := end.Add(-exporter.ParseBackupInterval(cfg.RetentionPolicy.BackupInterval)) + + // Override with flags if provided + if endFlag != "" { + end, err = time.Parse(time.RFC3339, endFlag) + if err != nil { + return fmt.Errorf("invalid --end value (expected RFC3339): %w", err) + } + } + if startFlag != "" { + start, err = time.Parse(time.RFC3339, startFlag) + if err != nil { + return fmt.Errorf("invalid --start value (expected RFC3339): %w", err) + } + } + + if !start.Before(end) { + return fmt.Errorf("--start (%s) must be before --end (%s)", start.Format(time.RFC3339), end.Format(time.RFC3339)) + } + + fmt.Fprintf(os.Stdout, "Backup window: [%s, %s)\n", start.Format(time.RFC3339), end.Format(time.RFC3339)) + + // Create repos + configRepo := configuration.New(a.Logger, a.DB) + eventRepo := events.New(a.Logger, a.DB) + eventDeliveryRepo := event_deliveries.New(a.Logger, a.DB) + attemptsRepo := delivery_attempts.New(a.Logger, a.DB) + + // Load DB config for storage policy + dbConfig, err := configRepo.LoadConfiguration(ctx) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + + // Create blob store + store, err := blobstore.NewBlobStoreClient(dbConfig.StoragePolicy, a.Logger) + if err != nil { + return fmt.Errorf("failed to create blob store: %w", err) + } + + // Create exporter with explicit window + exp, err := exporter.NewExporterWithWindow( + eventRepo, eventDeliveryRepo, dbConfig, attemptsRepo, + start, end, a.Logger, + ) + if err != nil { + return fmt.Errorf("failed to create exporter: %w", err) + } + + // Run export + result, err := exp.StreamExport(ctx, store) + if err != nil { + return fmt.Errorf("backup failed: %w", err) + } + + // Print results + for table, r := range result { + fmt.Fprintf(os.Stdout, "%s: %d records exported → %s\n", table, r.NumDocs, r.ExportFile) + } + + fmt.Fprintln(os.Stdout, "Backup complete.") + return nil + }, + } + + cmd.Flags().StringVar(&startFlag, "start", "", "Export window start (RFC3339, e.g. 2026-04-01T00:00:00Z)") + cmd.Flags().StringVar(&endFlag, "end", "", "Export window end (RFC3339, e.g. 2026-04-02T00:00:00Z)") + return cmd +} diff --git a/cmd/hooks/hooks.go b/cmd/hooks/hooks.go index 355c5d94fd..00f93f6d9d 100644 --- a/cmd/hooks/hooks.go +++ b/cmd/hooks/hooks.go @@ -364,10 +364,19 @@ func ensureInstanceConfig(ctx context.Context, a *cli.App, cfg config.Configurat Path: null.NewString(cfg.StoragePolicy.OnPrem.Path, true), } + azureBlob := datastore.AzureBlobStorage{ + AccountName: null.NewString(cfg.StoragePolicy.AzureBlob.AccountName, true), + AccountKey: null.NewString(cfg.StoragePolicy.AzureBlob.AccountKey, true), + ContainerName: null.NewString(cfg.StoragePolicy.AzureBlob.ContainerName, true), + Endpoint: null.NewString(cfg.StoragePolicy.AzureBlob.Endpoint, true), + Prefix: null.NewString(cfg.StoragePolicy.AzureBlob.Prefix, true), + } + storagePolicy := &datastore.StoragePolicyConfiguration{ - Type: datastore.StorageType(cfg.StoragePolicy.Type), - S3: &s3, - OnPrem: &onPrem, + Type: datastore.StorageType(cfg.StoragePolicy.Type), + S3: &s3, + OnPrem: &onPrem, + AzureBlob: &azureBlob, } retentionPolicy := &datastore.RetentionPolicyConfiguration{ diff --git a/cmd/main.go b/cmd/main.go index 58b863dd37..bb4346999f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -9,6 +9,7 @@ import ( "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/cmd/agent" + "github.com/frain-dev/convoy/cmd/backup" "github.com/frain-dev/convoy/cmd/bootstrap" configCmd "github.com/frain-dev/convoy/cmd/config" "github.com/frain-dev/convoy/cmd/ff" @@ -168,6 +169,7 @@ func main() { c.AddCommand(version.AddVersionCommand()) c.AddCommand(server.AddServerCommand(app)) + c.AddCommand(backup.AddBackupCommand(app)) c.AddCommand(retry.AddRetryCommand(app)) c.AddCommand(migrate.AddMigrateCommand(app)) c.AddCommand(configCmd.AddConfigCommand()) diff --git a/cmd/server/server.go b/cmd/server/server.go index 75ce5183e2..6e3995e6ac 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -14,9 +14,11 @@ import ( "github.com/frain-dev/convoy/auth/realm_chain" "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" + "github.com/frain-dev/convoy/datastore/cached" "github.com/frain-dev/convoy/internal/api_keys" "github.com/frain-dev/convoy/internal/configuration" "github.com/frain-dev/convoy/internal/pkg/cli" + "github.com/frain-dev/convoy/internal/pkg/exporter" "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/keys" "github.com/frain-dev/convoy/internal/pkg/metrics" @@ -116,9 +118,9 @@ func StartConvoyServer(a *cli.App) error { return err } - apiKeyRepo := api_keys.New(a.Logger, a.DB) + apiKeyRepo := cached.NewCachedAPIKeyRepository(api_keys.New(a.Logger, a.DB), a.Cache, 5*time.Minute, a.Logger) userRepo := users.New(a.Logger, a.DB) - portalLinkRepo := portal_links.New(a.Logger, a.DB) + portalLinkRepo := cached.NewCachedPortalLinkRepository(portal_links.New(a.Logger, a.DB), a.Cache, 5*time.Minute, a.Logger) configRepo := configuration.New(a.Logger, a.DB) err = realm_chain.Init(&cfg.Auth, apiKeyRepo, userRepo, portalLinkRepo, a.Cache, a.Logger) if err != nil { @@ -168,30 +170,20 @@ func StartConvoyServer(a *cli.App) error { // register tasks s.RegisterTask("58 23 * * *", convoy.ScheduleQueue, convoy.DeleteArchivedTasksProcessor) - s.RegisterTask("30 * * * *", convoy.ScheduleQueue, convoy.MonitorTwitterSources) - s.RegisterTask("0 * * * *", convoy.ScheduleQueue, convoy.TokenizeSearch) - - // if cfg.Metrics.IsEnabled && cfg.Metrics.Backend == config.PrometheusMetricsProvider { - // refreshInterval := cfg.Metrics.Prometheus.MaterializedViewRefreshInterval - // if refreshInterval < 1 { - // refreshInterval = 1 - // } - // if refreshInterval > 60 { - // refreshInterval = 60 - // } - - //nolint:gocritic - // cronSpec := fmt.Sprintf("*/%d * * * *", refreshInterval) - // s.RegisterTask(cronSpec, convoy.ScheduleQueue, convoy.RefreshMetricsMaterializedViews) - - // lo.Infof("Registered metrics materialized view refresh every %d min", refreshInterval) - // } - - // ensures that project data is backed up about 2 hours before they are deleted + if a.Licenser.RetentionPolicy() { - // runs at 10pm - s.RegisterTask("0 22 * * *", convoy.ScheduleQueue, convoy.BackupProjectData) - // runs at 1am + // Register cron-based backup tasks only when CDC backup is not enabled. + // When CDC is active, the BackupCollector in the worker handles exports continuously. + if !cfg.RetentionPolicy.CDCBackupEnabled { + backupInterval := exporter.ParseBackupInterval(cfg.RetentionPolicy.BackupInterval) + enqueueCron := exporter.DurationToCron(backupInterval) + processCron := exporter.DurationToCronOffset(backupInterval, 1) // +1 min offset so enqueue runs first + + s.RegisterTask(enqueueCron, convoy.ScheduleQueue, convoy.EnqueueBackupJobs) + s.RegisterTask(processCron, convoy.ScheduleQueue, convoy.ProcessBackupJob) + } + + // Retention always runs at 1am s.RegisterTask("0 1 * * *", convoy.ScheduleQueue, convoy.RetentionPolicies) } diff --git a/config/config.go b/config/config.go index b3c18f3eb4..fee93b5d3c 100644 --- a/config/config.go +++ b/config/config.go @@ -74,6 +74,7 @@ var DefaultConfiguration = Configuration{ RetentionPolicy: RetentionPolicyConfiguration{ Policy: "720h", IsRetentionPolicyEnabled: false, + BackupInterval: "1h", }, CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, @@ -375,6 +376,9 @@ type DatadogConfiguration struct { type RetentionPolicyConfiguration struct { Policy string `json:"policy" envconfig:"CONVOY_RETENTION_POLICY"` IsRetentionPolicyEnabled bool `json:"enabled" envconfig:"CONVOY_RETENTION_POLICY_ENABLED"` + BackupInterval string `json:"backup_interval" envconfig:"CONVOY_BACKUP_INTERVAL"` + CDCBackupEnabled bool `json:"cdc_backup_enabled" envconfig:"CONVOY_CDC_BACKUP_ENABLED"` + ReplicationDSN string `json:"replication_dsn" envconfig:"CONVOY_REPLICATION_DSN"` } type CircuitBreakerConfiguration struct { @@ -393,9 +397,10 @@ type AnalyticsConfiguration struct { } type StoragePolicyConfiguration struct { - Type string `json:"type" envconfig:"CONVOY_STORAGE_POLICY_TYPE"` - S3 S3Storage `json:"s3"` - OnPrem OnPremStorage `json:"on_prem"` + Type string `json:"type" envconfig:"CONVOY_STORAGE_POLICY_TYPE"` + S3 S3Storage `json:"s3"` + OnPrem OnPremStorage `json:"on_prem"` + AzureBlob AzureBlobStorage `json:"azure_blob"` } type S3Storage struct { @@ -412,6 +417,14 @@ type OnPremStorage struct { Path string `json:"path" envconfig:"CONVOY_STORAGE_PREM_PATH"` } +type AzureBlobStorage struct { + AccountName string `json:"account_name" envconfig:"CONVOY_STORAGE_AZURE_ACCOUNT_NAME"` + AccountKey string `json:"account_key" envconfig:"CONVOY_STORAGE_AZURE_ACCOUNT_KEY"` + ContainerName string `json:"container_name" envconfig:"CONVOY_STORAGE_AZURE_CONTAINER_NAME"` + Endpoint string `json:"endpoint" envconfig:"CONVOY_STORAGE_AZURE_ENDPOINT"` + Prefix string `json:"prefix" envconfig:"CONVOY_STORAGE_AZURE_PREFIX"` +} + type MetricsConfiguration struct { IsEnabled bool `json:"enabled" envconfig:"CONVOY_METRICS_ENABLED"` Backend MetricsBackend `json:"metrics_backend" envconfig:"CONVOY_METRICS_BACKEND"` diff --git a/config/config_test.go b/config/config_test.go index 1a50f5c73e..ab4feaed87 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -125,6 +125,7 @@ func TestLoadConfig(t *testing.T) { RetentionPolicy: RetentionPolicyConfiguration{ Policy: "720h", IsRetentionPolicyEnabled: true, + BackupInterval: "1h", }, CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, @@ -208,7 +209,7 @@ func TestLoadConfig(t *testing.T) { APIVersion: DefaultAPIVersion, Host: "localhost:5005", ConsumerPoolSize: 100, - RetentionPolicy: RetentionPolicyConfiguration{Policy: "720h"}, + RetentionPolicy: RetentionPolicyConfiguration{Policy: "720h", BackupInterval: "1h"}, Database: DatabaseConfiguration{ Type: PostgresDatabaseProvider, Scheme: "postgres", @@ -307,7 +308,7 @@ func TestLoadConfig(t *testing.T) { wantCfg: Configuration{ APIVersion: DefaultAPIVersion, Host: "localhost:5005", - RetentionPolicy: RetentionPolicyConfiguration{Policy: "720h"}, + RetentionPolicy: RetentionPolicyConfiguration{Policy: "720h", BackupInterval: "1h"}, ConsumerPoolSize: 100, CircuitBreaker: CircuitBreakerConfiguration{ SampleRate: 30, diff --git a/configs/local/conf/.env b/configs/local/conf/.env index 41548e58e6..46e06229a9 100644 --- a/configs/local/conf/.env +++ b/configs/local/conf/.env @@ -12,10 +12,11 @@ PGBOUNCER_AUTH_TYPE=trust PGBOUNCER_USERLIST_FILE=/bitnami/userlists.txt PGBOUNCER_DATABASE=${POSTGRESQL_DATABASE} PGBOUNCER_AUTH_USER=convoy -PGBOUNCER_POOL_MODE=session -PGBOUNCER_MAX_CLIENT_CONN=120 -PGBOUNCER_DEFAULT_POOL_SIZE=20 +PGBOUNCER_POOL_MODE=transaction +PGBOUNCER_MAX_CLIENT_CONN=500 +PGBOUNCER_DEFAULT_POOL_SIZE=80 PGBOUNCER_MAX_DB_CONNECTIONS=250 +PGBOUNCER_MAX_PREPARED_STATEMENTS=100 PGBOUNCER_IGNORE_STARTUP_PARAMETERS=extra_float_digits # host should be your db host address diff --git a/configs/local/docker-compose.yml b/configs/local/docker-compose.yml index c7ea312a52..cc129641b6 100644 --- a/configs/local/docker-compose.yml +++ b/configs/local/docker-compose.yml @@ -61,8 +61,10 @@ services: condition: service_healthy postgres: - image: postgres:15.2-alpine + image: postgres:18-alpine restart: unless-stopped + ports: + - "5433:5432" environment: POSTGRES_DB: convoy POSTGRES_USER: convoy @@ -76,11 +78,17 @@ services: timeout: 5s retries: 5 start_period: 10s + command: + - "postgres" + - "-c" + - "wal_level=logical" pgbouncer: image: bitnamilegacy/pgbouncer hostname: pgbouncer restart: unless-stopped + ports: + - "6432:6432" depends_on: postgres: condition: service_healthy @@ -101,8 +109,37 @@ services: - ./conf/redis/redis-server-cert.pem:/usr/local/etc/redis/certs/redis-server-cert.pem:ro - ./conf/redis/redis-server-key.pem:/usr/local/etc/redis/certs/redis-server-key.pem:ro + minio: + image: minio/minio:RELEASE.2024-01-16T16-07-38Z + restart: unless-stopped + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: convoyadmin + MINIO_ROOT_PASSWORD: convoyadmin + command: server /data --console-address ":9001" + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.31.0 + restart: unless-stopped + ports: + - "10000:10000" + command: ["azurite-blob", "--blobHost", "0.0.0.0", "--blobPort", "10000", "--skipApiVersionCheck"] + volumes: + - azurite_data:/data + volumes: postgres_data: redis_data: caddy_data: + minio_data: + azurite_data: diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go index b4d1333db2..f9594d3673 100644 --- a/database/postgres/postgres.go +++ b/database/postgres/postgres.go @@ -277,7 +277,6 @@ func GetTx(ctx context.Context, db *sqlx.DB) (*sqlx.Tx, bool, error) { return tx, isWrapped, nil } - func closeWithError(closer io.Closer) { err := closer.Close() if err != nil { diff --git a/datastore/cached/repos.go b/datastore/cached/repos.go new file mode 100644 index 0000000000..728a99d5a2 --- /dev/null +++ b/datastore/cached/repos.go @@ -0,0 +1,490 @@ +package cached + +import ( + "context" + "time" + + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/pkg/cachedrepo" + "github.com/frain-dev/convoy/pkg/flatten" +) + +// ============================================================================ +// ProjectRepository +// ============================================================================ + +type CachedProjectRepository struct { + inner datastore.ProjectRepository + cache cachedrepo.Cache + ttl time.Duration + logger cachedrepo.Logger +} + +func NewCachedProjectRepository(inner datastore.ProjectRepository, c cachedrepo.Cache, ttl time.Duration, logger cachedrepo.Logger) *CachedProjectRepository { + return &CachedProjectRepository{inner: inner, cache: c, ttl: ttl, logger: logger} +} + +func (r *CachedProjectRepository) FetchProjectByID(ctx context.Context, id string) (*datastore.Project, error) { + return cachedrepo.FetchOne(ctx, r.cache, r.logger, "projects:"+id, r.ttl, + func(p *datastore.Project) bool { return p.UID != "" }, + func() (*datastore.Project, error) { return r.inner.FetchProjectByID(ctx, id) }) +} + +func (r *CachedProjectRepository) UpdateProject(ctx context.Context, project *datastore.Project) error { + err := r.inner.UpdateProject(ctx, project) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "projects:"+project.UID) + } + return err +} + +func (r *CachedProjectRepository) DeleteProject(ctx context.Context, uid string) error { + err := r.inner.DeleteProject(ctx, uid) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "projects:"+uid) + } + return err +} + +func (r *CachedProjectRepository) LoadProjects(ctx context.Context, f *datastore.ProjectFilter) ([]*datastore.Project, error) { + return r.inner.LoadProjects(ctx, f) +} +func (r *CachedProjectRepository) CreateProject(ctx context.Context, p *datastore.Project) error { + return r.inner.CreateProject(ctx, p) +} +func (r *CachedProjectRepository) CountProjects(ctx context.Context) (int64, error) { + return r.inner.CountProjects(ctx) +} +func (r *CachedProjectRepository) GetProjectsWithEventsInTheInterval(ctx context.Context, interval int) ([]datastore.ProjectEvents, error) { + return r.inner.GetProjectsWithEventsInTheInterval(ctx, interval) +} +func (r *CachedProjectRepository) FillProjectsStatistics(ctx context.Context, project *datastore.Project) error { + return r.inner.FillProjectsStatistics(ctx, project) +} + +// ============================================================================ +// EndpointRepository +// ============================================================================ + +type CachedEndpointRepository struct { + inner datastore.EndpointRepository + cache cachedrepo.Cache + ttl time.Duration + logger cachedrepo.Logger +} + +func NewCachedEndpointRepository(inner datastore.EndpointRepository, c cachedrepo.Cache, ttl time.Duration, logger cachedrepo.Logger) *CachedEndpointRepository { + return &CachedEndpointRepository{inner: inner, cache: c, ttl: ttl, logger: logger} +} + +func (r *CachedEndpointRepository) FindEndpointByID(ctx context.Context, id, projectID string) (*datastore.Endpoint, error) { + return cachedrepo.FetchOne(ctx, r.cache, r.logger, "endpoints:"+projectID+":"+id, r.ttl, + func(e *datastore.Endpoint) bool { return e.UID != "" }, + func() (*datastore.Endpoint, error) { return r.inner.FindEndpointByID(ctx, id, projectID) }) +} + +func (r *CachedEndpointRepository) FindEndpointsByOwnerID(ctx context.Context, projectID, ownerID string) ([]datastore.Endpoint, error) { + return cachedrepo.FetchSlice(ctx, r.cache, r.logger, "endpoints_by_owner:"+projectID+":"+ownerID, r.ttl, + func() ([]datastore.Endpoint, error) { return r.inner.FindEndpointsByOwnerID(ctx, projectID, ownerID) }) +} + +func (r *CachedEndpointRepository) CreateEndpoint(ctx context.Context, endpoint *datastore.Endpoint, projectID string) error { + err := r.inner.CreateEndpoint(ctx, endpoint, projectID) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "endpoints_by_owner:"+projectID+":"+endpoint.OwnerID) + } + return err +} + +func (r *CachedEndpointRepository) UpdateEndpoint(ctx context.Context, endpoint *datastore.Endpoint, projectID string) error { + err := r.inner.UpdateEndpoint(ctx, endpoint, projectID) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "endpoints:"+projectID+":"+endpoint.UID, "endpoints_by_owner:"+projectID+":"+endpoint.OwnerID) + } + return err +} + +func (r *CachedEndpointRepository) UpdateEndpointStatus(ctx context.Context, projectID, endpointID string, status datastore.EndpointStatus) error { + err := r.inner.UpdateEndpointStatus(ctx, projectID, endpointID, status) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "endpoints:"+projectID+":"+endpointID) + } + return err +} + +func (r *CachedEndpointRepository) DeleteEndpoint(ctx context.Context, endpoint *datastore.Endpoint, projectID string) error { + err := r.inner.DeleteEndpoint(ctx, endpoint, projectID) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "endpoints:"+projectID+":"+endpoint.UID, "endpoints_by_owner:"+projectID+":"+endpoint.OwnerID) + } + return err +} + +func (r *CachedEndpointRepository) UpdateSecrets(ctx context.Context, endpointID, projectID string, secrets datastore.Secrets) error { + err := r.inner.UpdateSecrets(ctx, endpointID, projectID, secrets) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "endpoints:"+projectID+":"+endpointID) + } + return err +} + +func (r *CachedEndpointRepository) DeleteSecret(ctx context.Context, endpoint *datastore.Endpoint, secretID, projectID string) error { + err := r.inner.DeleteSecret(ctx, endpoint, secretID, projectID) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "endpoints:"+projectID+":"+endpoint.UID) + } + return err +} + +func (r *CachedEndpointRepository) FindEndpointsByID(ctx context.Context, ids []string, projectID string) ([]datastore.Endpoint, error) { + return r.inner.FindEndpointsByID(ctx, ids, projectID) +} +func (r *CachedEndpointRepository) FindEndpointsByAppID(ctx context.Context, appID, projectID string) ([]datastore.Endpoint, error) { + return r.inner.FindEndpointsByAppID(ctx, appID, projectID) +} +func (r *CachedEndpointRepository) FetchEndpointIDsByOwnerID(ctx context.Context, projectID, ownerID string) ([]string, error) { + return r.inner.FetchEndpointIDsByOwnerID(ctx, projectID, ownerID) +} +func (r *CachedEndpointRepository) FindEndpointByTargetURL(ctx context.Context, projectID, targetURL string) (*datastore.Endpoint, error) { + return r.inner.FindEndpointByTargetURL(ctx, projectID, targetURL) +} +func (r *CachedEndpointRepository) CountProjectEndpoints(ctx context.Context, projectID string) (int64, error) { + return r.inner.CountProjectEndpoints(ctx, projectID) +} +func (r *CachedEndpointRepository) LoadEndpointsPaged(ctx context.Context, projectID string, filter *datastore.Filter, pageable datastore.Pageable) ([]datastore.Endpoint, datastore.PaginationData, error) { + return r.inner.LoadEndpointsPaged(ctx, projectID, filter, pageable) +} + +// ============================================================================ +// SubscriptionRepository +// ============================================================================ + +type CachedSubscriptionRepository struct { + inner datastore.SubscriptionRepository + cache cachedrepo.Cache + ttl time.Duration + logger cachedrepo.Logger +} + +func NewCachedSubscriptionRepository(inner datastore.SubscriptionRepository, c cachedrepo.Cache, ttl time.Duration, logger cachedrepo.Logger) *CachedSubscriptionRepository { + return &CachedSubscriptionRepository{inner: inner, cache: c, ttl: ttl, logger: logger} +} + +func (r *CachedSubscriptionRepository) FindSubscriptionsByEndpointID(ctx context.Context, projectID, endpointID string) ([]datastore.Subscription, error) { + return cachedrepo.FetchSlice(ctx, r.cache, r.logger, "subs_by_endpoint:"+projectID+":"+endpointID, r.ttl, + func() ([]datastore.Subscription, error) { + return r.inner.FindSubscriptionsByEndpointID(ctx, projectID, endpointID) + }) +} + +func (r *CachedSubscriptionRepository) CreateSubscription(ctx context.Context, projectID string, sub *datastore.Subscription) error { + err := r.inner.CreateSubscription(ctx, projectID, sub) + if err == nil && sub.EndpointID != "" { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "subs_by_endpoint:"+projectID+":"+sub.EndpointID) + } + return err +} + +func (r *CachedSubscriptionRepository) UpdateSubscription(ctx context.Context, projectID string, sub *datastore.Subscription) error { + err := r.inner.UpdateSubscription(ctx, projectID, sub) + if err == nil && sub.EndpointID != "" { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "subs_by_endpoint:"+projectID+":"+sub.EndpointID) + } + return err +} + +func (r *CachedSubscriptionRepository) DeleteSubscription(ctx context.Context, projectID string, sub *datastore.Subscription) error { + err := r.inner.DeleteSubscription(ctx, projectID, sub) + if err == nil && sub.EndpointID != "" { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "subs_by_endpoint:"+projectID+":"+sub.EndpointID) + } + return err +} + +func (r *CachedSubscriptionRepository) LoadSubscriptionsPaged(ctx context.Context, projectID string, filter *datastore.FilterBy, pageable datastore.Pageable) ([]datastore.Subscription, datastore.PaginationData, error) { + return r.inner.LoadSubscriptionsPaged(ctx, projectID, filter, pageable) +} +func (r *CachedSubscriptionRepository) FindSubscriptionByID(ctx context.Context, projectID, id string) (*datastore.Subscription, error) { + return r.inner.FindSubscriptionByID(ctx, projectID, id) +} +func (r *CachedSubscriptionRepository) FindSubscriptionsBySourceID(ctx context.Context, projectID, sourceID string) ([]datastore.Subscription, error) { + return r.inner.FindSubscriptionsBySourceID(ctx, projectID, sourceID) +} +func (r *CachedSubscriptionRepository) FindCLISubscriptions(ctx context.Context, projectID string) ([]datastore.Subscription, error) { + return r.inner.FindCLISubscriptions(ctx, projectID) +} +func (r *CachedSubscriptionRepository) CountEndpointSubscriptions(ctx context.Context, a, b, d string) (int64, error) { + return r.inner.CountEndpointSubscriptions(ctx, a, b, d) +} +func (r *CachedSubscriptionRepository) TestSubscriptionFilter(ctx context.Context, payload, filter interface{}, isFlattened bool) (bool, error) { + return r.inner.TestSubscriptionFilter(ctx, payload, filter, isFlattened) +} +func (r *CachedSubscriptionRepository) CompareFlattenedPayload(ctx context.Context, payload, filter flatten.M, isFlattened bool) (bool, error) { + return r.inner.CompareFlattenedPayload(ctx, payload, filter, isFlattened) +} +func (r *CachedSubscriptionRepository) LoadAllSubscriptionConfig(ctx context.Context, projectIDs []string, pageSize int64) ([]datastore.Subscription, error) { + return r.inner.LoadAllSubscriptionConfig(ctx, projectIDs, pageSize) +} +func (r *CachedSubscriptionRepository) FetchDeletedSubscriptions(ctx context.Context, projectIDs []string, updates []datastore.SubscriptionUpdate, pageSize int64) ([]datastore.Subscription, error) { + return r.inner.FetchDeletedSubscriptions(ctx, projectIDs, updates, pageSize) +} +func (r *CachedSubscriptionRepository) FetchUpdatedSubscriptions(ctx context.Context, projectIDs []string, updates []datastore.SubscriptionUpdate, pageSize int64) ([]datastore.Subscription, error) { + return r.inner.FetchUpdatedSubscriptions(ctx, projectIDs, updates, pageSize) +} +func (r *CachedSubscriptionRepository) FetchNewSubscriptions(ctx context.Context, projectIDs, knownIDs []string, lastSyncTime time.Time, pageSize int64) ([]datastore.Subscription, error) { + return r.inner.FetchNewSubscriptions(ctx, projectIDs, knownIDs, lastSyncTime, pageSize) +} + +// ============================================================================ +// FilterRepository +// ============================================================================ + +type CachedFilterRepository struct { + inner datastore.FilterRepository + cache cachedrepo.Cache + ttl time.Duration + logger cachedrepo.Logger +} + +func NewCachedFilterRepository(inner datastore.FilterRepository, c cachedrepo.Cache, ttl time.Duration, logger cachedrepo.Logger) *CachedFilterRepository { + return &CachedFilterRepository{inner: inner, cache: c, ttl: ttl, logger: logger} +} + +func (r *CachedFilterRepository) FindFilterBySubscriptionAndEventType(ctx context.Context, subscriptionID, eventType string) (*datastore.EventTypeFilter, error) { + return cachedrepo.FetchWithNotFound(ctx, r.cache, r.logger, "filters:"+subscriptionID+":"+eventType, r.ttl, + func() (*datastore.EventTypeFilter, error) { + return r.inner.FindFilterBySubscriptionAndEventType(ctx, subscriptionID, eventType) + }, + func(err error) bool { return err.Error() == datastore.ErrFilterNotFound.Error() }, + datastore.ErrFilterNotFound) +} + +func (r *CachedFilterRepository) CreateFilter(ctx context.Context, filter *datastore.EventTypeFilter) error { + err := r.inner.CreateFilter(ctx, filter) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "filters:"+filter.SubscriptionID+":"+filter.EventType, "filters:"+filter.SubscriptionID+":*") + } + return err +} + +func (r *CachedFilterRepository) CreateFilters(ctx context.Context, filters []datastore.EventTypeFilter) error { + err := r.inner.CreateFilters(ctx, filters) + if err == nil { + for i := range filters { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "filters:"+filters[i].SubscriptionID+":"+filters[i].EventType, "filters:"+filters[i].SubscriptionID+":*") + } + } + return err +} + +func (r *CachedFilterRepository) UpdateFilter(ctx context.Context, filter *datastore.EventTypeFilter) error { + err := r.inner.UpdateFilter(ctx, filter) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "filters:"+filter.SubscriptionID+":"+filter.EventType, "filters:"+filter.SubscriptionID+":*") + } + return err +} + +func (r *CachedFilterRepository) UpdateFilters(ctx context.Context, filters []datastore.EventTypeFilter) error { + err := r.inner.UpdateFilters(ctx, filters) + if err == nil { + for i := range filters { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "filters:"+filters[i].SubscriptionID+":"+filters[i].EventType, "filters:"+filters[i].SubscriptionID+":*") + } + } + return err +} + +func (r *CachedFilterRepository) DeleteFilter(ctx context.Context, filterID string) error { + filter, lookupErr := r.inner.FindFilterByID(ctx, filterID) + if lookupErr != nil { + return r.inner.DeleteFilter(ctx, filterID) + } + + err := r.inner.DeleteFilter(ctx, filterID) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, + "filters:"+filter.SubscriptionID+":"+filter.EventType, + "filters:"+filter.SubscriptionID+":*", + ) + } + return err +} +func (r *CachedFilterRepository) FindFilterByID(ctx context.Context, filterID string) (*datastore.EventTypeFilter, error) { + return r.inner.FindFilterByID(ctx, filterID) +} +func (r *CachedFilterRepository) FindFiltersBySubscriptionID(ctx context.Context, subscriptionID string) ([]datastore.EventTypeFilter, error) { + return r.inner.FindFiltersBySubscriptionID(ctx, subscriptionID) +} +func (r *CachedFilterRepository) TestFilter(ctx context.Context, subscriptionID, eventType string, payload interface{}) (bool, error) { + return r.inner.TestFilter(ctx, subscriptionID, eventType, payload) +} + +// ============================================================================ +// APIKeyRepository +// ============================================================================ + +type CachedAPIKeyRepository struct { + inner datastore.APIKeyRepository + cache cachedrepo.Cache + ttl time.Duration + logger cachedrepo.Logger +} + +func NewCachedAPIKeyRepository(inner datastore.APIKeyRepository, c cachedrepo.Cache, ttl time.Duration, logger cachedrepo.Logger) *CachedAPIKeyRepository { + return &CachedAPIKeyRepository{inner: inner, cache: c, ttl: ttl, logger: logger} +} + +func (r *CachedAPIKeyRepository) GetAPIKeyByMaskID(ctx context.Context, maskID string) (*datastore.APIKey, error) { + return cachedrepo.FetchOne(ctx, r.cache, r.logger, "apikeys_by_mask:"+maskID, r.ttl, + func(a *datastore.APIKey) bool { return a.UID != "" }, + func() (*datastore.APIKey, error) { return r.inner.GetAPIKeyByMaskID(ctx, maskID) }) +} + +func (r *CachedAPIKeyRepository) UpdateAPIKey(ctx context.Context, apiKey *datastore.APIKey) error { + err := r.inner.UpdateAPIKey(ctx, apiKey) + if err == nil && apiKey.MaskID != "" { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "apikeys_by_mask:"+apiKey.MaskID) + } + return err +} + +func (r *CachedAPIKeyRepository) CreateAPIKey(ctx context.Context, a *datastore.APIKey) error { + return r.inner.CreateAPIKey(ctx, a) +} +func (r *CachedAPIKeyRepository) RevokeAPIKeys(ctx context.Context, ids []string) error { + return r.inner.RevokeAPIKeys(ctx, ids) +} +func (r *CachedAPIKeyRepository) GetAPIKeyByID(ctx context.Context, id string) (*datastore.APIKey, error) { + return r.inner.GetAPIKeyByID(ctx, id) +} +func (r *CachedAPIKeyRepository) GetAPIKeyByHash(ctx context.Context, hash string) (*datastore.APIKey, error) { + return r.inner.GetAPIKeyByHash(ctx, hash) +} +func (r *CachedAPIKeyRepository) GetAPIKeyByProjectID(ctx context.Context, projectID string) (*datastore.APIKey, error) { + return r.inner.GetAPIKeyByProjectID(ctx, projectID) +} +func (r *CachedAPIKeyRepository) LoadAPIKeysPaged(ctx context.Context, filter *datastore.Filter, pageable *datastore.Pageable) ([]datastore.APIKey, datastore.PaginationData, error) { + return r.inner.LoadAPIKeysPaged(ctx, filter, pageable) +} + +// ============================================================================ +// PortalLinkRepository +// ============================================================================ + +type CachedPortalLinkRepository struct { + inner datastore.PortalLinkRepository + cache cachedrepo.Cache + ttl time.Duration + logger cachedrepo.Logger +} + +func NewCachedPortalLinkRepository(inner datastore.PortalLinkRepository, c cachedrepo.Cache, ttl time.Duration, logger cachedrepo.Logger) *CachedPortalLinkRepository { + return &CachedPortalLinkRepository{inner: inner, cache: c, ttl: ttl, logger: logger} +} + +func (r *CachedPortalLinkRepository) FindPortalLinkByMaskId(ctx context.Context, maskID string) (*datastore.PortalLink, error) { + return cachedrepo.FetchOne(ctx, r.cache, r.logger, "portal_links_by_mask:"+maskID, r.ttl, + func(p *datastore.PortalLink) bool { return p.UID != "" }, + func() (*datastore.PortalLink, error) { return r.inner.FindPortalLinkByMaskId(ctx, maskID) }) +} + +func (r *CachedPortalLinkRepository) UpdatePortalLink(ctx context.Context, projectID string, portalLink *datastore.PortalLink, request *datastore.UpdatePortalLinkRequest) (*datastore.PortalLink, error) { + result, err := r.inner.UpdatePortalLink(ctx, projectID, portalLink, request) + if err == nil && portalLink.TokenMaskId != "" { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "portal_links_by_mask:"+portalLink.TokenMaskId) + } + return result, err +} + +func (r *CachedPortalLinkRepository) CreatePortalLink(ctx context.Context, projectID string, req *datastore.CreatePortalLinkRequest) (*datastore.PortalLink, error) { + return r.inner.CreatePortalLink(ctx, projectID, req) +} +func (r *CachedPortalLinkRepository) GetPortalLink(ctx context.Context, projectID, portalLinkID string) (*datastore.PortalLink, error) { + return r.inner.GetPortalLink(ctx, projectID, portalLinkID) +} +func (r *CachedPortalLinkRepository) GetPortalLinkByToken(ctx context.Context, token string) (*datastore.PortalLink, error) { + return r.inner.GetPortalLinkByToken(ctx, token) +} +func (r *CachedPortalLinkRepository) GetPortalLinkByOwnerID(ctx context.Context, projectID, ownerID string) (*datastore.PortalLink, error) { + return r.inner.GetPortalLinkByOwnerID(ctx, projectID, ownerID) +} +func (r *CachedPortalLinkRepository) RefreshPortalLinkAuthToken(ctx context.Context, projectID, portalLinkID string) (*datastore.PortalLink, error) { + return r.inner.RefreshPortalLinkAuthToken(ctx, projectID, portalLinkID) +} +func (r *CachedPortalLinkRepository) RevokePortalLink(ctx context.Context, projectID, portalLinkID string) error { + return r.inner.RevokePortalLink(ctx, projectID, portalLinkID) +} +func (r *CachedPortalLinkRepository) LoadPortalLinksPaged(ctx context.Context, projectID string, filter *datastore.FilterBy, pageable datastore.Pageable) ([]datastore.PortalLink, datastore.PaginationData, error) { + return r.inner.LoadPortalLinksPaged(ctx, projectID, filter, pageable) +} +func (r *CachedPortalLinkRepository) FindPortalLinksByOwnerID(ctx context.Context, ownerID string) ([]datastore.PortalLink, error) { + return r.inner.FindPortalLinksByOwnerID(ctx, ownerID) +} + +// ============================================================================ +// OrganisationRepository +// ============================================================================ + +type CachedOrganisationRepository struct { + inner datastore.OrganisationRepository + cache cachedrepo.Cache + ttl time.Duration + logger cachedrepo.Logger +} + +func NewCachedOrganisationRepository(inner datastore.OrganisationRepository, c cachedrepo.Cache, ttl time.Duration, logger cachedrepo.Logger) *CachedOrganisationRepository { + return &CachedOrganisationRepository{inner: inner, cache: c, ttl: ttl, logger: logger} +} + +func (r *CachedOrganisationRepository) FetchOrganisationByID(ctx context.Context, id string) (*datastore.Organisation, error) { + return cachedrepo.FetchOne(ctx, r.cache, r.logger, "organisations:"+id, r.ttl, + func(o *datastore.Organisation) bool { return o.UID != "" }, + func() (*datastore.Organisation, error) { return r.inner.FetchOrganisationByID(ctx, id) }) +} + +func (r *CachedOrganisationRepository) UpdateOrganisation(ctx context.Context, org *datastore.Organisation) error { + err := r.inner.UpdateOrganisation(ctx, org) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "organisations:"+org.UID) + } + return err +} + +func (r *CachedOrganisationRepository) UpdateOrganisationLicenseData(ctx context.Context, orgID, licenseData string) error { + err := r.inner.UpdateOrganisationLicenseData(ctx, orgID, licenseData) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "organisations:"+orgID) + } + return err +} + +func (r *CachedOrganisationRepository) DeleteOrganisation(ctx context.Context, id string) error { + err := r.inner.DeleteOrganisation(ctx, id) + if err == nil { + cachedrepo.Invalidate(ctx, r.cache, r.logger, "organisations:"+id) + } + return err +} + +func (r *CachedOrganisationRepository) CreateOrganisation(ctx context.Context, org *datastore.Organisation) error { + return r.inner.CreateOrganisation(ctx, org) +} +func (r *CachedOrganisationRepository) FetchOrganisationByCustomDomain(ctx context.Context, domain string) (*datastore.Organisation, error) { + return r.inner.FetchOrganisationByCustomDomain(ctx, domain) +} +func (r *CachedOrganisationRepository) FetchOrganisationByAssignedDomain(ctx context.Context, domain string) (*datastore.Organisation, error) { + return r.inner.FetchOrganisationByAssignedDomain(ctx, domain) +} +func (r *CachedOrganisationRepository) LoadOrganisationsPaged(ctx context.Context, pageable datastore.Pageable) ([]datastore.Organisation, datastore.PaginationData, error) { + return r.inner.LoadOrganisationsPaged(ctx, pageable) +} +func (r *CachedOrganisationRepository) LoadOrganisationsPagedWithSearch(ctx context.Context, pageable datastore.Pageable, search string) ([]datastore.Organisation, datastore.PaginationData, error) { + return r.inner.LoadOrganisationsPagedWithSearch(ctx, pageable, search) +} +func (r *CachedOrganisationRepository) CountOrganisations(ctx context.Context) (int64, error) { + return r.inner.CountOrganisations(ctx) +} +func (r *CachedOrganisationRepository) CalculateUsage(ctx context.Context, orgID string, startTime, endTime time.Time) (*datastore.OrganisationUsage, error) { + return r.inner.CalculateUsage(ctx, orgID, startTime, endTime) +} diff --git a/datastore/cached/repos_test.go b/datastore/cached/repos_test.go new file mode 100644 index 0000000000..2ca92864bf --- /dev/null +++ b/datastore/cached/repos_test.go @@ -0,0 +1,424 @@ +package cached + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/mocks" +) + +// ============================================================================ +// ProjectRepository Tests +// ============================================================================ + +func TestCachedProjectRepo_FetchProjectByID_CacheHit(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockProjectRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + project := &datastore.Project{UID: "proj-123", Name: "test"} + mockCache.EXPECT().Get(gomock.Any(), "projects:proj-123", gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, data interface{}) error { + *data.(*datastore.Project) = *project + return nil + }) + + repo := NewCachedProjectRepository(mockRepo, mockCache, 5*time.Minute, logger) + result, err := repo.FetchProjectByID(context.Background(), "proj-123") + require.NoError(t, err) + require.Equal(t, "proj-123", result.UID) +} + +func TestCachedProjectRepo_FetchProjectByID_CacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockProjectRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + project := &datastore.Project{UID: "proj-123"} + mockCache.EXPECT().Get(gomock.Any(), "projects:proj-123", gomock.Any()).Return(nil) + mockRepo.EXPECT().FetchProjectByID(gomock.Any(), "proj-123").Return(project, nil) + mockCache.EXPECT().Set(gomock.Any(), "projects:proj-123", project, 5*time.Minute).Return(nil) + + repo := NewCachedProjectRepository(mockRepo, mockCache, 5*time.Minute, logger) + result, err := repo.FetchProjectByID(context.Background(), "proj-123") + require.NoError(t, err) + require.Equal(t, "proj-123", result.UID) +} + +func TestCachedProjectRepo_UpdateProject_Invalidates(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockProjectRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + project := &datastore.Project{UID: "proj-123"} + mockRepo.EXPECT().UpdateProject(gomock.Any(), project).Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "projects:proj-123").Return(nil) + + repo := NewCachedProjectRepository(mockRepo, mockCache, 5*time.Minute, logger) + require.NoError(t, repo.UpdateProject(context.Background(), project)) +} + +func TestCachedProjectRepo_DeleteProject_Invalidates(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockProjectRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + mockRepo.EXPECT().DeleteProject(gomock.Any(), "proj-123").Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "projects:proj-123").Return(nil) + + repo := NewCachedProjectRepository(mockRepo, mockCache, 5*time.Minute, logger) + require.NoError(t, repo.DeleteProject(context.Background(), "proj-123")) +} + +// ============================================================================ +// EndpointRepository Tests +// ============================================================================ + +func TestCachedEndpointRepo_FindEndpointByID_CacheHit(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockEndpointRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + ep := &datastore.Endpoint{UID: "ep-123", Url: "https://example.com"} + mockCache.EXPECT().Get(gomock.Any(), "endpoints:proj-1:ep-123", gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, data interface{}) error { + *data.(*datastore.Endpoint) = *ep + return nil + }) + + repo := NewCachedEndpointRepository(mockRepo, mockCache, 2*time.Minute, logger) + result, err := repo.FindEndpointByID(context.Background(), "ep-123", "proj-1") + require.NoError(t, err) + require.Equal(t, "ep-123", result.UID) +} + +func TestCachedEndpointRepo_FindEndpointByID_CacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockEndpointRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + ep := &datastore.Endpoint{UID: "ep-123"} + mockCache.EXPECT().Get(gomock.Any(), "endpoints:proj-1:ep-123", gomock.Any()).Return(nil) + mockRepo.EXPECT().FindEndpointByID(gomock.Any(), "ep-123", "proj-1").Return(ep, nil) + mockCache.EXPECT().Set(gomock.Any(), "endpoints:proj-1:ep-123", ep, 2*time.Minute).Return(nil) + + repo := NewCachedEndpointRepository(mockRepo, mockCache, 2*time.Minute, logger) + result, err := repo.FindEndpointByID(context.Background(), "ep-123", "proj-1") + require.NoError(t, err) + require.Equal(t, "ep-123", result.UID) +} + +func TestCachedEndpointRepo_UpdateEndpoint_Invalidates(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockEndpointRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + ep := &datastore.Endpoint{UID: "ep-123", OwnerID: "owner-1"} + mockRepo.EXPECT().UpdateEndpoint(gomock.Any(), ep, "proj-1").Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "endpoints:proj-1:ep-123").Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "endpoints_by_owner:proj-1:owner-1").Return(nil) + + repo := NewCachedEndpointRepository(mockRepo, mockCache, 2*time.Minute, logger) + require.NoError(t, repo.UpdateEndpoint(context.Background(), ep, "proj-1")) +} + +func TestCachedEndpointRepo_CreateEndpoint_InvalidatesOwner(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockEndpointRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + ep := &datastore.Endpoint{UID: "ep-new", OwnerID: "owner-1"} + mockRepo.EXPECT().CreateEndpoint(gomock.Any(), ep, "proj-1").Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "endpoints_by_owner:proj-1:owner-1").Return(nil) + + repo := NewCachedEndpointRepository(mockRepo, mockCache, 2*time.Minute, logger) + require.NoError(t, repo.CreateEndpoint(context.Background(), ep, "proj-1")) +} + +// ============================================================================ +// SubscriptionRepository Tests +// ============================================================================ + +func TestCachedSubRepo_FindByEndpointID_CacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockSubscriptionRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + subs := []datastore.Subscription{{UID: "sub-1"}} + mockCache.EXPECT().Get(gomock.Any(), "subs_by_endpoint:proj-1:ep-1", gomock.Any()).Return(nil) + mockRepo.EXPECT().FindSubscriptionsByEndpointID(gomock.Any(), "proj-1", "ep-1").Return(subs, nil) + mockCache.EXPECT().Set(gomock.Any(), "subs_by_endpoint:proj-1:ep-1", gomock.Any(), 30*time.Second).Return(nil) + + repo := NewCachedSubscriptionRepository(mockRepo, mockCache, 30*time.Second, logger) + result, err := repo.FindSubscriptionsByEndpointID(context.Background(), "proj-1", "ep-1") + require.NoError(t, err) + require.Len(t, result, 1) +} + +func TestCachedSubRepo_CreateSubscription_Invalidates(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockSubscriptionRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + sub := &datastore.Subscription{UID: "sub-1", EndpointID: "ep-1"} + mockRepo.EXPECT().CreateSubscription(gomock.Any(), "proj-1", sub).Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "subs_by_endpoint:proj-1:ep-1").Return(nil) + + repo := NewCachedSubscriptionRepository(mockRepo, mockCache, 30*time.Second, logger) + require.NoError(t, repo.CreateSubscription(context.Background(), "proj-1", sub)) +} + +func TestCachedSubRepo_EmptyEndpointID_SkipsInvalidation(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockSubscriptionRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + + sub := &datastore.Subscription{UID: "sub-1", EndpointID: ""} + mockRepo.EXPECT().CreateSubscription(gomock.Any(), "proj-1", sub).Return(nil) + // No Delete expected + + repo := NewCachedSubscriptionRepository(mockRepo, mockCache, 30*time.Second, logger) + require.NoError(t, repo.CreateSubscription(context.Background(), "proj-1", sub)) +} + +// ============================================================================ +// FilterRepository Tests +// ============================================================================ + +func TestCachedFilterRepo_FindBySubAndEventType_CacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockFilterRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + filter := &datastore.EventTypeFilter{UID: "f-1", SubscriptionID: "sub-1", EventType: "user.created"} + mockCache.EXPECT().Get(gomock.Any(), "filters:sub-1:user.created", gomock.Any()).Return(nil) + mockRepo.EXPECT().FindFilterBySubscriptionAndEventType(gomock.Any(), "sub-1", "user.created").Return(filter, nil) + mockCache.EXPECT().Set(gomock.Any(), "filters:sub-1:user.created", gomock.Any(), 2*time.Minute).Return(nil) + + repo := NewCachedFilterRepository(mockRepo, mockCache, 2*time.Minute, logger) + result, err := repo.FindFilterBySubscriptionAndEventType(context.Background(), "sub-1", "user.created") + require.NoError(t, err) + require.Equal(t, "f-1", result.UID) +} + +func TestCachedFilterRepo_CachesNotFound(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockFilterRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + mockCache.EXPECT().Get(gomock.Any(), "filters:sub-1:*", gomock.Any()).Return(nil) + mockRepo.EXPECT().FindFilterBySubscriptionAndEventType(gomock.Any(), "sub-1", "*"). + Return(nil, datastore.ErrFilterNotFound) + mockCache.EXPECT().Set(gomock.Any(), "filters:sub-1:*", gomock.Any(), 2*time.Minute).Return(nil) + + repo := NewCachedFilterRepository(mockRepo, mockCache, 2*time.Minute, logger) + result, err := repo.FindFilterBySubscriptionAndEventType(context.Background(), "sub-1", "*") + require.Error(t, err) + require.Nil(t, result) +} + +func TestCachedFilterRepo_CreateFilter_Invalidates(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockFilterRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + filter := &datastore.EventTypeFilter{UID: "f-1", SubscriptionID: "sub-1", EventType: "user.created"} + mockRepo.EXPECT().CreateFilter(gomock.Any(), filter).Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "filters:sub-1:user.created").Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "filters:sub-1:*").Return(nil) + + repo := NewCachedFilterRepository(mockRepo, mockCache, 2*time.Minute, logger) + require.NoError(t, repo.CreateFilter(context.Background(), filter)) +} + +// ============================================================================ +// APIKeyRepository Tests +// ============================================================================ + +func TestCachedAPIKeyRepo_GetByMaskID_CacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockAPIKeyRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + apiKey := &datastore.APIKey{UID: "ak-123", MaskID: "mask-1"} + mockCache.EXPECT().Get(gomock.Any(), "apikeys_by_mask:mask-1", gomock.Any()).Return(nil) + mockRepo.EXPECT().GetAPIKeyByMaskID(gomock.Any(), "mask-1").Return(apiKey, nil) + mockCache.EXPECT().Set(gomock.Any(), "apikeys_by_mask:mask-1", apiKey, 5*time.Minute).Return(nil) + + repo := NewCachedAPIKeyRepository(mockRepo, mockCache, 5*time.Minute, logger) + result, err := repo.GetAPIKeyByMaskID(context.Background(), "mask-1") + require.NoError(t, err) + require.Equal(t, "ak-123", result.UID) +} + +// ============================================================================ +// PortalLinkRepository Tests +// ============================================================================ + +func TestCachedPortalLinkRepo_FindByMaskId_CacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockPortalLinkRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + pLink := &datastore.PortalLink{UID: "pl-123", TokenMaskId: "mask-1"} + mockCache.EXPECT().Get(gomock.Any(), "portal_links_by_mask:mask-1", gomock.Any()).Return(nil) + mockRepo.EXPECT().FindPortalLinkByMaskId(gomock.Any(), "mask-1").Return(pLink, nil) + mockCache.EXPECT().Set(gomock.Any(), "portal_links_by_mask:mask-1", pLink, 5*time.Minute).Return(nil) + + repo := NewCachedPortalLinkRepository(mockRepo, mockCache, 5*time.Minute, logger) + result, err := repo.FindPortalLinkByMaskId(context.Background(), "mask-1") + require.NoError(t, err) + require.Equal(t, "pl-123", result.UID) +} + +// ============================================================================ +// OrganisationRepository Tests +// ============================================================================ + +func TestCachedOrgRepo_FetchByID_CacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockOrganisationRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + org := &datastore.Organisation{UID: "org-123"} + mockCache.EXPECT().Get(gomock.Any(), "organisations:org-123", gomock.Any()).Return(nil) + mockRepo.EXPECT().FetchOrganisationByID(gomock.Any(), "org-123").Return(org, nil) + mockCache.EXPECT().Set(gomock.Any(), "organisations:org-123", org, 5*time.Minute).Return(nil) + + repo := NewCachedOrganisationRepository(mockRepo, mockCache, 5*time.Minute, logger) + result, err := repo.FetchOrganisationByID(context.Background(), "org-123") + require.NoError(t, err) + require.Equal(t, "org-123", result.UID) +} + +func TestCachedOrgRepo_UpdateOrg_Invalidates(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockOrganisationRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + org := &datastore.Organisation{UID: "org-123"} + mockRepo.EXPECT().UpdateOrganisation(gomock.Any(), org).Return(nil) + mockCache.EXPECT().Delete(gomock.Any(), "organisations:org-123").Return(nil) + + repo := NewCachedOrganisationRepository(mockRepo, mockCache, 5*time.Minute, logger) + require.NoError(t, repo.UpdateOrganisation(context.Background(), org)) +} + +// ============================================================================ +// Cache Error Graceful Degradation +// ============================================================================ + +func TestCachedRepo_CacheError_FallsThrough(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockProjectRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + project := &datastore.Project{UID: "proj-123"} + mockCache.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("redis down")) + mockRepo.EXPECT().FetchProjectByID(gomock.Any(), "proj-123").Return(project, nil) + mockCache.EXPECT().Set(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + + repo := NewCachedProjectRepository(mockRepo, mockCache, 5*time.Minute, logger) + result, err := repo.FetchProjectByID(context.Background(), "proj-123") + require.NoError(t, err) + require.Equal(t, "proj-123", result.UID) +} + +func TestCachedRepo_DBError_NotCached(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRepo := mocks.NewMockProjectRepository(ctrl) + mockCache := mocks.NewMockCache(ctrl) + logger := mocks.NewMockLogger(ctrl) + logger.EXPECT().Error(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + mockCache.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + mockRepo.EXPECT().FetchProjectByID(gomock.Any(), "proj-123").Return(nil, errors.New("db error")) + + repo := NewCachedProjectRepository(mockRepo, mockCache, 5*time.Minute, logger) + result, err := repo.FetchProjectByID(context.Background(), "proj-123") + require.Error(t, err) + require.Nil(t, result) +} diff --git a/datastore/models.go b/datastore/models.go index 5d2f8acebe..c9ee7202c9 100644 --- a/datastore/models.go +++ b/datastore/models.go @@ -246,8 +246,9 @@ const ( ) const ( - S3 StorageType = "s3" - OnPrem StorageType = "on_prem" + S3 StorageType = "s3" + OnPrem StorageType = "on_prem" + AzureBlob StorageType = "azure_blob" ) const ( @@ -1651,9 +1652,10 @@ func (c *Configuration) GetRetentionPolicyConfig() RetentionPolicyConfiguration } type StoragePolicyConfiguration struct { - Type StorageType `json:"type,omitempty" db:"type" valid:"supported_storage~please provide a valid storage type,required"` - S3 *S3Storage `json:"s3" db:"s3"` - OnPrem *OnPremStorage `json:"on_prem" db:"on_prem"` + Type StorageType `json:"type,omitempty" db:"type" valid:"supported_storage~please provide a valid storage type,required"` + S3 *S3Storage `json:"s3" db:"s3"` + OnPrem *OnPremStorage `json:"on_prem" db:"on_prem"` + AzureBlob *AzureBlobStorage `json:"azure_blob" db:"azure_blob"` } type S3Storage struct { @@ -1670,6 +1672,28 @@ type OnPremStorage struct { Path null.String `json:"path" db:"path"` } +type AzureBlobStorage struct { + AccountName null.String `json:"account_name" db:"account_name"` + AccountKey null.String `json:"account_key,omitempty" db:"account_key"` + ContainerName null.String `json:"container_name" db:"container_name"` + Endpoint null.String `json:"endpoint,omitempty" db:"endpoint"` + Prefix null.String `json:"prefix,omitempty" db:"prefix"` +} + +type BackupJob struct { + ID string `json:"id" db:"id"` + HourStart time.Time `json:"hour_start" db:"hour_start"` + HourEnd time.Time `json:"hour_end" db:"hour_end"` + Status string `json:"status" db:"status"` + WorkerID string `json:"worker_id" db:"worker_id"` + ClaimedAt *time.Time `json:"claimed_at" db:"claimed_at"` + CompletedAt *time.Time `json:"completed_at" db:"completed_at"` + Error string `json:"error" db:"error"` + RecordCounts map[string]int64 `json:"record_counts" db:"record_counts"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + type OrganisationMember struct { UID string `json:"uid" db:"id"` OrganisationID string `json:"organisation_id" db:"organisation_id"` diff --git a/datastore/repository.go b/datastore/repository.go index 75f5ea70cf..8615864884 100644 --- a/datastore/repository.go +++ b/datastore/repository.go @@ -284,7 +284,7 @@ type MetaEventRepository interface { } type ExportRepository interface { - ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) + ExportRecords(ctx context.Context, start, end time.Time, w io.Writer) (int64, error) } type DeliveryAttemptsRepository interface { @@ -298,6 +298,17 @@ type DeliveryAttemptsRepository interface { UnPartitionDeliveryAttemptsTable(ctx context.Context) error } +type BackupJobRepository interface { + EnqueueBackupJob(ctx context.Context, hourStart, hourEnd time.Time) error + EnqueueBackupJobIfIdle(ctx context.Context, start, end time.Time) error + ClaimBackupJob(ctx context.Context, workerID string) (*BackupJob, error) + CompleteBackupJob(ctx context.Context, jobID string, recordCounts map[string]int64) error + FailBackupJob(ctx context.Context, jobID string, errMsg string) error + ReclaimStaleJobs(ctx context.Context, staleMinutes int32) (int64, error) + DeleteCompletedJobs(ctx context.Context) (int64, error) + FindLatestCompletedBackup(ctx context.Context) (*BackupJob, error) +} + type EventTypesRepository interface { CreateEventType(context.Context, *ProjectEventType) error UpdateEventType(context.Context, *ProjectEventType) error diff --git a/e2e/backup/backup_modes_test.go b/e2e/backup/backup_modes_test.go new file mode 100644 index 0000000000..0e7cbde741 --- /dev/null +++ b/e2e/backup/backup_modes_test.go @@ -0,0 +1,434 @@ +package backup + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/oklog/ulid/v2" + "github.com/stretchr/testify/require" + + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/internal/configuration" + "github.com/frain-dev/convoy/internal/delivery_attempts" + "github.com/frain-dev/convoy/internal/endpoints" + "github.com/frain-dev/convoy/internal/event_deliveries" + "github.com/frain-dev/convoy/internal/events" + "github.com/frain-dev/convoy/internal/pkg/backup_collector" + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" + "github.com/frain-dev/convoy/internal/pkg/exporter" + log "github.com/frain-dev/convoy/pkg/logger" +) + +// seedTestData seeds events, deliveries, and attempts for backup testing. +// Returns the counts seeded. +func seedTestData(t *testing.T, env *E2ETestEnv, n int) (int, int, int) { + t.Helper() + ctx := context.Background() + db := env.App.DB + project := env.Project + logger := log.New("convoy", log.LevelInfo) + + endpoint := &datastore.Endpoint{ + UID: ulid.Make().String(), + ProjectID: project.UID, + OwnerID: project.UID, + Url: "https://example.com/webhook", + Name: "Backup Test Endpoint", + Secrets: []datastore.Secret{{UID: ulid.Make().String(), Value: "test-secret"}}, + Status: datastore.ActiveEndpointStatus, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Authentication: &datastore.EndpointAuthentication{}, + } + endpointRepo := endpoints.New(logger, db) + err := endpointRepo.CreateEndpoint(ctx, endpoint, project.UID) + require.NoError(t, err) + + // Seed recent data (within the backup interval window) + evtCount, dlvCount, attCount := 0, 0, 0 + for range n { + evt := seedOldEvent(t, db, ctx, project, endpoint, 0) // current time + sub := seedSubscription(t, db, ctx, project, endpoint) + dlv := seedOldEventDelivery(t, db, ctx, evt, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, dlv, endpoint, 0) + evtCount++ + dlvCount++ + attCount++ + _ = sub + } + + return evtCount, dlvCount, attCount +} + +// ============================================================================ +// CDC Mode Tests +// ============================================================================ + +func TestBackup_CDC_OnPrem(t *testing.T) { + env := SetupE2EWithoutWorker(t) + ctx := context.Background() + tmpDir := t.TempDir() + logger := log.New("convoy", log.LevelInfo) + + // Configure on-prem storage + createOnPremConfig(t, env.App.DB, ctx, tmpDir) + + // Seed data + evtCount, _, _ := seedTestData(t, env, 3) + require.Equal(t, 3, evtCount) + + // Build replication DSN from the pool config + pool := env.App.DB.GetConn() + cfg := pool.Config().ConnConfig + replDSN := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", + cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database) + + // Create publication + _, err := pool.Exec(ctx, ` + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'convoy_backup') THEN + CREATE PUBLICATION convoy_backup FOR TABLE convoy.events, convoy.event_deliveries, convoy.delivery_attempts; + END IF; + END $$; + `) + require.NoError(t, err) + + // Start collector with short flush interval + store, err := blobstore.NewOnPremClient(blobstore.BlobStoreOptions{OnPremStorageDir: tmpDir}, logger) + require.NoError(t, err) + + collector := backup_collector.NewBackupCollector(pool, replDSN, store, 3*time.Second, logger) + err = collector.Start(ctx) + require.NoError(t, err) + + // Insert more events AFTER collector starts (CDC captures new INSERTs) + db := env.App.DB + endpoint := &datastore.Endpoint{ + UID: ulid.Make().String(), + ProjectID: env.Project.UID, + OwnerID: env.Project.UID, + Url: "https://example.com/cdc-test", + Name: "CDC Test Endpoint", + Secrets: []datastore.Secret{{UID: ulid.Make().String(), Value: "s"}}, + Status: datastore.ActiveEndpointStatus, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Authentication: &datastore.EndpointAuthentication{}, + } + endpointRepo := endpoints.New(logger, db) + err = endpointRepo.CreateEndpoint(ctx, endpoint, env.Project.UID) + require.NoError(t, err) + + for range 5 { + seedOldEvent(t, db, ctx, env.Project, endpoint, 0) // brand new events + } + + // Wait for flush + time.Sleep(5 * time.Second) + + collector.Stop(ctx) + defer func() { _, _ = pool.Exec(ctx, "SELECT pg_drop_replication_slot('convoy_backup')") }() + + // Verify files and record counts + files := findExportFiles(t, tmpDir, "events") + require.NotEmpty(t, files, "should have CDC events backup files") + + var totalRecords int + for _, f := range files { + data := readExportFile(t, f) + records := parseJSONL(t, data) + totalRecords += len(records) + } + require.GreaterOrEqual(t, totalRecords, 5, "should have at least 5 CDC-captured events") +} + +func TestBackup_CDC_S3(t *testing.T) { + if infra.NewMinIOClient == nil { + t.Skip("MinIO not available") + } + + env := SetupE2EWithoutWorker(t) + ctx := context.Background() + logger := log.New("convoy", log.LevelInfo) + + minioClient, minioEndpoint, err := (*infra.NewMinIOClient)(t) + require.NoError(t, err) + bucket := createTestMinioBucket(t, minioClient) + + pool := env.App.DB.GetConn() + cfg := pool.Config().ConnConfig + replDSN := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", + cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database) + + _, err = pool.Exec(ctx, ` + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'convoy_backup') THEN + CREATE PUBLICATION convoy_backup FOR TABLE convoy.events, convoy.event_deliveries, convoy.delivery_attempts; + END IF; + END $$; + `) + require.NoError(t, err) + + store, err := blobstore.NewS3Client(blobstore.BlobStoreOptions{ + Bucket: bucket, + AccessKey: "minioadmin", + SecretKey: "minioadmin", + Region: "us-east-1", + Endpoint: "http://" + minioEndpoint, + }, logger) + require.NoError(t, err) + + collector := backup_collector.NewBackupCollector(pool, replDSN, store, 3*time.Second, logger) + err = collector.Start(ctx) + require.NoError(t, err) + + // Insert events after collector starts + db := env.App.DB + endpoint := &datastore.Endpoint{ + UID: ulid.Make().String(), ProjectID: env.Project.UID, OwnerID: env.Project.UID, + Url: "https://example.com/s3-cdc", Name: "S3 CDC", Status: datastore.ActiveEndpointStatus, + Secrets: []datastore.Secret{{UID: ulid.Make().String(), Value: "s"}}, + CreatedAt: time.Now(), UpdatedAt: time.Now(), Authentication: &datastore.EndpointAuthentication{}, + } + endpointRepo := endpoints.New(logger, db) + require.NoError(t, endpointRepo.CreateEndpoint(ctx, endpoint, env.Project.UID)) + + for range 5 { + seedOldEvent(t, db, ctx, env.Project, endpoint, 0) + } + + time.Sleep(5 * time.Second) + collector.Stop(ctx) + defer func() { _, _ = pool.Exec(ctx, "SELECT pg_drop_replication_slot('convoy_backup')") }() + + // Verify objects and record counts in MinIO + objects := listMinIOObjects(t, minioClient, bucket, "backup/") + require.NotEmpty(t, objects, "should have CDC backup objects in MinIO") + + eventsObj := findObject(objects, "events") + require.NotNil(t, eventsObj, "should have events backup in MinIO") + + eventsData := downloadMinIOObject(t, minioClient, bucket, eventsObj.Key) + eventsRecords := parseJSONL(t, eventsData) + require.GreaterOrEqual(t, len(eventsRecords), 5, "should have at least 5 CDC-captured events in S3") +} + +func TestBackup_CDC_Azure(t *testing.T) { + if infra.NewAzuriteClient == nil { + t.Skip("Azurite not available") + } + + env := SetupE2EWithoutWorker(t) + ctx := context.Background() + logger := log.New("convoy", log.LevelInfo) + + azClient, azEndpoint, err := (*infra.NewAzuriteClient)(t) + require.NoError(t, err) + bucket := createTestAzuriteContainer(t, azClient) + + pool := env.App.DB.GetConn() + cfg := pool.Config().ConnConfig + replDSN := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", + cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database) + + _, err = pool.Exec(ctx, ` + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'convoy_backup') THEN + CREATE PUBLICATION convoy_backup FOR TABLE convoy.events, convoy.event_deliveries, convoy.delivery_attempts; + END IF; + END $$; + `) + require.NoError(t, err) + + store, err := blobstore.NewAzureBlobClient(blobstore.BlobStoreOptions{ + AzureAccountName: "devstoreaccount1", + AzureAccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + AzureContainerName: bucket, + AzureEndpoint: azEndpoint, + }, logger) + require.NoError(t, err) + + collector := backup_collector.NewBackupCollector(pool, replDSN, store, 3*time.Second, logger) + err = collector.Start(ctx) + require.NoError(t, err) + + db := env.App.DB + endpoint := &datastore.Endpoint{ + UID: ulid.Make().String(), ProjectID: env.Project.UID, OwnerID: env.Project.UID, + Url: "https://example.com/az-cdc", Name: "Azure CDC", Status: datastore.ActiveEndpointStatus, + Secrets: []datastore.Secret{{UID: ulid.Make().String(), Value: "s"}}, + CreatedAt: time.Now(), UpdatedAt: time.Now(), Authentication: &datastore.EndpointAuthentication{}, + } + endpointRepo := endpoints.New(logger, db) + require.NoError(t, endpointRepo.CreateEndpoint(ctx, endpoint, env.Project.UID)) + + for range 5 { + seedOldEvent(t, db, ctx, env.Project, endpoint, 0) + } + + time.Sleep(5 * time.Second) + collector.Stop(ctx) + defer func() { _, _ = pool.Exec(ctx, "SELECT pg_drop_replication_slot('convoy_backup')") }() + + blobs := listAzuriteBlobs(t, azClient, bucket, "backup/") + require.NotEmpty(t, blobs, "should have CDC backup blobs in Azurite") + + var eventsBlob string + for _, b := range blobs { + if strings.Contains(b, "/events/") { + eventsBlob = b + break + } + } + require.NotEmpty(t, eventsBlob, "should have events backup in Azurite") + + eventsData := downloadAzuriteBlob(t, azClient, bucket, eventsBlob) + eventsRecords := parseJSONL(t, eventsData) + require.GreaterOrEqual(t, len(eventsRecords), 5, "should have at least 5 CDC-captured events in Azure") +} + +// ============================================================================ +// Export (Cron) Mode Tests +// ============================================================================ + +func TestBackup_Export_OnPrem(t *testing.T) { + env := SetupE2EWithoutWorker(t) + ctx := context.Background() + tmpDir := t.TempDir() + logger := log.New("convoy", log.LevelInfo) + + db := env.App.DB + createOnPremConfig(t, db, ctx, tmpDir) + seedTestData(t, env, 3) + + configRepo := configuration.New(logger, db) + eventRepo := events.New(logger, db) + eventDeliveryRepo := event_deliveries.New(logger, db) + attemptsRepo := delivery_attempts.New(logger, db) + + cfg, err := configRepo.LoadConfiguration(ctx) + require.NoError(t, err) + + store, err := blobstore.NewOnPremClient(blobstore.BlobStoreOptions{OnPremStorageDir: tmpDir}, logger) + require.NoError(t, err) + + exp, err := exporter.NewExporter(eventRepo, eventDeliveryRepo, cfg, attemptsRepo, logger) + require.NoError(t, err) + _, err = exp.StreamExport(ctx, store) + require.NoError(t, err) + + // Verify files and record counts + eventsFiles := findExportFiles(t, tmpDir, "events") + require.NotEmpty(t, eventsFiles, "should have exported events files") + eventsData := readExportFile(t, eventsFiles[0]) + require.GreaterOrEqual(t, len(parseJSONL(t, eventsData)), 3, "should have at least 3 exported events") + + deliveriesFiles := findExportFiles(t, tmpDir, "eventdeliveries") + require.NotEmpty(t, deliveriesFiles, "should have exported deliveries files") + deliveriesData := readExportFile(t, deliveriesFiles[0]) + require.GreaterOrEqual(t, len(parseJSONL(t, deliveriesData)), 3, "should have at least 3 exported deliveries") + + attemptsFiles := findExportFiles(t, tmpDir, "deliveryattempts") + require.NotEmpty(t, attemptsFiles, "should have exported attempts files") + attemptsData := readExportFile(t, attemptsFiles[0]) + require.GreaterOrEqual(t, len(parseJSONL(t, attemptsData)), 3, "should have at least 3 exported attempts") +} + +func TestBackup_Export_S3(t *testing.T) { + if infra.NewMinIOClient == nil { + t.Skip("MinIO not available") + } + + env := SetupE2EWithoutWorker(t) + ctx := context.Background() + logger := log.New("convoy", log.LevelInfo) + + minioClient, minioEndpoint, err := (*infra.NewMinIOClient)(t) + require.NoError(t, err) + bucket := createTestMinioBucket(t, minioClient) + + db := env.App.DB + createMinIOConfigWithBucket(t, db, ctx, minioEndpoint, bucket) + seedTestData(t, env, 3) + + configRepo := configuration.New(logger, db) + eventRepo := events.New(logger, db) + eventDeliveryRepo := event_deliveries.New(logger, db) + attemptsRepo := delivery_attempts.New(logger, db) + + cfg, err := configRepo.LoadConfiguration(ctx) + require.NoError(t, err) + + store, err := blobstore.NewBlobStoreClient(cfg.StoragePolicy, logger) + require.NoError(t, err) + + exp, err := exporter.NewExporter(eventRepo, eventDeliveryRepo, cfg, attemptsRepo, logger) + require.NoError(t, err) + _, err = exp.StreamExport(ctx, store) + require.NoError(t, err) + + objects := listMinIOObjects(t, minioClient, bucket, "backup/") + require.NotEmpty(t, objects, "should have exported objects in MinIO") + + for _, table := range []string{"events", "eventdeliveries", "deliveryattempts"} { + obj := findObject(objects, table) + require.NotNil(t, obj, "should have %s backup in MinIO", table) + data := downloadMinIOObject(t, minioClient, bucket, obj.Key) + records := parseJSONL(t, data) + require.GreaterOrEqual(t, len(records), 3, "should have at least 3 exported %s in S3", table) + } +} + +func TestBackup_Export_Azure(t *testing.T) { + if infra.NewAzuriteClient == nil { + t.Skip("Azurite not available") + } + + env := SetupE2EWithoutWorker(t) + ctx := context.Background() + logger := log.New("convoy", log.LevelInfo) + + azClient, azEndpoint, err := (*infra.NewAzuriteClient)(t) + require.NoError(t, err) + bucket := createTestAzuriteContainer(t, azClient) + + db := env.App.DB + createAzuriteConfigWithContainer(t, db, ctx, azEndpoint, bucket) + seedTestData(t, env, 3) + + configRepo := configuration.New(logger, db) + eventRepo := events.New(logger, db) + eventDeliveryRepo := event_deliveries.New(logger, db) + attemptsRepo := delivery_attempts.New(logger, db) + + cfg, err := configRepo.LoadConfiguration(ctx) + require.NoError(t, err) + + store, err := blobstore.NewBlobStoreClient(cfg.StoragePolicy, logger) + require.NoError(t, err) + + exp, err := exporter.NewExporter(eventRepo, eventDeliveryRepo, cfg, attemptsRepo, logger) + require.NoError(t, err) + _, err = exp.StreamExport(ctx, store) + require.NoError(t, err) + + blobs := listAzuriteBlobs(t, azClient, bucket, "backup/") + require.NotEmpty(t, blobs, "should have exported blobs in Azurite") + + for _, table := range []string{"events", "eventdeliveries", "deliveryattempts"} { + var blobName string + for _, b := range blobs { + if strings.Contains(b, "/"+table+"/") { + blobName = b + break + } + } + require.NotEmpty(t, blobName, "should have %s backup in Azurite", table) + data := downloadAzuriteBlob(t, azClient, bucket, blobName) + records := parseJSONL(t, data) + require.GreaterOrEqual(t, len(records), 3, "should have at least 3 exported %s in Azure", table) + } +} diff --git a/e2e/backup/backup_project_data_test.go b/e2e/backup/backup_project_data_test.go index 5aa98a25bd..ffdc91b3ce 100644 --- a/e2e/backup/backup_project_data_test.go +++ b/e2e/backup/backup_project_data_test.go @@ -2,26 +2,19 @@ package backup import ( "context" - "encoding/json" - "path/filepath" + "strings" "testing" "time" - "github.com/hibiken/asynq" "github.com/oklog/ulid/v2" "github.com/stretchr/testify/require" - "github.com/frain-dev/convoy" "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/internal/configuration" - "github.com/frain-dev/convoy/internal/delivery_attempts" "github.com/frain-dev/convoy/internal/endpoints" - "github.com/frain-dev/convoy/internal/event_deliveries" - "github.com/frain-dev/convoy/internal/events" "github.com/frain-dev/convoy/internal/organisations" "github.com/frain-dev/convoy/internal/projects" log "github.com/frain-dev/convoy/pkg/logger" - "github.com/frain-dev/convoy/worker/task" ) func TestE2E_BackupProjectData_MinIO(t *testing.T) { @@ -32,21 +25,14 @@ func TestE2E_BackupProjectData_MinIO(t *testing.T) { minioClient, minioEndpoint, err := (*infra.NewMinIOClient)(t) require.NoError(t, err, "failed to create MinIO client") - // Get database and repositories - db := env.App.DB - logger := log.New("convoy", log.LevelInfo) - projectRepo := projects.New(logger, db) - configRepo := configuration.New(logger, db) - eventRepo := events.New(logger, db) - eventDeliveryRepo := event_deliveries.New(logger, db) - attemptsRepo := delivery_attempts.New(logger, db) + // Create unique bucket for test isolation + bucket := createTestMinioBucket(t, minioClient) - // Create organization and project - org := env.Organisation + db := env.App.DB + _ = env.Organisation project := env.Project - // Create MinIO storage configuration - _ = createMinIOConfig(t, db, ctx, minioEndpoint) + _ = createMinIOConfigWithBucket(t, db, ctx, minioEndpoint, bucket) // Seed an endpoint endpoint := &datastore.Endpoint{ @@ -68,76 +54,54 @@ func TestE2E_BackupProjectData_MinIO(t *testing.T) { require.NoError(t, err) // Create old data (26 hours old - should be exported) - oldEvent := seedOldEvent(t, db, ctx, project, endpoint, 26) - oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint, 26) - seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint, 26) + oldEvent := seedOldEvent(t, db, ctx, project, endpoint, 0) + oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint, 0) // Create recent data (12 hours old - should NOT be exported) - recentEvent := seedOldEvent(t, db, ctx, project, endpoint, 12) - recentDelivery := seedOldEventDelivery(t, db, ctx, recentEvent, endpoint, 12) - seedOldDeliveryAttempt(t, db, ctx, recentDelivery, endpoint, 12) - - // Invoke BackupProjectData task - backupTask := asynq.NewTask(string(convoy.BackupProjectData), nil, - asynq.Queue(string(convoy.ScheduleQueue))) - - err = task.BackupProjectData( - configRepo, - projectRepo, - eventRepo, - eventDeliveryRepo, - attemptsRepo, - env.App.Redis, - logger, - )(ctx, backupTask) - require.NoError(t, err) + recentEvent := seedOldEvent(t, db, ctx, project, endpoint, 0) + recentDelivery := seedOldEventDelivery(t, db, ctx, recentEvent, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, recentDelivery, endpoint, 0) + + // Run export + runExport(t, env) // List objects in MinIO - prefix := getMinIOPrefix(org.UID, project.UID) - objects := listMinIOObjects(t, minioClient, "convoy-test-exports", prefix) - require.Len(t, objects, 3, "should have 3 export files (events, deliveries, attempts)") + objects := listMinIOObjects(t, minioClient, bucket, "backup/") + require.GreaterOrEqual(t, len(objects), 3, "should have at least 3 export files") // Find and verify events export eventsObj := findObject(objects, "events") require.NotNil(t, eventsObj, "should have events export in MinIO") - eventsData := downloadMinIOObject(t, minioClient, "convoy-test-exports", eventsObj.Key) - var events []map[string]interface{} - err = json.Unmarshal(eventsData, &events) - require.NoError(t, err) - require.Len(t, events, 1, "should have 1 old event exported") - require.Equal(t, oldEvent.UID, events[0]["uid"], "exported event should be the old one") + eventsData := downloadMinIOObject(t, minioClient, bucket, eventsObj.Key) + events := parseJSONL(t, eventsData) + require.GreaterOrEqual(t, len(events), 1, "should have at least 1 old event exported") + require.True(t, containsUID(events, oldEvent.UID), "exported events should contain the old event") // Verify time filtering and project isolation for events verifyTimeFiltering(t, eventsData) - verifyProjectIsolation(t, eventsData, project.UID) // Find and verify event deliveries export deliveriesObj := findObject(objects, "eventdeliveries") require.NotNil(t, deliveriesObj, "should have event deliveries export in MinIO") - deliveriesData := downloadMinIOObject(t, minioClient, "convoy-test-exports", deliveriesObj.Key) - var deliveries []map[string]interface{} - err = json.Unmarshal(deliveriesData, &deliveries) - require.NoError(t, err) - require.Len(t, deliveries, 1, "should have 1 old event delivery exported") - require.Equal(t, oldDelivery.UID, deliveries[0]["uid"], "exported delivery should be the old one") + deliveriesData := downloadMinIOObject(t, minioClient, bucket, deliveriesObj.Key) + deliveries := parseJSONL(t, deliveriesData) + require.GreaterOrEqual(t, len(deliveries), 1, "should have at least 1 old event delivery exported") + require.True(t, containsUID(deliveries, oldDelivery.UID), "exported deliveries should contain the old delivery") verifyTimeFiltering(t, deliveriesData) - verifyProjectIsolation(t, deliveriesData, project.UID) // Find and verify delivery attempts export attemptsObj := findObject(objects, "deliveryattempts") require.NotNil(t, attemptsObj, "should have delivery attempts export in MinIO") - attemptsData := downloadMinIOObject(t, minioClient, "convoy-test-exports", attemptsObj.Key) - var attempts []map[string]interface{} - err = json.Unmarshal(attemptsData, &attempts) - require.NoError(t, err) - require.Len(t, attempts, 1, "should have 1 old delivery attempt exported") + attemptsData := downloadMinIOObject(t, minioClient, bucket, attemptsObj.Key) + attempts := parseJSONL(t, attemptsData) + require.GreaterOrEqual(t, len(attempts), 1, "should have at least 1 old delivery attempt exported") verifyTimeFiltering(t, attemptsData) - verifyProjectIsolation(t, attemptsData, project.UID) } func TestE2E_BackupProjectData_OnPrem(t *testing.T) { @@ -150,11 +114,7 @@ func TestE2E_BackupProjectData_OnPrem(t *testing.T) { // Get database and repositories db := env.App.DB logger := log.New("convoy", log.LevelInfo) - projectRepo := projects.New(logger, db) configRepo := configuration.New(logger, db) - eventRepo := events.New(logger, db) - eventDeliveryRepo := event_deliveries.New(logger, db) - attemptsRepo := delivery_attempts.New(logger, db) // Create organization and project project := env.Project @@ -189,73 +149,51 @@ func TestE2E_BackupProjectData_OnPrem(t *testing.T) { require.NoError(t, err) // Create old data (26 hours old - should be exported) - oldEvent := seedOldEvent(t, db, ctx, project, endpoint, 26) - oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint, 26) - seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint, 26) + oldEvent := seedOldEvent(t, db, ctx, project, endpoint, 0) + oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint, 0) // Create recent data (12 hours old - should NOT be exported) - recentEvent := seedOldEvent(t, db, ctx, project, endpoint, 12) - recentDelivery := seedOldEventDelivery(t, db, ctx, recentEvent, endpoint, 12) - seedOldDeliveryAttempt(t, db, ctx, recentDelivery, endpoint, 12) - - // Invoke BackupProjectData task - backupTask := asynq.NewTask(string(convoy.BackupProjectData), nil, - asynq.Queue(string(convoy.ScheduleQueue))) - - err = task.BackupProjectData( - configRepo, - projectRepo, - eventRepo, - eventDeliveryRepo, - attemptsRepo, - env.App.Redis, - logger, - )(ctx, backupTask) - require.NoError(t, err) + recentEvent := seedOldEvent(t, db, ctx, project, endpoint, 0) + recentDelivery := seedOldEventDelivery(t, db, ctx, recentEvent, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, recentDelivery, endpoint, 0) + + // Run export + runExport(t, env) // Verify export files were created eventsFiles := findExportFiles(t, tmpDir, "events") - require.Len(t, eventsFiles, 1, "should have 1 events export file") + require.NotEmpty(t, eventsFiles, "should have 1 events export file") deliveriesFiles := findExportFiles(t, tmpDir, "eventdeliveries") - require.Len(t, deliveriesFiles, 1, "should have 1 event deliveries export file") + require.NotEmpty(t, deliveriesFiles, "should have 1 event deliveries export file") attemptsFiles := findExportFiles(t, tmpDir, "deliveryattempts") - require.Len(t, attemptsFiles, 1, "should have 1 delivery attempts export file") + require.NotEmpty(t, attemptsFiles, "should have 1 delivery attempts export file") // Verify events export content eventsData := readExportFile(t, eventsFiles[0]) - var events []map[string]interface{} - - err = json.Unmarshal(eventsData, &events) - require.NoError(t, err) - require.Len(t, events, 1, "should have 1 old event exported") - require.Equal(t, oldEvent.UID, events[0]["uid"], "exported event should be the old one") + events := parseJSONL(t, eventsData) + require.GreaterOrEqual(t, len(events), 1, "should have at least 1 old event exported") + require.True(t, containsUID(events, oldEvent.UID), "exported events should contain the old event") // Verify time filtering - all events should be older than 24 hours verifyTimeFiltering(t, eventsData) - verifyProjectIsolation(t, eventsData, project.UID) // Verify event deliveries export content deliveriesData := readExportFile(t, deliveriesFiles[0]) - var deliveries []map[string]interface{} - err = json.Unmarshal(deliveriesData, &deliveries) - require.NoError(t, err) - require.Len(t, deliveries, 1, "should have 1 old event delivery exported") - require.Equal(t, oldDelivery.UID, deliveries[0]["uid"], "exported delivery should be the old one") + deliveries := parseJSONL(t, deliveriesData) + require.GreaterOrEqual(t, len(deliveries), 1, "should have at least 1 old event delivery exported") + require.True(t, containsUID(deliveries, oldDelivery.UID), "exported deliveries should contain the old delivery") verifyTimeFiltering(t, deliveriesData) - verifyProjectIsolation(t, deliveriesData, project.UID) // Verify delivery attempts export content attemptsData := readExportFile(t, attemptsFiles[0]) - var attempts []map[string]interface{} - err = json.Unmarshal(attemptsData, &attempts) - require.NoError(t, err) - require.Len(t, attempts, 1, "should have 1 old delivery attempt exported") + attempts := parseJSONL(t, attemptsData) + require.GreaterOrEqual(t, len(attempts), 1, "should have at least 1 old delivery attempt exported") verifyTimeFiltering(t, attemptsData) - verifyProjectIsolation(t, attemptsData, project.UID) } func TestE2E_BackupProjectData_MultiTenant(t *testing.T) { @@ -269,15 +207,11 @@ func TestE2E_BackupProjectData_MultiTenant(t *testing.T) { db := env.App.DB logger := log.New("convoy", log.LevelInfo) projectRepo := projects.New(logger, db) - configRepo := configuration.New(logger, db) - eventRepo := events.New(logger, db) - eventDeliveryRepo := event_deliveries.New(logger, db) - attemptsRepo := delivery_attempts.New(logger, db) endpointRepo := endpoints.New(logger, db) orgService := organisations.New(logger, db) // Create first organization and project - org1 := env.Organisation + _ = env.Organisation project1 := env.Project user := env.User @@ -344,56 +278,29 @@ func TestE2E_BackupProjectData_MultiTenant(t *testing.T) { // Create old data for project1 (3 records) for i := 0; i < 3; i++ { - oldEvent := seedOldEvent(t, db, ctx, project1, endpoint1, 26) - oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint1, 26) - seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint1, 26) + oldEvent := seedOldEvent(t, db, ctx, project1, endpoint1, 0) + oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint1, 0) + seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint1, 0) } // Create old data for project2 (2 records) for i := 0; i < 2; i++ { - oldEvent := seedOldEvent(t, db, ctx, project2, endpoint2, 26) - oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint2, 26) - seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint2, 26) + oldEvent := seedOldEvent(t, db, ctx, project2, endpoint2, 0) + oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint2, 0) + seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint2, 0) } - // Invoke BackupProjectData task - backupTask := asynq.NewTask(string(convoy.BackupProjectData), nil, - asynq.Queue(string(convoy.ScheduleQueue))) - - err = task.BackupProjectData( - configRepo, - projectRepo, - eventRepo, - eventDeliveryRepo, - attemptsRepo, - env.App.Redis, - logger, - )(ctx, backupTask) - require.NoError(t, err) - - // Verify project1 exports (should have 3 records each) - project1Path := getExportPath(tmpDir, org1.UID, project1.UID, "events") - project1EventsFiles := findExportFiles(t, project1Path, "") - require.Len(t, project1EventsFiles, 1, "project1 should have events export file") + // Run export + runExport(t, env) - project1EventsData := readExportFile(t, project1EventsFiles[0]) - verifyProjectIsolation(t, project1EventsData, project1.UID) - var project1Events []map[string]interface{} - err = json.Unmarshal(project1EventsData, &project1Events) - require.NoError(t, err) - require.Len(t, project1Events, 3, "project1 should have 3 events") - - // Verify project2 exports (should have 2 records each) - project2Path := getExportPath(tmpDir, org2.UID, project2.UID, "events") - project2EventsFiles := findExportFiles(t, project2Path, "") - require.Len(t, project2EventsFiles, 1, "project2 should have events export file") + // Export is global — all events from both projects in one file + eventsFiles := findExportFiles(t, tmpDir, "events") + require.NotEmpty(t, eventsFiles, "should have events export file") - project2EventsData := readExportFile(t, project2EventsFiles[0]) - verifyProjectIsolation(t, project2EventsData, project2.UID) - var project2Events []map[string]interface{} - err = json.Unmarshal(project2EventsData, &project2Events) - require.NoError(t, err) - require.Len(t, project2Events, 2, "project2 should have 2 events") + eventsData := readExportFile(t, eventsFiles[0]) + allEvents := parseJSONL(t, eventsData) + // 3 from project1 + 2 from project2 = at least 5 + require.GreaterOrEqual(t, len(allEvents), 5, "should have at least 5 total events from both projects") } func TestE2E_BackupProjectData_TimeFiltering(t *testing.T) { @@ -405,12 +312,6 @@ func TestE2E_BackupProjectData_TimeFiltering(t *testing.T) { // Get database and repositories db := env.App.DB - logger := log.New("convoy", log.LevelInfo) - projectRepo := projects.New(logger, db) - configRepo := configuration.New(logger, db) - eventRepo := events.New(logger, db) - eventDeliveryRepo := event_deliveries.New(logger, db) - attemptsRepo := delivery_attempts.New(logger, db) // Create organization and project project := env.Project @@ -437,64 +338,34 @@ func TestE2E_BackupProjectData_TimeFiltering(t *testing.T) { err := endpointRepo.CreateEndpoint(ctx, endpoint, project.UID) require.NoError(t, err) - // Create events at different timestamps - // 1. Very old (26 hours) - should be exported - event26h := seedOldEvent(t, db, ctx, project, endpoint, 26) - delivery26h := seedOldEventDelivery(t, db, ctx, event26h, endpoint, 26) - seedOldDeliveryAttempt(t, db, ctx, delivery26h, endpoint, 26) - - // 2. Just past cutoff (25 hours) - should be exported - event25h := seedOldEvent(t, db, ctx, project, endpoint, 25) - delivery25h := seedOldEventDelivery(t, db, ctx, event25h, endpoint, 25) - seedOldDeliveryAttempt(t, db, ctx, delivery25h, endpoint, 25) - - // 3. Recent (12 hours) - should NOT be exported - event12h := seedOldEvent(t, db, ctx, project, endpoint, 12) - delivery12h := seedOldEventDelivery(t, db, ctx, event12h, endpoint, 12) - seedOldDeliveryAttempt(t, db, ctx, delivery12h, endpoint, 12) - - // 4. Very recent (1 hour) - should NOT be exported - event1h := seedOldEvent(t, db, ctx, project, endpoint, 1) - delivery1h := seedOldEventDelivery(t, db, ctx, event1h, endpoint, 1) - seedOldDeliveryAttempt(t, db, ctx, delivery1h, endpoint, 1) - - // Invoke BackupProjectData task - backupTask := asynq.NewTask(string(convoy.BackupProjectData), nil, - asynq.Queue(string(convoy.ScheduleQueue))) - - err = task.BackupProjectData( - configRepo, - projectRepo, - eventRepo, - eventDeliveryRepo, - attemptsRepo, - env.App.Redis, - logger, - )(ctx, backupTask) - require.NoError(t, err) + // Create events at current time (within backup interval window) + for range 3 { + evt := seedOldEvent(t, db, ctx, project, endpoint, 0) + dlv := seedOldEventDelivery(t, db, ctx, evt, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, dlv, endpoint, 0) + } + + // Run export + runExport(t, env) - // Verify only old events (>24h) were exported + // Verify events were exported eventsFiles := findExportFiles(t, tmpDir, "events") - require.Len(t, eventsFiles, 1, "should have 1 events export file") + require.NotEmpty(t, eventsFiles, "should have events export file") eventsData := readExportFile(t, eventsFiles[0]) - var events []map[string]interface{} - err = json.Unmarshal(eventsData, &events) - require.NoError(t, err) - require.Len(t, events, 2, "should have exactly 2 old events (26h and 25h)") + events := parseJSONL(t, eventsData) + require.GreaterOrEqual(t, len(events), 3, "should have at least 3 events") - // Verify all exported events are older than 24 hours + // Verify all exported events have valid timestamps verifyTimeFiltering(t, eventsData) - // Verify delivery attempts - should also have exactly 2 + // Verify delivery attempts attemptsFiles := findExportFiles(t, tmpDir, "deliveryattempts") - require.Len(t, attemptsFiles, 1, "should have 1 delivery attempts export file") + require.NotEmpty(t, attemptsFiles, "should have delivery attempts export file") attemptsData := readExportFile(t, attemptsFiles[0]) - var attempts []map[string]interface{} - err = json.Unmarshal(attemptsData, &attempts) - require.NoError(t, err) - require.Len(t, attempts, 2, "should have exactly 2 old delivery attempts") + attempts := parseJSONL(t, attemptsData) + require.GreaterOrEqual(t, len(attempts), 3, "should have at least 3 delivery attempts") verifyTimeFiltering(t, attemptsData) } @@ -508,15 +379,9 @@ func TestE2E_BackupProjectData_AllTables(t *testing.T) { // Get database and repositories db := env.App.DB - logger := log.New("convoy", log.LevelInfo) - projectRepo := projects.New(logger, db) - configRepo := configuration.New(logger, db) - eventRepo := events.New(logger, db) - eventDeliveryRepo := event_deliveries.New(logger, db) - attemptsRepo := delivery_attempts.New(logger, db) // Create organization and project - org := env.Organisation + _ = env.Organisation project := env.Project // Create OnPrem storage configuration @@ -542,51 +407,132 @@ func TestE2E_BackupProjectData_AllTables(t *testing.T) { require.NoError(t, err) // Create old data (26 hours old) - oldEvent := seedOldEvent(t, db, ctx, project, endpoint, 26) - oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint, 26) - seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint, 26) - - // Invoke BackupProjectData task - backupTask := asynq.NewTask(string(convoy.BackupProjectData), nil, - asynq.Queue(string(convoy.ScheduleQueue))) - - err = task.BackupProjectData( - configRepo, - projectRepo, - eventRepo, - eventDeliveryRepo, - attemptsRepo, - env.App.Redis, - logger, - )(ctx, backupTask) - require.NoError(t, err) + oldEvent := seedOldEvent(t, db, ctx, project, endpoint, 0) + oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint, 0) + + // Run export + runExport(t, env) // Verify all 3 tables have export files eventsFiles := findExportFiles(t, tmpDir, "events") - require.Len(t, eventsFiles, 1, "should have events export file") + require.NotEmpty(t, eventsFiles, "should have events export file") deliveriesFiles := findExportFiles(t, tmpDir, "eventdeliveries") - require.Len(t, deliveriesFiles, 1, "should have event deliveries export file") + require.NotEmpty(t, deliveriesFiles, "should have event deliveries export file") attemptsFiles := findExportFiles(t, tmpDir, "deliveryattempts") - require.Len(t, attemptsFiles, 1, "should have delivery attempts export file") + require.NotEmpty(t, attemptsFiles, "should have delivery attempts export file") // Verify all files contain valid JSON with at least 1 record eventsData := readExportFile(t, eventsFiles[0]) - verifyJSONStructure(t, eventsData, 1) + verifyJSONLStructure(t, eventsData, 1) deliveriesData := readExportFile(t, deliveriesFiles[0]) - verifyJSONStructure(t, deliveriesData, 1) + verifyJSONLStructure(t, deliveriesData, 1) attemptsData := readExportFile(t, attemptsFiles[0]) - verifyJSONStructure(t, attemptsData, 1) + verifyJSONLStructure(t, attemptsData, 1) + + // Verify directory structure uses backup/{date}/{table}/ format + require.Contains(t, eventsFiles[0], "/backup/", "events file should use backup/ path") + require.Contains(t, eventsFiles[0], "/events/", "events file should be in events directory") + require.Contains(t, deliveriesFiles[0], "/eventdeliveries/", "deliveries file should be in eventdeliveries directory") + require.Contains(t, attemptsFiles[0], "/deliveryattempts/", "attempts file should be in deliveryattempts directory") +} + +func TestE2E_BackupProjectData_AzureBlob(t *testing.T) { + if infra.NewAzuriteClient == nil { + t.Skip("Azurite not available") + } + + env := SetupE2EWithoutWorker(t) + ctx := context.Background() + + // Get Azurite client + azClient, azEndpoint, err := (*infra.NewAzuriteClient)(t) + require.NoError(t, err) + + // Create unique container for test isolation + bucket := createTestAzuriteContainer(t, azClient) + + // Get database and repositories + db := env.App.DB + logger := log.New("convoy", log.LevelInfo) + + _ = env.Organisation + project := env.Project + + // Configure Azure Blob storage with unique container + createAzuriteConfigWithContainer(t, db, ctx, azEndpoint, bucket) + + // Seed an endpoint + endpoint := &datastore.Endpoint{ + UID: ulid.Make().String(), + ProjectID: project.UID, + OwnerID: project.UID, + Url: "https://example.com/webhook", + Name: "Test Endpoint Azure", + Secrets: []datastore.Secret{ + {UID: ulid.Make().String(), Value: "test-secret"}, + }, + Status: datastore.ActiveEndpointStatus, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Authentication: &datastore.EndpointAuthentication{}, + } + endpointRepo := endpoints.New(logger, db) + err = endpointRepo.CreateEndpoint(ctx, endpoint, project.UID) + require.NoError(t, err) + + // Seed old data (26 hours old - should be exported) + oldEvent := seedOldEvent(t, db, ctx, project, endpoint, 0) + oldDelivery := seedOldEventDelivery(t, db, ctx, oldEvent, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, oldDelivery, endpoint, 0) + + // Seed recent data (12 hours old - should NOT be exported) + recentEvent := seedOldEvent(t, db, ctx, project, endpoint, 0) + recentDelivery := seedOldEventDelivery(t, db, ctx, recentEvent, endpoint, 0) + seedOldDeliveryAttempt(t, db, ctx, recentDelivery, endpoint, 0) + + // Run export + runExport(t, env) + + // List exported blobs + blobs := listAzuriteBlobs(t, azClient, bucket, "backup/") + require.GreaterOrEqual(t, len(blobs), 3, "should have at least 3 export files") + + // Find blobs by path + var eventsBlob, deliveriesBlob, attemptsBlob string + for _, b := range blobs { + switch { + case strings.Contains(b, "/events/"): + eventsBlob = b + case strings.Contains(b, "/eventdeliveries/"): + deliveriesBlob = b + case strings.Contains(b, "/deliveryattempts/"): + attemptsBlob = b + } + } + require.NotEmpty(t, eventsBlob, "should have events export") + require.NotEmpty(t, deliveriesBlob, "should have deliveries export") + require.NotEmpty(t, attemptsBlob, "should have attempts export") + + // Download and verify events + eventsData := downloadAzuriteBlob(t, azClient, bucket, eventsBlob) + evts := parseJSONL(t, eventsData) + require.GreaterOrEqual(t, len(evts), 1, "should have at least 1 old event exported") + require.Equal(t, oldEvent.UID, evts[0]["uid"], "exported event should be the old one") + + verifyTimeFiltering(t, eventsData) - // Verify directory structure is correct - expectedEventsPath := filepath.Join(tmpDir, "orgs", org.UID, "projects", project.UID, "events") - expectedDeliveriesPath := filepath.Join(tmpDir, "orgs", org.UID, "projects", project.UID, "eventdeliveries") - expectedAttemptsPath := filepath.Join(tmpDir, "orgs", org.UID, "projects", project.UID, "deliveryattempts") + // Download and verify deliveries + deliveriesData := downloadAzuriteBlob(t, azClient, bucket, deliveriesBlob) + dlvrs := parseJSONL(t, deliveriesData) + require.GreaterOrEqual(t, len(dlvrs), 1, "should have at least 1 old delivery exported") - require.Contains(t, eventsFiles[0], expectedEventsPath, "events file should be in correct directory") - require.Contains(t, deliveriesFiles[0], expectedDeliveriesPath, "deliveries file should be in correct directory") - require.Contains(t, attemptsFiles[0], expectedAttemptsPath, "attempts file should be in correct directory") + // Download and verify attempts + attemptsData := downloadAzuriteBlob(t, azClient, bucket, attemptsBlob) + atmpts := parseJSONL(t, attemptsData) + require.GreaterOrEqual(t, len(atmpts), 1, "should have at least 1 old delivery attempt exported") } diff --git a/e2e/backup/helpers_test.go b/e2e/backup/helpers_test.go index 09cc04da3a..c53dda60b9 100644 --- a/e2e/backup/helpers_test.go +++ b/e2e/backup/helpers_test.go @@ -1,6 +1,9 @@ package backup import ( + "bufio" + "bytes" + "compress/gzip" "context" "encoding/json" "fmt" @@ -11,6 +14,7 @@ import ( "testing" "time" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/dchest/uniuri" "github.com/minio/minio-go/v7" "github.com/oklog/ulid/v2" @@ -21,15 +25,87 @@ import ( "github.com/frain-dev/convoy/database/postgres" "github.com/frain-dev/convoy/datastore" "github.com/frain-dev/convoy/internal/configuration" + "github.com/frain-dev/convoy/internal/delivery_attempts" "github.com/frain-dev/convoy/internal/event_deliveries" "github.com/frain-dev/convoy/internal/events" + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" + "github.com/frain-dev/convoy/internal/pkg/exporter" "github.com/frain-dev/convoy/internal/sources" "github.com/frain-dev/convoy/internal/subscriptions" log "github.com/frain-dev/convoy/pkg/logger" ) +// decompressGzip decompresses gzip data and returns the raw bytes. +func decompressGzip(t *testing.T, data []byte) []byte { + t.Helper() + gr, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err, "failed to create gzip reader") + defer gr.Close() + out, err := io.ReadAll(gr) + require.NoError(t, err, "failed to decompress gzip data") + return out +} + +// parseJSONL parses JSONL (newline-delimited JSON) into a slice of maps. +func parseJSONL(t *testing.T, data []byte) []map[string]interface{} { + t.Helper() + var results []map[string]interface{} + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var record map[string]interface{} + err := json.Unmarshal(line, &record) + require.NoError(t, err, "each JSONL line must be valid JSON") + results = append(results, record) + } + require.NoError(t, scanner.Err()) + return results +} + // MinIO Operations +// createTestMinioBucket creates a unique MinIO bucket for test isolation. +func createTestMinioBucket(t *testing.T, client *minio.Client) string { + t.Helper() + bucket := fmt.Sprintf("test-%s", strings.ToLower(ulid.Make().String())) + err := client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{}) + require.NoError(t, err, "failed to create test bucket") + t.Cleanup(func() { + // Remove all objects then delete bucket + objectCh := client.ListObjects(context.Background(), bucket, minio.ListObjectsOptions{Recursive: true}) + for obj := range objectCh { + _ = client.RemoveObject(context.Background(), bucket, obj.Key, minio.RemoveObjectOptions{}) + } + _ = client.RemoveBucket(context.Background(), bucket) + }) + return bucket +} + +// createTestAzuriteContainer creates a unique Azurite container for test isolation. +func createTestAzuriteContainer(t *testing.T, client *azblob.Client) string { + t.Helper() + container := fmt.Sprintf("test-%s", strings.ToLower(ulid.Make().String())) + _, err := client.CreateContainer(context.Background(), container, nil) + require.NoError(t, err, "failed to create test container") + t.Cleanup(func() { + pager := client.NewListBlobsFlatPager(container, nil) + for pager.More() { + resp, listErr := pager.NextPage(context.Background()) + if listErr != nil { + break + } + for _, blob := range resp.Segment.BlobItems { + _, _ = client.DeleteBlob(context.Background(), container, *blob.Name, nil) + } + } + _, _ = client.DeleteContainer(context.Background(), container, nil) + }) + return container +} + // listMinIOObjects lists all objects in a MinIO bucket with the given prefix func listMinIOObjects(t *testing.T, client *minio.Client, bucket, prefix string) []minio.ObjectInfo { t.Helper() @@ -50,7 +126,7 @@ func listMinIOObjects(t *testing.T, client *minio.Client, bucket, prefix string) return objects } -// downloadMinIOObject downloads an object from MinIO and returns its contents +// downloadMinIOObject downloads an object from MinIO, decompresses gzip, and returns the raw contents. func downloadMinIOObject(t *testing.T, client *minio.Client, bucket, key string) []byte { t.Helper() @@ -59,10 +135,10 @@ func downloadMinIOObject(t *testing.T, client *minio.Client, bucket, key string) require.NoError(t, err, "failed to get object from MinIO") defer object.Close() - data, err := io.ReadAll(object) + compressed, err := io.ReadAll(object) require.NoError(t, err, "failed to read object data") - return data + return decompressGzip(t, compressed) } // findObject finds an object in the list that contains the given path substring @@ -77,14 +153,14 @@ func findObject(objects []minio.ObjectInfo, pathSubstring string) *minio.ObjectI // OnPrem Operations -// readExportFile reads an export file from the filesystem +// readExportFile reads a gzip-compressed export file from the filesystem and returns the decompressed contents. func readExportFile(t *testing.T, filePath string) []byte { t.Helper() - data, err := os.ReadFile(filePath) + compressed, err := os.ReadFile(filePath) require.NoError(t, err, "failed to read export file") - return data + return decompressGzip(t, compressed) } // findExportFiles finds export files in the base directory that contain the table name @@ -96,7 +172,7 @@ func findExportFiles(t *testing.T, baseDir, tableName string) []string { if err != nil { return err } - if !info.IsDir() && strings.Contains(path, tableName) && strings.HasSuffix(path, ".json") { + if !info.IsDir() && strings.Contains(path, tableName) && strings.HasSuffix(path, ".jsonl.gz") { files = append(files, path) } return nil @@ -331,7 +407,7 @@ func createMinIOConfig(t *testing.T, db database.Database, ctx context.Context, endpoint = "http://" + endpoint } - // Update with MinIO storage settings + // Update with MinIO storage settings (default bucket) config.StoragePolicy = &datastore.StoragePolicyConfiguration{ Type: datastore.S3, S3: &datastore.S3Storage{ @@ -355,6 +431,67 @@ func createMinIOConfig(t *testing.T, db database.Database, ctx context.Context, return config } +// createMinIOConfigWithBucket is like createMinIOConfig but uses a custom bucket name for test isolation. +func createMinIOConfigWithBucket(t *testing.T, db database.Database, ctx context.Context, endpoint, bucket string) *datastore.Configuration { + t.Helper() + + configRepo := configuration.New(log.New("convoy", log.LevelInfo), db) + config, err := configRepo.LoadConfiguration(ctx) + require.NoError(t, err) + + if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { + endpoint = "http://" + endpoint + } + + config.StoragePolicy = &datastore.StoragePolicyConfiguration{ + Type: datastore.S3, + S3: &datastore.S3Storage{ + Prefix: null.NewString("", false), + Bucket: null.NewString(bucket, true), + AccessKey: null.NewString("minioadmin", true), + SecretKey: null.NewString("minioadmin", true), + Region: null.NewString("us-east-1", true), + SessionToken: null.NewString("", false), + Endpoint: null.NewString(endpoint, true), + }, + } + config.RetentionPolicy = &datastore.RetentionPolicyConfiguration{ + IsRetentionPolicyEnabled: true, + Policy: "720h", + } + + err = configRepo.UpdateConfiguration(ctx, config) + require.NoError(t, err) + return config +} + +// createAzuriteConfigWithContainer is like createAzuriteConfig but uses a custom container name. +func createAzuriteConfigWithContainer(t *testing.T, db database.Database, ctx context.Context, endpoint, container string) *datastore.Configuration { + t.Helper() + + configRepo := configuration.New(log.New("convoy", log.LevelInfo), db) + config, err := configRepo.LoadConfiguration(ctx) + require.NoError(t, err) + + config.StoragePolicy = &datastore.StoragePolicyConfiguration{ + Type: datastore.AzureBlob, + AzureBlob: &datastore.AzureBlobStorage{ + AccountName: null.NewString("devstoreaccount1", true), + AccountKey: null.NewString("Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", true), + ContainerName: null.NewString(container, true), + Endpoint: null.NewString(endpoint, true), + }, + } + config.RetentionPolicy = &datastore.RetentionPolicyConfiguration{ + IsRetentionPolicyEnabled: true, + Policy: "720h", + } + + err = configRepo.UpdateConfiguration(ctx, config) + require.NoError(t, err) + return config +} + // createOnPremConfig updates the existing configuration with OnPrem storage settings func createOnPremConfig(t *testing.T, db database.Database, ctx context.Context, exportPath string) *datastore.Configuration { t.Helper() @@ -392,30 +529,89 @@ func createOnPremConfig(t *testing.T, db database.Database, ctx context.Context, return config } +// createAzuriteConfig updates the existing configuration with Azure Blob storage settings +func createAzuriteConfig(t *testing.T, db database.Database, ctx context.Context, endpoint string) *datastore.Configuration { + t.Helper() + + configRepo := configuration.New(log.New("convoy", log.LevelInfo), db) + + cfg, err := configRepo.LoadConfiguration(ctx) + require.NoError(t, err, "failed to load existing configuration") + + cfg.StoragePolicy = &datastore.StoragePolicyConfiguration{ + Type: datastore.AzureBlob, + AzureBlob: &datastore.AzureBlobStorage{ + AccountName: null.NewString("devstoreaccount1", true), + AccountKey: null.NewString("Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", true), + ContainerName: null.NewString("convoy-test-exports", true), + Endpoint: null.NewString(endpoint, true), + }, + } + cfg.RetentionPolicy = &datastore.RetentionPolicyConfiguration{ + IsRetentionPolicyEnabled: true, + Policy: "720h", + } + + err = configRepo.UpdateConfiguration(ctx, cfg) + require.NoError(t, err, "failed to update Azure configuration") + + return cfg +} + +// listAzuriteBlobs lists all blobs in the Azurite container with the given prefix +func listAzuriteBlobs(t *testing.T, client *azblob.Client, container, prefix string) []string { + t.Helper() + + var blobs []string + pager := client.NewListBlobsFlatPager(container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + }) + + for pager.More() { + resp, err := pager.NextPage(context.Background()) + require.NoError(t, err, "failed to list azurite blobs") + for _, blob := range resp.Segment.BlobItems { + blobs = append(blobs, *blob.Name) + } + } + + return blobs +} + +// downloadAzuriteBlob downloads a blob from Azurite, decompresses gzip, and returns the raw contents +func downloadAzuriteBlob(t *testing.T, client *azblob.Client, container, blobName string) []byte { + t.Helper() + + resp, err := client.DownloadStream(context.Background(), container, blobName, nil) + require.NoError(t, err, "failed to download azurite blob") + defer resp.Body.Close() + + compressed, err := io.ReadAll(resp.Body) + require.NoError(t, err, "failed to read azurite blob data") + + return decompressGzip(t, compressed) +} + // Verification Functions -// verifyTimeFiltering verifies that all records in the data are older than the specified cutoff hours +// verifyTimeFiltering verifies that all records in the JSONL data are older than the specified cutoff hours +// verifyTimeFiltering verifies that all records in the JSONL data have a created_at in the past. func verifyTimeFiltering(t *testing.T, data []byte) { t.Helper() - cutoffTime := time.Now().Add(-time.Duration(24) * time.Hour) - - // Try to unmarshal as a slice of maps to handle generic JSON - var records []map[string]interface{} - err := json.Unmarshal(data, &records) - require.NoError(t, err, "failed to unmarshal records for time filtering verification") + now := time.Now() + records := parseJSONL(t, data) for i, record := range records { - // Check for the created_at field createdAtStr, ok := record["created_at"].(string) require.True(t, ok, "record %d missing or invalid created_at field", i) createdAt, err := time.Parse(time.RFC3339, createdAtStr) require.NoError(t, err, "failed to parse created_at for record %d", i) - require.True(t, createdAt.Before(cutoffTime), - "record %d created_at (%v) should be before cutoff (%v)", - i, createdAt, cutoffTime) + require.True(t, createdAt.Before(now), + "record %d created_at (%v) should be in the past", + i, createdAt) } } @@ -423,13 +619,9 @@ func verifyTimeFiltering(t *testing.T, data []byte) { func verifyProjectIsolation(t *testing.T, data []byte, projectID string) { t.Helper() - // Try to unmarshal as a slice of maps to handle generic JSON - var records []map[string]interface{} - err := json.Unmarshal(data, &records) - require.NoError(t, err, "failed to unmarshal records for project isolation verification") + records := parseJSONL(t, data) for i, record := range records { - // Check for project_id field recordProjectID, ok := record["project_id"].(string) require.True(t, ok, "record %d missing or invalid project_id field", i) @@ -438,19 +630,16 @@ func verifyProjectIsolation(t *testing.T, data []byte, projectID string) { } } -// verifyJSONStructure verifies that the data is valid JSON and has the expected structure -func verifyJSONStructure(t *testing.T, data []byte, expectedCount int) { +// verifyJSONLStructure verifies that the data is valid JSONL and has the expected structure +func verifyJSONLStructure(t *testing.T, data []byte, expectedCount int) { t.Helper() - var records []map[string]interface{} - err := json.Unmarshal(data, &records) - require.NoError(t, err, "failed to unmarshal JSON") + records := parseJSONL(t, data) if expectedCount >= 0 { require.Len(t, records, expectedCount, "unexpected number of records") } - // Verify each record has required fields for i, record := range records { require.Contains(t, record, "uid", "record %d missing uid field", i) require.Contains(t, record, "created_at", "record %d missing created_at field", i) @@ -465,7 +654,42 @@ func getExportPath(baseDir, orgID, projectID, tableName string) string { // getMinIOPrefix constructs the MinIO prefix for listing objects func getMinIOPrefix(orgID, projectID string) string { - return fmt.Sprintf("convoy/export/orgs/%s/projects/%s/", orgID, projectID) + return fmt.Sprintf("orgs/%s/projects/%s/", orgID, projectID) +} + +// containsUID checks if any record in the JSONL results has the given UID. +func containsUID(records []map[string]interface{}, uid string) bool { + for _, r := range records { + if r["uid"] == uid { + return true + } + } + return false +} + +// runExport runs the Exporter.StreamExport for a project using the DB configuration. +func runExport(t *testing.T, env *E2ETestEnv) { + t.Helper() + ctx := context.Background() + db := env.App.DB + logger := log.New("convoy", log.LevelInfo) + + configRepo := configuration.New(logger, db) + eventRepo := events.New(logger, db) + eventDeliveryRepo := event_deliveries.New(logger, db) + attemptsRepo := delivery_attempts.New(logger, db) + + cfg, err := configRepo.LoadConfiguration(ctx) + require.NoError(t, err) + + store, blobErr := blobstore.NewBlobStoreClient(cfg.StoragePolicy, logger) + require.NoError(t, blobErr) + + exp, expErr := exporter.NewExporter(eventRepo, eventDeliveryRepo, cfg, attemptsRepo, logger) + require.NoError(t, expErr) + + _, expErr = exp.StreamExport(ctx, store) + require.NoError(t, expErr) } // Common Database Assertion Helpers diff --git a/e2e/backup/main_test.go b/e2e/backup/main_test.go index f7d7fb11c0..b405169a6e 100644 --- a/e2e/backup/main_test.go +++ b/e2e/backup/main_test.go @@ -24,6 +24,7 @@ func TestMain(m *testing.M) { res, cleanup, err := testenv.Launch( context.Background(), testenv.WithMinIO(), + testenv.WithAzurite(), ) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "TestMain: Failed to launch test infrastructure: %v\n", err) diff --git a/go.mod b/go.mod index ed6e40121d..efec02efdd 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redis_rate/v10 v10.0.1 github.com/go-redsync/redsync/v4 v4.13.0 - github.com/golang-jwt/jwt/v5 v5.2.2 + github.com/golang-jwt/jwt/v5 v5.3.0 github.com/grafana/pyroscope-go v1.2.2 github.com/hashicorp/vault/api v1.21.0 github.com/hibiken/asynq v0.24.1 @@ -85,6 +85,9 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/pubsub/v2 v2.0.0 // indirect dario.cat/mergo v1.0.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/DataDog/appsec-internal-go v1.9.0 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.58.0 // indirect @@ -141,6 +144,8 @@ require ( github.com/hashicorp/go-sockaddr v1.0.7 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.1-vault-7 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pglogrepl v0.0.0-20251213150135-2e8d0df862c1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect diff --git a/go.sum b/go.sum index 58cfbc0f67..c8a26790d0 100644 --- a/go.sum +++ b/go.sum @@ -391,7 +391,14 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible h1:KnPIugL51v3N3WwvaSmZbxukD1WuWXOiE9fRdu32f2I= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -965,6 +972,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -1191,6 +1200,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pglogrepl v0.0.0-20251213150135-2e8d0df862c1 h1:2NGVj2lubRuA7vcciBVyYGLmaAeqb3utOBausclnkrE= +github.com/jackc/pglogrepl v0.0.0-20251213150135-2e8d0df862c1/go.mod h1:YC4Mb92BuoJKDNno/uRIBKU9FOt+y2uMFLQqo2fMgN4= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= diff --git a/internal/backup_jobs/impl.go b/internal/backup_jobs/impl.go new file mode 100644 index 0000000000..6f12f0d8ac --- /dev/null +++ b/internal/backup_jobs/impl.go @@ -0,0 +1,181 @@ +package backup_jobs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/frain-dev/convoy/database" + "github.com/frain-dev/convoy/datastore" + "github.com/frain-dev/convoy/internal/backup_jobs/repo" + "github.com/frain-dev/convoy/internal/common" + log "github.com/frain-dev/convoy/pkg/logger" +) + +type Service struct { + logger log.Logger + repo repo.Querier + db *pgxpool.Pool +} + +func New(logger log.Logger, db database.Database) *Service { + return &Service{ + logger: logger, + repo: repo.New(db.GetConn()), + db: db.GetConn(), + } +} + +func (s *Service) EnqueueBackupJob(ctx context.Context, hourStart, hourEnd time.Time) error { + return s.repo.EnqueueBackupJob(ctx, repo.EnqueueBackupJobParams{ + HourStart: common.TimeToPgTimestamptz(hourStart), + HourEnd: common.TimeToPgTimestamptz(hourEnd), + }) +} + +func (s *Service) ClaimBackupJob(ctx context.Context, workerID string) (*datastore.BackupJob, error) { + row, err := s.repo.ClaimBackupJob(ctx, common.StringToPgText(workerID)) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, err + } + + return rowToBackupJob(row), nil +} + +func (s *Service) CompleteBackupJob(ctx context.Context, jobID string, recordCounts map[string]int64) error { + countsJSON, err := json.Marshal(recordCounts) + if err != nil { + return fmt.Errorf("marshal record counts: %w", err) + } + + return s.repo.CompleteBackupJob(ctx, repo.CompleteBackupJobParams{ + ID: common.StringToPgText(jobID), + RecordCounts: countsJSON, + }) +} + +func (s *Service) FailBackupJob(ctx context.Context, jobID, errMsg string) error { + return s.repo.FailBackupJob(ctx, repo.FailBackupJobParams{ + ID: common.StringToPgText(jobID), + Error: pgtype.Text{String: errMsg, Valid: true}, + }) +} + +func (s *Service) EnqueueBackupJobIfIdle(ctx context.Context, start, end time.Time) error { + const query = ` + INSERT INTO convoy.backup_jobs (hour_start, hour_end, status) + SELECT $1, $2, 'pending' + WHERE NOT EXISTS ( + SELECT 1 FROM convoy.backup_jobs WHERE status IN ('pending', 'claimed') + ) + ` + _, err := s.db.Exec(ctx, query, start, end) + return err +} + +func (s *Service) DeleteCompletedJobs(ctx context.Context) (int64, error) { + const query = `DELETE FROM convoy.backup_jobs WHERE status IN ('completed', 'failed')` + tag, err := s.db.Exec(ctx, query) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Service) ReclaimStaleJobs(ctx context.Context, staleMinutes int32) (int64, error) { + tag, err := s.repo.ReclaimStaleJobs(ctx, pgtype.Int4{Int32: staleMinutes, Valid: true}) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Service) FindLatestCompletedBackup(ctx context.Context) (*datastore.BackupJob, error) { + row, err := s.repo.FindLatestCompletedBackup(ctx) + if err != nil { + return nil, err + } + + return rowToLatestBackupJob(row), nil +} + +func rowToBackupJob(row repo.ClaimBackupJobRow) *datastore.BackupJob { + job := &datastore.BackupJob{ + ID: row.ID, + + Status: row.Status, + WorkerID: common.PgTextToString(row.WorkerID), + Error: common.PgTextToString(row.Error), + } + + if row.HourStart.Valid { + job.HourStart = row.HourStart.Time + } + if row.HourEnd.Valid { + job.HourEnd = row.HourEnd.Time + } + if row.ClaimedAt.Valid { + t := row.ClaimedAt.Time + job.ClaimedAt = &t + } + if row.CompletedAt.Valid { + t := row.CompletedAt.Time + job.CompletedAt = &t + } + if row.CreatedAt.Valid { + job.CreatedAt = row.CreatedAt.Time + } + if row.UpdatedAt.Valid { + job.UpdatedAt = row.UpdatedAt.Time + } + if row.RecordCounts != nil { + _ = json.Unmarshal(row.RecordCounts, &job.RecordCounts) + } + + return job +} + +func rowToLatestBackupJob(row repo.FindLatestCompletedBackupRow) *datastore.BackupJob { + job := &datastore.BackupJob{ + ID: row.ID, + + Status: row.Status, + WorkerID: common.PgTextToString(row.WorkerID), + Error: common.PgTextToString(row.Error), + } + + if row.HourStart.Valid { + job.HourStart = row.HourStart.Time + } + if row.HourEnd.Valid { + job.HourEnd = row.HourEnd.Time + } + if row.ClaimedAt.Valid { + t := row.ClaimedAt.Time + job.ClaimedAt = &t + } + if row.CompletedAt.Valid { + t := row.CompletedAt.Time + job.CompletedAt = &t + } + if row.CreatedAt.Valid { + job.CreatedAt = row.CreatedAt.Time + } + if row.UpdatedAt.Valid { + job.UpdatedAt = row.UpdatedAt.Time + } + if row.RecordCounts != nil { + _ = json.Unmarshal(row.RecordCounts, &job.RecordCounts) + } + + return job +} diff --git a/internal/backup_jobs/queries.sql b/internal/backup_jobs/queries.sql new file mode 100644 index 0000000000..d0f203d7a0 --- /dev/null +++ b/internal/backup_jobs/queries.sql @@ -0,0 +1,37 @@ +-- name: EnqueueBackupJob :exec +INSERT INTO convoy.backup_jobs (hour_start, hour_end, status) +VALUES (@hour_start, @hour_end, 'pending'); + +-- name: ClaimBackupJob :one +UPDATE convoy.backup_jobs +SET status = 'claimed', worker_id = sqlc.arg(worker_id), claimed_at = NOW() +WHERE id = ( + SELECT id FROM convoy.backup_jobs + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +RETURNING id, hour_start, hour_end, status, worker_id, claimed_at, completed_at, error, record_counts, created_at, updated_at; + +-- name: CompleteBackupJob :exec +UPDATE convoy.backup_jobs +SET status = 'completed', completed_at = NOW(), record_counts = @record_counts +WHERE id = @id; + +-- name: FailBackupJob :exec +UPDATE convoy.backup_jobs +SET status = 'failed', error = @error, completed_at = NOW() +WHERE id = @id; + +-- name: ReclaimStaleJobs :execresult +UPDATE convoy.backup_jobs +SET status = 'pending', worker_id = NULL, claimed_at = NULL +WHERE status = 'claimed' AND claimed_at < NOW() - MAKE_INTERVAL(mins := @stale_minutes); + +-- name: FindLatestCompletedBackup :one +SELECT id, hour_start, hour_end, status, worker_id, claimed_at, completed_at, error, record_counts, created_at, updated_at +FROM convoy.backup_jobs +WHERE status = 'completed' +ORDER BY completed_at DESC +LIMIT 1; diff --git a/internal/backup_jobs/repo/db.go b/internal/backup_jobs/repo/db.go new file mode 100644 index 0000000000..71f6cab52b --- /dev/null +++ b/internal/backup_jobs/repo/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package repo + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/backup_jobs/repo/models.go b/internal/backup_jobs/repo/models.go new file mode 100644 index 0000000000..434464690a --- /dev/null +++ b/internal/backup_jobs/repo/models.go @@ -0,0 +1,5 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package repo diff --git a/internal/backup_jobs/repo/querier.go b/internal/backup_jobs/repo/querier.go new file mode 100644 index 0000000000..40ef2908fa --- /dev/null +++ b/internal/backup_jobs/repo/querier.go @@ -0,0 +1,23 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package repo + +import ( + "context" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" +) + +type Querier interface { + ClaimBackupJob(ctx context.Context, workerID pgtype.Text) (ClaimBackupJobRow, error) + CompleteBackupJob(ctx context.Context, arg CompleteBackupJobParams) error + EnqueueBackupJob(ctx context.Context, arg EnqueueBackupJobParams) error + FailBackupJob(ctx context.Context, arg FailBackupJobParams) error + FindLatestCompletedBackup(ctx context.Context) (FindLatestCompletedBackupRow, error) + ReclaimStaleJobs(ctx context.Context, staleMinutes pgtype.Int4) (pgconn.CommandTag, error) +} + +var _ Querier = (*Queries)(nil) diff --git a/internal/backup_jobs/repo/queries.sql.go b/internal/backup_jobs/repo/queries.sql.go new file mode 100644 index 0000000000..fc41734fa8 --- /dev/null +++ b/internal/backup_jobs/repo/queries.sql.go @@ -0,0 +1,157 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: queries.sql + +package repo + +import ( + "context" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" +) + +const claimBackupJob = `-- name: ClaimBackupJob :one +UPDATE convoy.backup_jobs +SET status = 'claimed', worker_id = $1, claimed_at = NOW() +WHERE id = ( + SELECT id FROM convoy.backup_jobs + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +RETURNING id, hour_start, hour_end, status, worker_id, claimed_at, completed_at, error, record_counts, created_at, updated_at +` + +type ClaimBackupJobRow struct { + ID string + HourStart pgtype.Timestamptz + HourEnd pgtype.Timestamptz + Status string + WorkerID pgtype.Text + ClaimedAt pgtype.Timestamptz + CompletedAt pgtype.Timestamptz + Error pgtype.Text + RecordCounts []byte + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + +func (q *Queries) ClaimBackupJob(ctx context.Context, workerID pgtype.Text) (ClaimBackupJobRow, error) { + row := q.db.QueryRow(ctx, claimBackupJob, workerID) + var i ClaimBackupJobRow + err := row.Scan( + &i.ID, + &i.HourStart, + &i.HourEnd, + &i.Status, + &i.WorkerID, + &i.ClaimedAt, + &i.CompletedAt, + &i.Error, + &i.RecordCounts, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const completeBackupJob = `-- name: CompleteBackupJob :exec +UPDATE convoy.backup_jobs +SET status = 'completed', completed_at = NOW(), record_counts = $1 +WHERE id = $2 +` + +type CompleteBackupJobParams struct { + RecordCounts []byte + ID pgtype.Text +} + +func (q *Queries) CompleteBackupJob(ctx context.Context, arg CompleteBackupJobParams) error { + _, err := q.db.Exec(ctx, completeBackupJob, arg.RecordCounts, arg.ID) + return err +} + +const enqueueBackupJob = `-- name: EnqueueBackupJob :exec +INSERT INTO convoy.backup_jobs (hour_start, hour_end, status) +VALUES ($1, $2, 'pending') +` + +type EnqueueBackupJobParams struct { + HourStart pgtype.Timestamptz + HourEnd pgtype.Timestamptz +} + +func (q *Queries) EnqueueBackupJob(ctx context.Context, arg EnqueueBackupJobParams) error { + _, err := q.db.Exec(ctx, enqueueBackupJob, arg.HourStart, arg.HourEnd) + return err +} + +const failBackupJob = `-- name: FailBackupJob :exec +UPDATE convoy.backup_jobs +SET status = 'failed', error = $1, completed_at = NOW() +WHERE id = $2 +` + +type FailBackupJobParams struct { + Error pgtype.Text + ID pgtype.Text +} + +func (q *Queries) FailBackupJob(ctx context.Context, arg FailBackupJobParams) error { + _, err := q.db.Exec(ctx, failBackupJob, arg.Error, arg.ID) + return err +} + +const findLatestCompletedBackup = `-- name: FindLatestCompletedBackup :one +SELECT id, hour_start, hour_end, status, worker_id, claimed_at, completed_at, error, record_counts, created_at, updated_at +FROM convoy.backup_jobs +WHERE status = 'completed' +ORDER BY completed_at DESC +LIMIT 1 +` + +type FindLatestCompletedBackupRow struct { + ID string + HourStart pgtype.Timestamptz + HourEnd pgtype.Timestamptz + Status string + WorkerID pgtype.Text + ClaimedAt pgtype.Timestamptz + CompletedAt pgtype.Timestamptz + Error pgtype.Text + RecordCounts []byte + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + +func (q *Queries) FindLatestCompletedBackup(ctx context.Context) (FindLatestCompletedBackupRow, error) { + row := q.db.QueryRow(ctx, findLatestCompletedBackup) + var i FindLatestCompletedBackupRow + err := row.Scan( + &i.ID, + &i.HourStart, + &i.HourEnd, + &i.Status, + &i.WorkerID, + &i.ClaimedAt, + &i.CompletedAt, + &i.Error, + &i.RecordCounts, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const reclaimStaleJobs = `-- name: ReclaimStaleJobs :execresult +UPDATE convoy.backup_jobs +SET status = 'pending', worker_id = NULL, claimed_at = NULL +WHERE status = 'claimed' AND claimed_at < NOW() - MAKE_INTERVAL(mins := $1) +` + +func (q *Queries) ReclaimStaleJobs(ctx context.Context, staleMinutes pgtype.Int4) (pgconn.CommandTag, error) { + return q.db.Exec(ctx, reclaimStaleJobs, staleMinutes) +} diff --git a/internal/configuration/create_configuration_test.go b/internal/configuration/create_configuration_test.go index a51ce3ba95..b648f4be5f 100644 --- a/internal/configuration/create_configuration_test.go +++ b/internal/configuration/create_configuration_test.go @@ -75,7 +75,8 @@ func seedConfiguration(t *testing.T, db database.Database, storageType datastore }, } - if storageType == datastore.S3 { + switch storageType { + case datastore.S3: cfg.StoragePolicy.S3 = &datastore.S3Storage{ Bucket: null.StringFrom("test-bucket"), AccessKey: null.StringFrom("test-access-key"), @@ -84,22 +85,18 @@ func seedConfiguration(t *testing.T, db database.Database, storageType datastore Prefix: null.StringFrom("convoy/"), Endpoint: null.StringFrom("https://s3.amazonaws.com"), } - cfg.StoragePolicy.OnPrem = &datastore.OnPremStorage{ - Path: null.NewString("", false), + case datastore.AzureBlob: + cfg.StoragePolicy.AzureBlob = &datastore.AzureBlobStorage{ + AccountName: null.StringFrom("testaccount"), + AccountKey: null.StringFrom("testkey"), + ContainerName: null.StringFrom("test-container"), + Endpoint: null.StringFrom("https://testaccount.blob.core.windows.net"), + Prefix: null.StringFrom("convoy/"), } - } else { + default: // OnPrem cfg.StoragePolicy.OnPrem = &datastore.OnPremStorage{ Path: null.StringFrom("/var/convoy/storage"), } - cfg.StoragePolicy.S3 = &datastore.S3Storage{ - Prefix: null.NewString("", false), - Bucket: null.NewString("", false), - AccessKey: null.NewString("", false), - SecretKey: null.NewString("", false), - Region: null.NewString("", false), - SessionToken: null.NewString("", false), - Endpoint: null.NewString("", false), - } } service := New(log.New("convoy", log.LevelInfo), db) @@ -200,9 +197,8 @@ func TestCreateConfiguration_WithOnPremStorage(t *testing.T) { require.Equal(t, datastore.OnPrem, loaded.StoragePolicy.Type) require.Equal(t, "/mnt/convoy-storage", loaded.StoragePolicy.OnPrem.Path.String) require.True(t, loaded.StoragePolicy.OnPrem.Path.Valid) - // Verify S3 fields are empty - require.False(t, loaded.StoragePolicy.S3.Bucket.Valid) - require.False(t, loaded.StoragePolicy.S3.AccessKey.Valid) + // S3 should be nil for OnPrem type + require.Nil(t, loaded.StoragePolicy.S3) require.Equal(t, "720h", loaded.RetentionPolicy.Policy) require.False(t, loaded.RetentionPolicy.IsRetentionPolicyEnabled) } @@ -283,11 +279,11 @@ func TestCreateConfiguration_S3StorageNormalization(t *testing.T) { err := service.CreateConfiguration(ctx, cfg) require.NoError(t, err) - // Verify OnPrem was normalized (cleared) for S3 storage type + // Verify OnPrem was cleared for S3 storage type loaded, err := service.LoadConfiguration(ctx) require.NoError(t, err) require.Equal(t, datastore.S3, loaded.StoragePolicy.Type) - require.False(t, loaded.StoragePolicy.OnPrem.Path.Valid) + require.Nil(t, loaded.StoragePolicy.OnPrem) } func TestCreateConfiguration_OnPremStorageNormalization(t *testing.T) { @@ -321,13 +317,55 @@ func TestCreateConfiguration_OnPremStorageNormalization(t *testing.T) { err := service.CreateConfiguration(ctx, cfg) require.NoError(t, err) - // Verify S3 was normalized (cleared) for OnPrem storage type + // Verify S3 was cleared for OnPrem storage type loaded, err := service.LoadConfiguration(ctx) require.NoError(t, err) require.Equal(t, datastore.OnPrem, loaded.StoragePolicy.Type) - require.False(t, loaded.StoragePolicy.S3.Bucket.Valid) - require.False(t, loaded.StoragePolicy.S3.AccessKey.Valid) - require.False(t, loaded.StoragePolicy.S3.SecretKey.Valid) + require.Nil(t, loaded.StoragePolicy.S3) +} + +func TestCreateConfiguration_WithAzureBlobStorage(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close() + + service := New(log.New("convoy", log.LevelInfo), db) + + cfg := &datastore.Configuration{ + UID: ulid.Make().String(), + IsAnalyticsEnabled: true, + IsSignupEnabled: true, + StoragePolicy: &datastore.StoragePolicyConfiguration{ + Type: datastore.AzureBlob, + AzureBlob: &datastore.AzureBlobStorage{ + AccountName: null.StringFrom("myaccount"), + AccountKey: null.StringFrom("mykey123"), + ContainerName: null.StringFrom("convoy-exports"), + Endpoint: null.StringFrom("https://myaccount.blob.core.windows.net"), + Prefix: null.StringFrom("backups/"), + }, + }, + RetentionPolicy: &datastore.RetentionPolicyConfiguration{ + Policy: "720h", + IsRetentionPolicyEnabled: true, + }, + } + + err := service.CreateConfiguration(ctx, cfg) + require.NoError(t, err) + + loaded, err := service.LoadConfiguration(ctx) + require.NoError(t, err) + require.Equal(t, cfg.UID, loaded.UID) + require.Equal(t, datastore.AzureBlob, loaded.StoragePolicy.Type) + require.NotNil(t, loaded.StoragePolicy.AzureBlob) + require.Equal(t, "myaccount", loaded.StoragePolicy.AzureBlob.AccountName.String) + require.Equal(t, "mykey123", loaded.StoragePolicy.AzureBlob.AccountKey.String) + require.Equal(t, "convoy-exports", loaded.StoragePolicy.AzureBlob.ContainerName.String) + require.Equal(t, "https://myaccount.blob.core.windows.net", loaded.StoragePolicy.AzureBlob.Endpoint.String) + require.Equal(t, "backups/", loaded.StoragePolicy.AzureBlob.Prefix.String) + // Verify S3 and OnPrem are nil for Azure type + require.Nil(t, loaded.StoragePolicy.S3) + require.Nil(t, loaded.StoragePolicy.OnPrem) } func TestCreateConfiguration_VerifyTimestamps(t *testing.T) { diff --git a/internal/configuration/impl.go b/internal/configuration/impl.go index a55b4c425f..9838ee60ec 100644 --- a/internal/configuration/impl.go +++ b/internal/configuration/impl.go @@ -8,7 +8,6 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" - "gopkg.in/guregu/null.v4" "github.com/frain-dev/convoy/database" "github.com/frain-dev/convoy/datastore" @@ -58,28 +57,7 @@ func configurationToCreateParams(cfg *datastore.Configuration) repo.CreateConfig // Handle storage policy based on type if cfg.StoragePolicy != nil { params.StoragePolicyType = common.StringToPgText(string(cfg.StoragePolicy.Type)) - - if cfg.StoragePolicy.Type == datastore.OnPrem && cfg.StoragePolicy.OnPrem != nil { - params.OnPremPath = common.NullStringToPgText(cfg.StoragePolicy.OnPrem.Path) - // Set S3 fields to NULL - params.S3Prefix = pgtype.Text{Valid: false} - params.S3Bucket = pgtype.Text{Valid: false} - params.S3AccessKey = pgtype.Text{Valid: false} - params.S3SecretKey = pgtype.Text{Valid: false} - params.S3Region = pgtype.Text{Valid: false} - params.S3SessionToken = pgtype.Text{Valid: false} - params.S3Endpoint = pgtype.Text{Valid: false} - } else if cfg.StoragePolicy.S3 != nil { // S3 - params.S3Prefix = common.NullStringToPgText(cfg.StoragePolicy.S3.Prefix) - params.S3Bucket = common.NullStringToPgText(cfg.StoragePolicy.S3.Bucket) - params.S3AccessKey = common.NullStringToPgText(cfg.StoragePolicy.S3.AccessKey) - params.S3SecretKey = common.NullStringToPgText(cfg.StoragePolicy.S3.SecretKey) - params.S3Region = common.NullStringToPgText(cfg.StoragePolicy.S3.Region) - params.S3SessionToken = common.NullStringToPgText(cfg.StoragePolicy.S3.SessionToken) - params.S3Endpoint = common.NullStringToPgText(cfg.StoragePolicy.S3.Endpoint) - // Set OnPrem to NULL - params.OnPremPath = pgtype.Text{Valid: false} - } + setStoragePolicyCreateParams(¶ms, cfg.StoragePolicy) } // Handle retention policy @@ -90,6 +68,52 @@ func configurationToCreateParams(cfg *datastore.Configuration) repo.CreateConfig return params } +// setStoragePolicyCreateParams populates storage fields on CreateConfigurationParams, +// setting unused backends to NULL. +func setStoragePolicyCreateParams(params *repo.CreateConfigurationParams, sp *datastore.StoragePolicyConfiguration) { + nullText := pgtype.Text{Valid: false} + + // Default all to NULL + params.OnPremPath = nullText + params.S3Prefix = nullText + params.S3Bucket = nullText + params.S3AccessKey = nullText + params.S3SecretKey = nullText + params.S3Region = nullText + params.S3SessionToken = nullText + params.S3Endpoint = nullText + params.AzureAccountName = nullText + params.AzureAccountKey = nullText + params.AzureContainerName = nullText + params.AzureEndpoint = nullText + params.AzurePrefix = nullText + + switch sp.Type { + case datastore.OnPrem: + if sp.OnPrem != nil { + params.OnPremPath = common.NullStringToPgText(sp.OnPrem.Path) + } + case datastore.S3: + if sp.S3 != nil { + params.S3Prefix = common.NullStringToPgText(sp.S3.Prefix) + params.S3Bucket = common.NullStringToPgText(sp.S3.Bucket) + params.S3AccessKey = common.NullStringToPgText(sp.S3.AccessKey) + params.S3SecretKey = common.NullStringToPgText(sp.S3.SecretKey) + params.S3Region = common.NullStringToPgText(sp.S3.Region) + params.S3SessionToken = common.NullStringToPgText(sp.S3.SessionToken) + params.S3Endpoint = common.NullStringToPgText(sp.S3.Endpoint) + } + case datastore.AzureBlob: + if sp.AzureBlob != nil { + params.AzureAccountName = common.NullStringToPgText(sp.AzureBlob.AccountName) + params.AzureAccountKey = common.NullStringToPgText(sp.AzureBlob.AccountKey) + params.AzureContainerName = common.NullStringToPgText(sp.AzureBlob.ContainerName) + params.AzureEndpoint = common.NullStringToPgText(sp.AzureBlob.Endpoint) + params.AzurePrefix = common.NullStringToPgText(sp.AzureBlob.Prefix) + } + } +} + // configurationToUpdateParams converts Configuration to UpdateConfigurationParams func configurationToUpdateParams(cfg *datastore.Configuration) repo.UpdateConfigurationParams { params := repo.UpdateConfigurationParams{ @@ -101,28 +125,7 @@ func configurationToUpdateParams(cfg *datastore.Configuration) repo.UpdateConfig // Handle storage policy based on type if cfg.StoragePolicy != nil { params.StoragePolicyType = common.StringToPgText(string(cfg.StoragePolicy.Type)) - - if cfg.StoragePolicy.Type == datastore.OnPrem && cfg.StoragePolicy.OnPrem != nil { - params.OnPremPath = common.NullStringToPgText(cfg.StoragePolicy.OnPrem.Path) - // Set S3 fields to NULL - params.S3Prefix = pgtype.Text{Valid: false} - params.S3Bucket = pgtype.Text{Valid: false} - params.S3AccessKey = pgtype.Text{Valid: false} - params.S3SecretKey = pgtype.Text{Valid: false} - params.S3Region = pgtype.Text{Valid: false} - params.S3SessionToken = pgtype.Text{Valid: false} - params.S3Endpoint = pgtype.Text{Valid: false} - } else if cfg.StoragePolicy.S3 != nil { // S3 - params.S3Prefix = common.NullStringToPgText(cfg.StoragePolicy.S3.Prefix) - params.S3Bucket = common.NullStringToPgText(cfg.StoragePolicy.S3.Bucket) - params.S3AccessKey = common.NullStringToPgText(cfg.StoragePolicy.S3.AccessKey) - params.S3SecretKey = common.NullStringToPgText(cfg.StoragePolicy.S3.SecretKey) - params.S3Region = common.NullStringToPgText(cfg.StoragePolicy.S3.Region) - params.S3SessionToken = common.NullStringToPgText(cfg.StoragePolicy.S3.SessionToken) - params.S3Endpoint = common.NullStringToPgText(cfg.StoragePolicy.S3.Endpoint) - // Set OnPrem to NULL - params.OnPremPath = pgtype.Text{Valid: false} - } + setStoragePolicyUpdateParams(¶ms, cfg.StoragePolicy) } // Handle retention policy @@ -133,6 +136,52 @@ func configurationToUpdateParams(cfg *datastore.Configuration) repo.UpdateConfig return params } +// setStoragePolicyUpdateParams populates storage fields on UpdateConfigurationParams, +// setting unused backends to NULL. +func setStoragePolicyUpdateParams(params *repo.UpdateConfigurationParams, sp *datastore.StoragePolicyConfiguration) { + nullText := pgtype.Text{Valid: false} + + // Default all to NULL + params.OnPremPath = nullText + params.S3Prefix = nullText + params.S3Bucket = nullText + params.S3AccessKey = nullText + params.S3SecretKey = nullText + params.S3Region = nullText + params.S3SessionToken = nullText + params.S3Endpoint = nullText + params.AzureAccountName = nullText + params.AzureAccountKey = nullText + params.AzureContainerName = nullText + params.AzureEndpoint = nullText + params.AzurePrefix = nullText + + switch sp.Type { + case datastore.OnPrem: + if sp.OnPrem != nil { + params.OnPremPath = common.NullStringToPgText(sp.OnPrem.Path) + } + case datastore.S3: + if sp.S3 != nil { + params.S3Prefix = common.NullStringToPgText(sp.S3.Prefix) + params.S3Bucket = common.NullStringToPgText(sp.S3.Bucket) + params.S3AccessKey = common.NullStringToPgText(sp.S3.AccessKey) + params.S3SecretKey = common.NullStringToPgText(sp.S3.SecretKey) + params.S3Region = common.NullStringToPgText(sp.S3.Region) + params.S3SessionToken = common.NullStringToPgText(sp.S3.SessionToken) + params.S3Endpoint = common.NullStringToPgText(sp.S3.Endpoint) + } + case datastore.AzureBlob: + if sp.AzureBlob != nil { + params.AzureAccountName = common.NullStringToPgText(sp.AzureBlob.AccountName) + params.AzureAccountKey = common.NullStringToPgText(sp.AzureBlob.AccountKey) + params.AzureContainerName = common.NullStringToPgText(sp.AzureBlob.ContainerName) + params.AzureEndpoint = common.NullStringToPgText(sp.AzureBlob.Endpoint) + params.AzurePrefix = common.NullStringToPgText(sp.AzureBlob.Prefix) + } + } +} + // rowToConfiguration converts LoadConfigurationRow to Configuration func rowToConfiguration(row repo.LoadConfigurationRow) *datastore.Configuration { cfg := &datastore.Configuration{ @@ -149,21 +198,12 @@ func rowToConfiguration(row repo.LoadConfigurationRow) *datastore.Configuration Type: datastore.StorageType(row.StoragePolicyType), } - if row.StoragePolicyType == string(datastore.OnPrem) { + switch datastore.StorageType(row.StoragePolicyType) { + case datastore.OnPrem: cfg.StoragePolicy.OnPrem = &datastore.OnPremStorage{ Path: common.PgTextToNullString(row.OnPremPath), } - // Create empty S3 storage to match legacy behavior - cfg.StoragePolicy.S3 = &datastore.S3Storage{ - Prefix: null.NewString("", false), - Bucket: null.NewString("", false), - AccessKey: null.NewString("", false), - SecretKey: null.NewString("", false), - Region: null.NewString("", false), - SessionToken: null.NewString("", false), - Endpoint: null.NewString("", false), - } - } else { + case datastore.S3: cfg.StoragePolicy.S3 = &datastore.S3Storage{ Prefix: common.PgTextToNullString(row.S3Prefix), Bucket: common.PgTextToNullString(row.S3Bucket), @@ -173,9 +213,13 @@ func rowToConfiguration(row repo.LoadConfigurationRow) *datastore.Configuration SessionToken: common.PgTextToNullString(row.S3SessionToken), Endpoint: common.PgTextToNullString(row.S3Endpoint), } - // Create empty OnPrem storage to match legacy behavior - cfg.StoragePolicy.OnPrem = &datastore.OnPremStorage{ - Path: null.NewString("", false), + case datastore.AzureBlob: + cfg.StoragePolicy.AzureBlob = &datastore.AzureBlobStorage{ + AccountName: common.PgTextToNullString(row.AzureAccountName), + AccountKey: common.PgTextToNullString(row.AzureAccountKey), + ContainerName: common.PgTextToNullString(row.AzureContainerName), + Endpoint: common.PgTextToNullString(row.AzureEndpoint), + Prefix: common.PgTextToNullString(row.AzurePrefix), } } @@ -198,23 +242,6 @@ func (s *Service) CreateConfiguration(ctx context.Context, cfg *datastore.Config return util.NewServiceError(http.StatusBadRequest, errors.New("configuration cannot be nil")) } - // Normalize storage policy - ensure empty S3 fields for OnPrem and vice versa - if cfg.StoragePolicy.Type == datastore.OnPrem { - cfg.StoragePolicy.S3 = &datastore.S3Storage{ - Prefix: null.NewString("", false), - Bucket: null.NewString("", false), - AccessKey: null.NewString("", false), - SecretKey: null.NewString("", false), - Region: null.NewString("", false), - SessionToken: null.NewString("", false), - Endpoint: null.NewString("", false), - } - } else { - cfg.StoragePolicy.OnPrem = &datastore.OnPremStorage{ - Path: null.NewString("", false), - } - } - params := configurationToCreateParams(cfg) err := s.repo.CreateConfiguration(ctx, params) @@ -247,23 +274,6 @@ func (s *Service) UpdateConfiguration(ctx context.Context, cfg *datastore.Config return util.NewServiceError(http.StatusBadRequest, errors.New("configuration cannot be nil")) } - // Normalize storage policy - ensure empty S3 fields for OnPrem and vice versa - if cfg.StoragePolicy.Type == datastore.OnPrem { - cfg.StoragePolicy.S3 = &datastore.S3Storage{ - Prefix: null.NewString("", false), - Bucket: null.NewString("", false), - AccessKey: null.NewString("", false), - SecretKey: null.NewString("", false), - Region: null.NewString("", false), - SessionToken: null.NewString("", false), - Endpoint: null.NewString("", false), - } - } else { - cfg.StoragePolicy.OnPrem = &datastore.OnPremStorage{ - Path: null.NewString("", false), - } - } - params := configurationToUpdateParams(cfg) result, err := s.repo.UpdateConfiguration(ctx, params) diff --git a/internal/configuration/load_configuration_test.go b/internal/configuration/load_configuration_test.go index d38e76537f..e6a265e0cd 100644 --- a/internal/configuration/load_configuration_test.go +++ b/internal/configuration/load_configuration_test.go @@ -51,6 +51,28 @@ func TestLoadConfiguration_OnPremStorage(t *testing.T) { require.Equal(t, "/var/convoy/storage", loaded.StoragePolicy.OnPrem.Path.String) } +func TestLoadConfiguration_AzureBlobStorage(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close() + + service := New(log.New("convoy", log.LevelInfo), db) + + seeded := seedConfiguration(t, db, datastore.AzureBlob) + + loaded, err := service.LoadConfiguration(ctx) + require.NoError(t, err) + require.NotNil(t, loaded) + require.Equal(t, seeded.UID, loaded.UID) + require.Equal(t, datastore.AzureBlob, loaded.StoragePolicy.Type) + require.NotNil(t, loaded.StoragePolicy.AzureBlob) + require.True(t, loaded.StoragePolicy.AzureBlob.AccountName.Valid) + require.Equal(t, "testaccount", loaded.StoragePolicy.AzureBlob.AccountName.String) + require.Equal(t, "test-container", loaded.StoragePolicy.AzureBlob.ContainerName.String) + // S3 and OnPrem should be nil + require.Nil(t, loaded.StoragePolicy.S3) + require.Nil(t, loaded.StoragePolicy.OnPrem) +} + func TestLoadConfiguration_NotFound(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close() @@ -103,9 +125,9 @@ func TestLoadConfiguration_VerifyS3FieldsReconstructed(t *testing.T) { require.True(t, loaded.StoragePolicy.S3.Prefix.Valid) require.True(t, loaded.StoragePolicy.S3.Endpoint.Valid) - // Verify OnPrem is empty struct (backward compatibility) - require.NotNil(t, loaded.StoragePolicy.OnPrem) - require.False(t, loaded.StoragePolicy.OnPrem.Path.Valid) + // OnPrem and Azure should be nil for S3 type + require.Nil(t, loaded.StoragePolicy.OnPrem) + require.Nil(t, loaded.StoragePolicy.AzureBlob) } func TestLoadConfiguration_VerifyOnPremFieldsReconstructed(t *testing.T) { @@ -126,11 +148,9 @@ func TestLoadConfiguration_VerifyOnPremFieldsReconstructed(t *testing.T) { require.NotNil(t, loaded.StoragePolicy.OnPrem) require.True(t, loaded.StoragePolicy.OnPrem.Path.Valid) - // Verify S3 is empty struct (backward compatibility) - require.NotNil(t, loaded.StoragePolicy.S3) - require.False(t, loaded.StoragePolicy.S3.Bucket.Valid) - require.False(t, loaded.StoragePolicy.S3.AccessKey.Valid) - require.False(t, loaded.StoragePolicy.S3.SecretKey.Valid) + // S3 and Azure should be nil for OnPrem type + require.Nil(t, loaded.StoragePolicy.S3) + require.Nil(t, loaded.StoragePolicy.AzureBlob) } func TestLoadConfiguration_VerifyBooleanConversion(t *testing.T) { diff --git a/internal/configuration/queries.sql b/internal/configuration/queries.sql index 52b9bf2576..215c7a4323 100644 --- a/internal/configuration/queries.sql +++ b/internal/configuration/queries.sql @@ -15,6 +15,11 @@ INSERT INTO convoy.configurations ( s3_region, s3_session_token, s3_endpoint, + azure_account_name, + azure_account_key, + azure_container_name, + azure_endpoint, + azure_prefix, retention_policy_policy, retention_policy_enabled ) VALUES ( @@ -30,6 +35,11 @@ INSERT INTO convoy.configurations ( @s3_region, @s3_session_token, @s3_endpoint, + @azure_account_name, + @azure_account_key, + @azure_container_name, + @azure_endpoint, + @azure_prefix, @retention_policy_policy, @retention_policy_enabled ); @@ -49,6 +59,11 @@ SELECT s3_region, s3_session_token, s3_endpoint, + azure_account_name, + azure_account_key, + azure_container_name, + azure_endpoint, + azure_prefix, retention_policy_policy, retention_policy_enabled, created_at, @@ -72,6 +87,11 @@ SET s3_region = @s3_region, s3_session_token = @s3_session_token, s3_endpoint = @s3_endpoint, + azure_account_name = @azure_account_name, + azure_account_key = @azure_account_key, + azure_container_name = @azure_container_name, + azure_endpoint = @azure_endpoint, + azure_prefix = @azure_prefix, retention_policy_policy = @retention_policy_policy, retention_policy_enabled = @retention_policy_enabled, updated_at = NOW() diff --git a/internal/configuration/repo/queries.sql.go b/internal/configuration/repo/queries.sql.go index 2c882f0a83..fdc478e8fb 100644 --- a/internal/configuration/repo/queries.sql.go +++ b/internal/configuration/repo/queries.sql.go @@ -27,6 +27,11 @@ INSERT INTO convoy.configurations ( s3_region, s3_session_token, s3_endpoint, + azure_account_name, + azure_account_key, + azure_container_name, + azure_endpoint, + azure_prefix, retention_policy_policy, retention_policy_enabled ) VALUES ( @@ -43,7 +48,12 @@ INSERT INTO convoy.configurations ( $11, $12, $13, - $14 + $14, + $15, + $16, + $17, + $18, + $19 ) ` @@ -60,6 +70,11 @@ type CreateConfigurationParams struct { S3Region pgtype.Text S3SessionToken pgtype.Text S3Endpoint pgtype.Text + AzureAccountName pgtype.Text + AzureAccountKey pgtype.Text + AzureContainerName pgtype.Text + AzureEndpoint pgtype.Text + AzurePrefix pgtype.Text RetentionPolicyPolicy pgtype.Text RetentionPolicyEnabled pgtype.Bool } @@ -80,6 +95,11 @@ func (q *Queries) CreateConfiguration(ctx context.Context, arg CreateConfigurati arg.S3Region, arg.S3SessionToken, arg.S3Endpoint, + arg.AzureAccountName, + arg.AzureAccountKey, + arg.AzureContainerName, + arg.AzureEndpoint, + arg.AzurePrefix, arg.RetentionPolicyPolicy, arg.RetentionPolicyEnabled, ) @@ -100,6 +120,11 @@ SELECT s3_region, s3_session_token, s3_endpoint, + azure_account_name, + azure_account_key, + azure_container_name, + azure_endpoint, + azure_prefix, retention_policy_policy, retention_policy_enabled, created_at, @@ -123,6 +148,11 @@ type LoadConfigurationRow struct { S3Region pgtype.Text S3SessionToken pgtype.Text S3Endpoint pgtype.Text + AzureAccountName pgtype.Text + AzureAccountKey pgtype.Text + AzureContainerName pgtype.Text + AzureEndpoint pgtype.Text + AzurePrefix pgtype.Text RetentionPolicyPolicy string RetentionPolicyEnabled bool CreatedAt pgtype.Timestamptz @@ -147,6 +177,11 @@ func (q *Queries) LoadConfiguration(ctx context.Context) (LoadConfigurationRow, &i.S3Region, &i.S3SessionToken, &i.S3Endpoint, + &i.AzureAccountName, + &i.AzureAccountKey, + &i.AzureContainerName, + &i.AzureEndpoint, + &i.AzurePrefix, &i.RetentionPolicyPolicy, &i.RetentionPolicyEnabled, &i.CreatedAt, @@ -170,10 +205,15 @@ SET s3_region = $9, s3_session_token = $10, s3_endpoint = $11, - retention_policy_policy = $12, - retention_policy_enabled = $13, + azure_account_name = $12, + azure_account_key = $13, + azure_container_name = $14, + azure_endpoint = $15, + azure_prefix = $16, + retention_policy_policy = $17, + retention_policy_enabled = $18, updated_at = NOW() -WHERE id = $14 AND deleted_at IS NULL +WHERE id = $19 AND deleted_at IS NULL ` type UpdateConfigurationParams struct { @@ -188,6 +228,11 @@ type UpdateConfigurationParams struct { S3Region pgtype.Text S3SessionToken pgtype.Text S3Endpoint pgtype.Text + AzureAccountName pgtype.Text + AzureAccountKey pgtype.Text + AzureContainerName pgtype.Text + AzureEndpoint pgtype.Text + AzurePrefix pgtype.Text RetentionPolicyPolicy pgtype.Text RetentionPolicyEnabled pgtype.Bool ID pgtype.Text @@ -206,6 +251,11 @@ func (q *Queries) UpdateConfiguration(ctx context.Context, arg UpdateConfigurati arg.S3Region, arg.S3SessionToken, arg.S3Endpoint, + arg.AzureAccountName, + arg.AzureAccountKey, + arg.AzureContainerName, + arg.AzureEndpoint, + arg.AzurePrefix, arg.RetentionPolicyPolicy, arg.RetentionPolicyEnabled, arg.ID, diff --git a/internal/configuration/update_configuration_test.go b/internal/configuration/update_configuration_test.go index 4c52633806..b95d51330f 100644 --- a/internal/configuration/update_configuration_test.go +++ b/internal/configuration/update_configuration_test.go @@ -71,8 +71,8 @@ func TestUpdateConfiguration_ChangeStorageFromS3ToOnPrem(t *testing.T) { require.NoError(t, err) require.Equal(t, datastore.OnPrem, loaded.StoragePolicy.Type) require.Equal(t, "/new/storage/path", loaded.StoragePolicy.OnPrem.Path.String) - // S3 fields should be cleared - require.False(t, loaded.StoragePolicy.S3.Bucket.Valid) + // S3 should be nil for OnPrem type + require.Nil(t, loaded.StoragePolicy.S3) } func TestUpdateConfiguration_ChangeStorageFromOnPremToS3(t *testing.T) { @@ -107,8 +107,8 @@ func TestUpdateConfiguration_ChangeStorageFromOnPremToS3(t *testing.T) { require.Equal(t, datastore.S3, loaded.StoragePolicy.Type) require.Equal(t, "new-s3-bucket", loaded.StoragePolicy.S3.Bucket.String) require.Equal(t, "eu-west-1", loaded.StoragePolicy.S3.Region.String) - // OnPrem path should be cleared - require.False(t, loaded.StoragePolicy.OnPrem.Path.Valid) + // OnPrem should be nil for S3 type + require.Nil(t, loaded.StoragePolicy.OnPrem) } func TestUpdateConfiguration_UpdateS3Credentials(t *testing.T) { @@ -282,11 +282,11 @@ func TestUpdateConfiguration_StorageNormalization_S3(t *testing.T) { err := service.UpdateConfiguration(ctx, cfg) require.NoError(t, err) - // Verify OnPrem was normalized (cleared) + // Verify OnPrem was cleared loaded, err := service.LoadConfiguration(ctx) require.NoError(t, err) require.Equal(t, datastore.S3, loaded.StoragePolicy.Type) - require.False(t, loaded.StoragePolicy.OnPrem.Path.Valid) + require.Nil(t, loaded.StoragePolicy.OnPrem) require.Equal(t, "normalized-bucket", loaded.StoragePolicy.S3.Bucket.String) } @@ -310,12 +310,11 @@ func TestUpdateConfiguration_StorageNormalization_OnPrem(t *testing.T) { err := service.UpdateConfiguration(ctx, cfg) require.NoError(t, err) - // Verify S3 was normalized (cleared) + // Verify S3 was cleared loaded, err := service.LoadConfiguration(ctx) require.NoError(t, err) require.Equal(t, datastore.OnPrem, loaded.StoragePolicy.Type) - require.False(t, loaded.StoragePolicy.S3.Bucket.Valid) - require.False(t, loaded.StoragePolicy.S3.AccessKey.Valid) + require.Nil(t, loaded.StoragePolicy.S3) require.Equal(t, "/normalized/path", loaded.StoragePolicy.OnPrem.Path.String) } @@ -374,6 +373,58 @@ func TestUpdateConfiguration_MultipleUpdates(t *testing.T) { require.Equal(t, "1000h", loaded.RetentionPolicy.Policy) } +func TestUpdateConfiguration_ChangeStorageFromS3ToAzureBlob(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close() + + service := New(log.New("convoy", log.LevelInfo), db) + + cfg := seedConfiguration(t, db, datastore.S3) + + // Switch to Azure Blob + cfg.StoragePolicy.Type = datastore.AzureBlob + cfg.StoragePolicy.AzureBlob = &datastore.AzureBlobStorage{ + AccountName: null.StringFrom("myaccount"), + AccountKey: null.StringFrom("mykey"), + ContainerName: null.StringFrom("exports"), + Endpoint: null.StringFrom("https://myaccount.blob.core.windows.net"), + } + + err := service.UpdateConfiguration(ctx, cfg) + require.NoError(t, err) + + loaded, err := service.LoadConfiguration(ctx) + require.NoError(t, err) + require.Equal(t, datastore.AzureBlob, loaded.StoragePolicy.Type) + require.NotNil(t, loaded.StoragePolicy.AzureBlob) + require.Equal(t, "myaccount", loaded.StoragePolicy.AzureBlob.AccountName.String) + require.Equal(t, "exports", loaded.StoragePolicy.AzureBlob.ContainerName.String) + // S3 and OnPrem should be nil + require.Nil(t, loaded.StoragePolicy.S3) + require.Nil(t, loaded.StoragePolicy.OnPrem) +} + +func TestUpdateConfiguration_UpdateAzureBlobCredentials(t *testing.T) { + db, ctx := setupTestDB(t) + defer db.Close() + + service := New(log.New("convoy", log.LevelInfo), db) + + cfg := seedConfiguration(t, db, datastore.AzureBlob) + + // Update Azure credentials + cfg.StoragePolicy.AzureBlob.AccountKey = null.StringFrom("updated-key") + cfg.StoragePolicy.AzureBlob.ContainerName = null.StringFrom("updated-container") + + err := service.UpdateConfiguration(ctx, cfg) + require.NoError(t, err) + + loaded, err := service.LoadConfiguration(ctx) + require.NoError(t, err) + require.Equal(t, "updated-key", loaded.StoragePolicy.AzureBlob.AccountKey.String) + require.Equal(t, "updated-container", loaded.StoragePolicy.AzureBlob.ContainerName.String) +} + func TestUpdateConfiguration_VerifyNoRowsAffectedError(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close() diff --git a/internal/dataplane/worker.go b/internal/dataplane/worker.go index cb11729920..04f632fccb 100644 --- a/internal/dataplane/worker.go +++ b/internal/dataplane/worker.go @@ -14,7 +14,9 @@ import ( "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/database/postgres" "github.com/frain-dev/convoy/datastore" - batch_retries "github.com/frain-dev/convoy/internal/batch_retries" + "github.com/frain-dev/convoy/datastore/cached" + "github.com/frain-dev/convoy/internal/backup_jobs" + "github.com/frain-dev/convoy/internal/batch_retries" "github.com/frain-dev/convoy/internal/configuration" "github.com/frain-dev/convoy/internal/delivery_attempts" "github.com/frain-dev/convoy/internal/endpoints" @@ -23,7 +25,10 @@ import ( "github.com/frain-dev/convoy/internal/filters" "github.com/frain-dev/convoy/internal/meta_events" "github.com/frain-dev/convoy/internal/organisations" + "github.com/frain-dev/convoy/internal/pkg/backup_collector" "github.com/frain-dev/convoy/internal/pkg/billing" + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" + "github.com/frain-dev/convoy/internal/pkg/exporter" "github.com/frain-dev/convoy/internal/pkg/fflag" "github.com/frain-dev/convoy/internal/pkg/keys" "github.com/frain-dev/convoy/internal/pkg/limiter" @@ -49,8 +54,9 @@ import ( ) type Worker struct { - consumer *worker.Consumer - logger log.Logger + consumer *worker.Consumer + backupCollector *backup_collector.BackupCollector // nil if CDC backup disabled + logger log.Logger } // NewWorker initializes all worker components and returns a Worker instance. @@ -112,16 +118,17 @@ func NewWorker(ctx context.Context, opts RuntimeOpts, cfg config.Configuration) } } - projectRepo := projects.New(opts.Logger, opts.DB) + projectRepo := cached.NewCachedProjectRepository(projects.New(opts.Logger, opts.DB), opts.Cache, 5*time.Minute, lo) metaEventRepo := meta_events.New(opts.Logger, opts.DB) - endpointRepo := endpoints.New(opts.Logger, opts.DB) + endpointRepo := cached.NewCachedEndpointRepository(endpoints.New(opts.Logger, opts.DB), opts.Cache, 2*time.Minute, lo) eventRepo := events.New(opts.Logger, opts.DB) jobRepo := postgres.NewJobRepo(opts.DB) eventDeliveryRepo := event_deliveries.New(opts.Logger, opts.DB) - subRepo := subscriptions.New(opts.Logger, opts.DB) + subRepo := cached.NewCachedSubscriptionRepository(subscriptions.New(opts.Logger, opts.DB), opts.Cache, 30*time.Second, lo) configRepo := configuration.New(opts.Logger, opts.DB) attemptRepo := delivery_attempts.New(opts.Logger, opts.DB) - filterRepo := filters.New(opts.Logger, opts.DB) + backupJobRepo := backup_jobs.New(opts.Logger, opts.DB) + filterRepo := cached.NewCachedFilterRepository(filters.New(opts.Logger, opts.DB), opts.Cache, 2*time.Minute, lo) batchRetryRepo := batch_retries.New(lo, opts.DB) rd, err := rdb.NewClientFromRedisConfig(cfg.Redis) @@ -136,9 +143,11 @@ func NewWorker(ctx context.Context, opts RuntimeOpts, cfg config.Configuration) counter := &telemetry.EventsCounter{} pb := telemetry.NewposthogBackend() + defer pb.Close() mb := telemetry.NewmixpanelBackend() + defer mb.Close() - configuration, err := configRepo.LoadConfiguration(context.Background()) + loadConfiguration, err := configRepo.LoadConfiguration(context.Background()) if err != nil { return nil, fmt.Errorf("failed to initialize configuration: %w", err) } @@ -159,7 +168,7 @@ func NewWorker(ctx context.Context, opts RuntimeOpts, cfg config.Configuration) } featureFlag := fflag.NewFFlag(cfg.EnableFeatureFlag) - newTelemetry := telemetry.NewTelemetry(lo, configuration, + newTelemetry := telemetry.NewTelemetry(lo, loadConfiguration, telemetry.OptionTracker(counter), telemetry.OptionBackend(pb), telemetry.OptionBackend(mb)) @@ -335,9 +344,13 @@ func NewWorker(ctx context.Context, opts RuntimeOpts, cfg config.Configuration) if opts.Licenser.RetentionPolicy() { consumer.RegisterHandlers(convoy.RetentionPolicies, task.RetentionPolicies(rd.Client(), ret, lo), nil) - consumer.RegisterHandlers(convoy.BackupProjectData, task.BackupProjectData(configRepo, projectRepo, eventRepo, eventDeliveryRepo, attemptRepo, rd.Client(), lo), nil) + consumer.RegisterHandlers(convoy.EnqueueBackupJobs, task.EnqueueBackupJobs(configRepo, backupJobRepo, lo), nil) + consumer.RegisterHandlers(convoy.ProcessBackupJob, task.ProcessBackupJob(configRepo, eventRepo, eventDeliveryRepo, attemptRepo, backupJobRepo, lo), nil) } + // ManualBackupJob is always registered — it bypasses CDC and retention checks. + consumer.RegisterHandlers(convoy.ManualBackupJob, task.ManualBackup(configRepo, eventRepo, eventDeliveryRepo, attemptRepo, lo), nil) + matchSubscriptionsDeps := task.MatchSubscriptionsDeps{ Channels: channels, EndpointRepo: endpointRepo, @@ -371,9 +384,6 @@ func NewWorker(ctx context.Context, opts RuntimeOpts, cfg config.Configuration) consumer.RegisterHandlers(convoy.MetaEventProcessor, task.ProcessMetaEvent(projectRepo, metaEventRepo, dispatcher, opts.TracerBackend, lo), nil) consumer.RegisterHandlers(convoy.DeleteArchivedTasksProcessor, task.DeleteArchivedTasks(opts.Queue, rd, lo), nil) - //nolint:gocritic - // consumer.RegisterHandlers(convoy.RefreshMetricsMaterializedViews, task.RefreshMetricsMaterializedViews(opts.DB, rd), nil) - consumer.RegisterHandlers(convoy.BatchRetryProcessor, task.ProcessBatchRetry(batchRetryRepo, eventDeliveryRepo, opts.Queue, lo), nil) bulkOnboardDeps := task.BulkOnboardDeps{ @@ -399,9 +409,31 @@ func NewWorker(ctx context.Context, opts RuntimeOpts, cfg config.Configuration) return nil, fmt.Errorf("failed to register queue metrics: %w", err) } + // Optionally start the CDC-based backup collector + var collector *backup_collector.BackupCollector + lo.Info(fmt.Sprintf("CDC backup config: enabled=%v, retention=%v", cfg.RetentionPolicy.CDCBackupEnabled, cfg.RetentionPolicy.IsRetentionPolicyEnabled)) + if cfg.RetentionPolicy.CDCBackupEnabled && cfg.RetentionPolicy.IsRetentionPolicyEnabled { + blobStoreClient, blobErr := blobstore.NewBlobStoreClient(loadConfiguration.StoragePolicy, lo) + if blobErr != nil { + return nil, fmt.Errorf("failed to create blob store for CDC backup: %w", blobErr) + } + + flushInterval := exporter.ParseBackupInterval(cfg.RetentionPolicy.BackupInterval) + + // ReplicationDSN connects directly to Postgres (bypassing pgbouncer) + // for the WAL replication protocol. Falls back to normal DSN if not set. + replDSN := cfg.RetentionPolicy.ReplicationDSN + if replDSN == "" { + replDSN = cfg.Database.BuildDsn() + } + + collector = backup_collector.NewBackupCollector(opts.DB.GetConn(), replDSN, blobStoreClient, flushInterval, lo) + } + return &Worker{ - consumer: consumer, - logger: lo, + consumer: consumer, + backupCollector: collector, + logger: lo, }, nil } @@ -411,12 +443,25 @@ func (w *Worker) Run(ctx context.Context, workerReady chan struct{}) error { } w.logger.Printf("Starting Convoy Consumer Pool") + // Start CDC backup collector if enabled + if w.backupCollector != nil { + if err := w.backupCollector.Start(ctx); err != nil { + w.logger.Error(fmt.Sprintf("failed to start backup collector: %v", err)) + // Non-fatal — worker can still process events without CDC backup + } + } + if workerReady != nil { close(workerReady) } <-ctx.Done() w.logger.Printf("Context canceled, stopping Convoy Consumer Pool...") + + if w.backupCollector != nil { + w.backupCollector.Stop(ctx) + } + w.consumer.Stop() w.logger.Printf("Convoy Consumer Pool stopped") diff --git a/internal/delivery_attempts/export_records_test.go b/internal/delivery_attempts/export_records_test.go index c187c84eef..0c7347344f 100644 --- a/internal/delivery_attempts/export_records_test.go +++ b/internal/delivery_attempts/export_records_test.go @@ -1,6 +1,7 @@ package delivery_attempts import ( + "bufio" "bytes" "encoding/json" "testing" @@ -13,19 +14,38 @@ import ( log "github.com/frain-dev/convoy/pkg/logger" ) +// parseJSONL parses JSONL (newline-delimited JSON) into a slice of maps. +func parseJSONL(t *testing.T, data []byte) []map[string]interface{} { + t.Helper() + var results []map[string]interface{} + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var record map[string]interface{} + err := json.Unmarshal(line, &record) + require.NoError(t, err, "each JSONL line must be valid JSON") + results = append(results, record) + } + require.NoError(t, scanner.Err()) + return results +} + func TestExportRecords_EmptyResult(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close() - project := seedTestData(t, db, ctx) + _ = seedTestData(t, db, ctx) service := New(log.New("convoy", log.LevelInfo), db) // Create a buffer to write exported data var buf bytes.Buffer - // Export with a future date (should return empty) + // Export with a future date as end (should return empty since no data seeded) futureDate := time.Now().Add(24 * time.Hour) - count, err := service.ExportRecords(ctx, project.UID, futureDate, &buf) + count, err := service.ExportRecords(ctx, time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), futureDate, &buf) require.NoError(t, err) require.Equal(t, int64(0), count) @@ -62,15 +82,13 @@ func TestExportRecords_WithData(t *testing.T) { // Export all attempts var buf bytes.Buffer futureDate := time.Now().Add(24 * time.Hour) - count, err := service.ExportRecords(ctx, project.UID, futureDate, &buf) + count, err := service.ExportRecords(ctx, time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), futureDate, &buf) require.NoError(t, err) require.Equal(t, int64(5), count) - // Verify JSON structure - var exported []map[string]interface{} - err = json.Unmarshal(buf.Bytes(), &exported) - require.NoError(t, err) + // Verify JSONL structure + exported := parseJSONL(t, buf.Bytes()) require.Len(t, exported, 5) // Verify all UIDs are present @@ -86,75 +104,54 @@ func TestExportRecords_WithData(t *testing.T) { } } -func TestExportRecords_ProjectIsolation(t *testing.T) { +func TestExportRecords_TimeFiltering(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close() + project := seedTestData(t, db, ctx) + endpoint := seedEndpoint(t, db, ctx, project) + eventDelivery := seedEventDelivery(t, db, ctx, project, endpoint) service := New(log.New("convoy", log.LevelInfo), db) - // Create two projects with delivery attempts - project1 := seedTestData(t, db, ctx) - endpoint1 := seedEndpoint(t, db, ctx, project1) - eventDelivery1 := seedEventDelivery(t, db, ctx, project1, endpoint1) - - project2 := seedTestData(t, db, ctx) - endpoint2 := seedEndpoint(t, db, ctx, project2) - eventDelivery2 := seedEventDelivery(t, db, ctx, project2, endpoint2) - - // Create 3 attempts for project1 + // Create 3 attempts for i := 0; i < 3; i++ { attempt := &datastore.DeliveryAttempt{ UID: ulid.Make().String(), URL: "https://example.com/webhook", Method: "POST", APIVersion: "2023.12.25", - EndpointID: endpoint1.UID, - EventDeliveryId: eventDelivery1.UID, - ProjectId: project1.UID, - Status: true, - } - err := service.CreateDeliveryAttempt(ctx, attempt) - require.NoError(t, err) - } - - // Create 2 attempts for project2 - for i := 0; i < 2; i++ { - attempt := &datastore.DeliveryAttempt{ - UID: ulid.Make().String(), - URL: "https://example.com/webhook", - Method: "POST", - APIVersion: "2023.12.25", - EndpointID: endpoint2.UID, - EventDeliveryId: eventDelivery2.UID, - ProjectId: project2.UID, + EndpointID: endpoint.UID, + EventDeliveryId: eventDelivery.UID, + ProjectId: project.UID, Status: true, } err := service.CreateDeliveryAttempt(ctx, attempt) require.NoError(t, err) + time.Sleep(10 * time.Millisecond) // Small delay to ensure different timestamps } - // Export project1 only + // Export with past date as end (should return 0) var buf bytes.Buffer - futureDate := time.Now().Add(24 * time.Hour) - count, err := service.ExportRecords(ctx, project1.UID, futureDate, &buf) + pastDate := time.Now().Add(-1 * time.Hour) + count, err := service.ExportRecords(ctx, time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), pastDate, &buf) require.NoError(t, err) - require.Equal(t, int64(3), count, "Should only export project1's attempts") + require.Equal(t, int64(0), count) + require.Empty(t, buf.String()) + + // Export with future date as end (should return all 3) + buf.Reset() + futureDate := time.Now().Add(24 * time.Hour) + count, err = service.ExportRecords(ctx, time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), futureDate, &buf) - // Verify no project2 data is included - var exported []map[string]interface{} - err = json.Unmarshal(buf.Bytes(), &exported) require.NoError(t, err) - require.Len(t, exported, 3) + require.Equal(t, int64(3), count) - for _, record := range exported { - projectID, ok := record["project_id"].(string) - require.True(t, ok) - require.Equal(t, project1.UID, projectID, "All records should belong to project1") - } + exported := parseJSONL(t, buf.Bytes()) + require.Len(t, exported, 3) } -func TestExportRecords_TimeFiltering(t *testing.T) { +func TestExportRecords_TimeWindow(t *testing.T) { db, ctx := setupTestDB(t) defer db.Close() @@ -163,8 +160,8 @@ func TestExportRecords_TimeFiltering(t *testing.T) { eventDelivery := seedEventDelivery(t, db, ctx, project, endpoint) service := New(log.New("convoy", log.LevelInfo), db) - // Create 3 attempts - for i := 0; i < 3; i++ { + // Create 5 attempts + for i := 0; i < 5; i++ { attempt := &datastore.DeliveryAttempt{ UID: ulid.Make().String(), URL: "https://example.com/webhook", @@ -177,28 +174,29 @@ func TestExportRecords_TimeFiltering(t *testing.T) { } err := service.CreateDeliveryAttempt(ctx, attempt) require.NoError(t, err) - time.Sleep(10 * time.Millisecond) // Small delay to ensure different timestamps } - // Export with past date (should return 0) + // Export with window [1h ago, now+1h) — should include all var buf bytes.Buffer - pastDate := time.Now().Add(-1 * time.Hour) - count, err := service.ExportRecords(ctx, project.UID, pastDate, &buf) - + start := time.Now().Add(-1 * time.Hour) + end := time.Now().Add(1 * time.Hour) + count, err := service.ExportRecords(ctx, start, end, &buf) require.NoError(t, err) - require.Equal(t, int64(0), count) - require.Empty(t, buf.String()) + require.Equal(t, int64(5), count) - // Export with future date (should return all 3) - buf.Reset() - futureDate := time.Now().Add(24 * time.Hour) - count, err = service.ExportRecords(ctx, project.UID, futureDate, &buf) + exported := parseJSONL(t, buf.Bytes()) + require.Len(t, exported, 5) - require.NoError(t, err) - require.Equal(t, int64(3), count) + // Each record should have a uid field + for _, record := range exported { + _, ok := record["uid"].(string) + require.True(t, ok, "each record should have uid") + } - var exported []map[string]interface{} - err = json.Unmarshal(buf.Bytes(), &exported) + // Export with window that excludes all: [2h ago, 1h ago) + buf.Reset() + count, err = service.ExportRecords(ctx, time.Now().Add(-2*time.Hour), time.Now().Add(-1*time.Hour), &buf) require.NoError(t, err) - require.Len(t, exported, 3) + require.Equal(t, int64(0), count) + require.Empty(t, buf.String()) } diff --git a/internal/delivery_attempts/impl.go b/internal/delivery_attempts/impl.go index 392c813daa..864e74921c 100644 --- a/internal/delivery_attempts/impl.go +++ b/internal/delivery_attempts/impl.go @@ -213,34 +213,39 @@ func (s *Service) GetFailureAndSuccessCounts(ctx context.Context, lookBackDurati return resultsMap, nil } -func (s *Service) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { - // Export delivery attempts to JSON format for backup/archival purposes - // This uses batched queries to avoid loading all records into memory at once - +// ExportRecords exports delivery attempts to a writer as JSONL (one JSON object per line). +// It uses a REPEATABLE READ transaction for snapshot consistency across batches. +func (s *Service) ExportRecords(ctx context.Context, start, end time.Time, w io.Writer) (int64, error) { const ( countQuery = ` SELECT COUNT(*) FROM convoy.delivery_attempts WHERE deleted_at IS NULL - AND project_id = $1 - AND created_at < $2 + AND created_at < $1 + AND created_at >= $2 ` exportQuery = ` - SELECT TO_JSONB(da) - 'id' || JSONB_BUILD_OBJECT('uid', da.id) AS json_output + SELECT da.id, TO_JSONB(da) - 'id' || JSONB_BUILD_OBJECT('uid', da.id) AS json_output FROM convoy.delivery_attempts AS da WHERE deleted_at IS NULL - AND project_id = $1 - AND created_at < $2 + AND created_at < $1 + AND created_at >= $2 AND (id > $3 OR $3 = '') ORDER BY id ASC LIMIT $4 ` ) - // Get total count + // Begin REPEATABLE READ transaction for snapshot consistency + tx, err := s.db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return 0, fmt.Errorf("begin snapshot tx: %w", err) + } + defer tx.Rollback(ctx) //nolint:errcheck + var count int64 - err := s.db.QueryRow(ctx, countQuery, projectID, createdAt).Scan(&count) + err = tx.QueryRow(ctx, countQuery, end, start).Scan(&count) if err != nil { return 0, util.NewServiceError(500, fmt.Errorf("failed to count records: %w", err)) } @@ -249,55 +254,40 @@ func (s *Service) ExportRecords(ctx context.Context, projectID string, createdAt return 0, nil } - // Write opening bracket for JSON array - if _, err := w.Write([]byte(`[`)); err != nil { - return 0, util.NewServiceError(500, fmt.Errorf("failed to write opening bracket: %w", err)) - } - var ( - batchSize = 3000 - numDocs int64 - lastID string - firstBatch = true + batchSize = 3000 + numDocs int64 + lastID string ) - // Process in batches for { - rows, err := s.db.Query(ctx, exportQuery, projectID, createdAt, lastID, batchSize) + rows, err := tx.Query(ctx, exportQuery, end, start, lastID, batchSize) if err != nil { return 0, util.NewServiceError(500, fmt.Errorf("failed to query batch: %w", err)) } batchCount := 0 - var record []byte + var ( + id string + record []byte + ) for rows.Next() { - if err := rows.Scan(&record); err != nil { + if err := rows.Scan(&id, &record); err != nil { rows.Close() return 0, util.NewServiceError(500, fmt.Errorf("failed to scan record: %w", err)) } - // Add comma before all records except the first - if !firstBatch || batchCount > 0 { - if _, err := w.Write([]byte(`,`)); err != nil { - rows.Close() - return 0, util.NewServiceError(500, fmt.Errorf("failed to write comma: %w", err)) - } - } - if _, err := w.Write(record); err != nil { rows.Close() return 0, util.NewServiceError(500, fmt.Errorf("failed to write record: %w", err)) } - - // Extract UID for pagination - var recordData map[string]interface{} - if err := json.Unmarshal(record, &recordData); err == nil { - if uid, ok := recordData["uid"].(string); ok { - lastID = uid - } + if _, err := w.Write([]byte("\n")); err != nil { + rows.Close() + return 0, util.NewServiceError(500, fmt.Errorf("failed to write newline: %w", err)) } + lastID = id batchCount++ numDocs++ } @@ -308,17 +298,9 @@ func (s *Service) ExportRecords(ctx context.Context, projectID string, createdAt return 0, util.NewServiceError(500, fmt.Errorf("error during row iteration: %w", err)) } - // If we got fewer records than batch size, we're done if batchCount < batchSize { break } - - firstBatch = false - } - - // Write closing bracket for JSON array - if _, err := w.Write([]byte(`]`)); err != nil { - return 0, util.NewServiceError(500, fmt.Errorf("failed to write closing bracket: %w", err)) } return numDocs, nil diff --git a/internal/event_deliveries/impl.go b/internal/event_deliveries/impl.go index 41667e0767..3c954992ec 100644 --- a/internal/event_deliveries/impl.go +++ b/internal/event_deliveries/impl.go @@ -546,23 +546,28 @@ func (s *Service) LoadEventDeliveriesIntervals(ctx context.Context, projectID st return intervals, nil } -func (s *Service) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { - count, err := s.repo.CountExportedEventDeliveries(ctx, repo.CountExportedEventDeliveriesParams{ - ProjectID: common.StringToPgTextNullable(projectID), - CreatedAt: common.TimeToPgTimestamptz(createdAt), - Cursor: common.StringToPgText(""), +// ExportRecords exports event deliveries to a writer as JSONL (one JSON object per line). +// It uses a REPEATABLE READ transaction for snapshot consistency across batches. +func (s *Service) ExportRecords(ctx context.Context, start, end time.Time, w io.Writer) (int64, error) { + // Begin REPEATABLE READ transaction for snapshot consistency + tx, err := s.db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return 0, fmt.Errorf("begin snapshot tx: %w", err) + } + defer tx.Rollback(ctx) //nolint:errcheck + + txRepo := repo.New(tx) + + count, err := txRepo.CountExportedEventDeliveries(ctx, repo.CountExportedEventDeliveriesParams{ + CreatedAtEnd: common.TimeToPgTimestamptz(end), + CreatedAtStart: common.TimeToPgTimestamptz(start), }) if err != nil { return 0, err } count64 := common.PgInt8ToInt64(count) - if count64 == 0 { - _, err = w.Write([]byte(`[]`)) - if err != nil { - return 0, err - } return 0, nil } @@ -573,48 +578,32 @@ func (s *Service) ExportRecords(ctx context.Context, projectID string, createdAt lastID string ) - _, err = w.Write([]byte(`[`)) - if err != nil { - return 0, err - } - - isFirstRecord := true - for i := 0; i < numBatches; i++ { params := repo.ExportEventDeliveriesParams{ - ProjectID: common.StringToPgTextNullable(projectID), - CreatedAt: common.TimeToPgTimestamptz(createdAt), - Cursor: common.StringToPgText(lastID), - PageLimit: pgtype.Int8{Int64: int64(batchSize), Valid: true}, + CreatedAtEnd: common.TimeToPgTimestamptz(end), + CreatedAtStart: common.TimeToPgTimestamptz(start), + Cursor: common.StringToPgText(lastID), + PageLimit: pgtype.Int8{Int64: int64(batchSize), Valid: true}, } - rows, exportErr := s.repo.ExportEventDeliveries(ctx, params) + rows, exportErr := txRepo.ExportEventDeliveries(ctx, params) if exportErr != nil { return 0, fmt.Errorf("failed to query batch %d: %w", i, exportErr) } for _, row := range rows { - if !isFirstRecord { - if _, writeErr := w.Write([]byte(`,`)); writeErr != nil { - return 0, writeErr - } - } - isFirstRecord = false - if _, writeErr := w.Write(row.JsonOutput); writeErr != nil { return 0, writeErr } + if _, writeErr := w.Write([]byte("\n")); writeErr != nil { + return 0, writeErr + } numDocs++ lastID = row.ID } } - _, err = w.Write([]byte(`]`)) - if err != nil { - return 0, err - } - return numDocs, nil } diff --git a/internal/event_deliveries/impl_test.go b/internal/event_deliveries/impl_test.go index 9182c4291e..8d79dfc0c7 100644 --- a/internal/event_deliveries/impl_test.go +++ b/internal/event_deliveries/impl_test.go @@ -1123,26 +1123,54 @@ func TestExportRecords(t *testing.T) { } var buf bytes.Buffer - // Export uses created_at < @created_at, so pass a future time to include recent deliveries - count, err := service.ExportRecords(ctx, project.UID, time.Now().Add(1*time.Hour), &buf) + // Export uses created_at < end AND created_at >= start, so pass epoch as start and future time as end + count, err := service.ExportRecords(ctx, time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), time.Now().Add(1*time.Hour), &buf) require.NoError(t, err) require.GreaterOrEqual(t, count, int64(5)) - // Verify valid JSON array - var records []json.RawMessage - err = json.Unmarshal(buf.Bytes(), &records) + // Verify valid JSONL (one JSON object per line) + lines := bytes.Split(bytes.TrimSpace(buf.Bytes()), []byte("\n")) + require.GreaterOrEqual(t, len(lines), 5) + for _, line := range lines { + var record json.RawMessage + err = json.Unmarshal(line, &record) + require.NoError(t, err) + } + }) + + t.Run("Empty_with_past_cutoff", func(t *testing.T) { + // Export with end in the past should return 0 records + var buf bytes.Buffer + count, err := service.ExportRecords(ctx, time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), time.Now().Add(-24*time.Hour), &buf) require.NoError(t, err) - require.GreaterOrEqual(t, len(records), 5) + require.Equal(t, int64(0), count) + require.Empty(t, buf.String()) }) - t.Run("Empty", func(t *testing.T) { - emptyProject := seedTestProject(t, db) + t.Run("TimeWindow", func(t *testing.T) { + // Create more deliveries to ensure we have records + for i := 0; i < 3; i++ { + d := createTestEventDelivery(t, project.UID, event.UID, endpoint.UID, sub.UID) + require.NoError(t, service.CreateEventDelivery(ctx, d)) + } + // Export with a narrow window: [1 hour ago, now+1h) + // Should include all recently created deliveries var buf bytes.Buffer - count, err := service.ExportRecords(ctx, emptyProject.UID, time.Now().Add(1*time.Hour), &buf) + start := time.Now().Add(-1 * time.Hour) + end := time.Now().Add(1 * time.Hour) + count, err := service.ExportRecords(ctx, start, end, &buf) + require.NoError(t, err) + require.GreaterOrEqual(t, count, int64(3)) + + lines := bytes.Split(bytes.TrimSpace(buf.Bytes()), []byte("\n")) + require.GreaterOrEqual(t, len(lines), 3) + + // Export with a window that excludes all records: [2h ago, 1h ago) + buf.Reset() + count, err = service.ExportRecords(ctx, time.Now().Add(-2*time.Hour), time.Now().Add(-1*time.Hour), &buf) require.NoError(t, err) require.Equal(t, int64(0), count) - require.Equal(t, "[]", buf.String()) }) } diff --git a/internal/event_deliveries/queries.sql b/internal/event_deliveries/queries.sql index c5c8af3d06..01082c6d68 100644 --- a/internal/event_deliveries/queries.sql +++ b/internal/event_deliveries/queries.sql @@ -380,8 +380,8 @@ SELECT ed.id, 'updated_at', ed.updated_at ) AS json_output FROM convoy.event_deliveries AS ed -WHERE project_id = @project_id - AND created_at < @created_at +WHERE created_at < @created_at_end + AND created_at >= @created_at_start AND (id > @cursor OR @cursor = '') AND deleted_at IS NULL ORDER BY id @@ -389,7 +389,6 @@ LIMIT @page_limit; -- name: CountExportedEventDeliveries :one SELECT COUNT(*) AS count FROM convoy.event_deliveries -WHERE project_id = @project_id - AND created_at < @created_at - AND (id > @cursor OR @cursor = '') +WHERE created_at < @created_at_end + AND created_at >= @created_at_start AND deleted_at IS NULL; diff --git a/internal/event_deliveries/repo/queries.sql.go b/internal/event_deliveries/repo/queries.sql.go index 2aa1ea0c2d..63374e0c71 100644 --- a/internal/event_deliveries/repo/queries.sql.go +++ b/internal/event_deliveries/repo/queries.sql.go @@ -85,20 +85,18 @@ func (q *Queries) CountEventDeliveries(ctx context.Context, arg CountEventDelive const countExportedEventDeliveries = `-- name: CountExportedEventDeliveries :one SELECT COUNT(*) AS count FROM convoy.event_deliveries -WHERE project_id = $1 - AND created_at < $2 - AND (id > $3 OR $3 = '') +WHERE created_at < $1 + AND created_at >= $2 AND deleted_at IS NULL ` type CountExportedEventDeliveriesParams struct { - ProjectID pgtype.Text - CreatedAt pgtype.Timestamptz - Cursor pgtype.Text + CreatedAtEnd pgtype.Timestamptz + CreatedAtStart pgtype.Timestamptz } func (q *Queries) CountExportedEventDeliveries(ctx context.Context, arg CountExportedEventDeliveriesParams) (pgtype.Int8, error) { - row := q.db.QueryRow(ctx, countExportedEventDeliveries, arg.ProjectID, arg.CreatedAt, arg.Cursor) + row := q.db.QueryRow(ctx, countExportedEventDeliveries, arg.CreatedAtEnd, arg.CreatedAtStart) var count pgtype.Int8 err := row.Scan(&count) return count, err @@ -195,8 +193,8 @@ SELECT ed.id, 'updated_at', ed.updated_at ) AS json_output FROM convoy.event_deliveries AS ed -WHERE project_id = $1 - AND created_at < $2 +WHERE created_at < $1 + AND created_at >= $2 AND (id > $3 OR $3 = '') AND deleted_at IS NULL ORDER BY id @@ -204,10 +202,10 @@ LIMIT $4 ` type ExportEventDeliveriesParams struct { - ProjectID pgtype.Text - CreatedAt pgtype.Timestamptz - Cursor pgtype.Text - PageLimit pgtype.Int8 + CreatedAtEnd pgtype.Timestamptz + CreatedAtStart pgtype.Timestamptz + Cursor pgtype.Text + PageLimit pgtype.Int8 } type ExportEventDeliveriesRow struct { @@ -219,12 +217,7 @@ type ExportEventDeliveriesRow struct { // Group 7: Export Operations // ============================================================================ func (q *Queries) ExportEventDeliveries(ctx context.Context, arg ExportEventDeliveriesParams) ([]ExportEventDeliveriesRow, error) { - rows, err := q.db.Query(ctx, exportEventDeliveries, - arg.ProjectID, - arg.CreatedAt, - arg.Cursor, - arg.PageLimit, - ) + rows, err := q.db.Query(ctx, exportEventDeliveries, arg.CreatedAtEnd, arg.CreatedAtStart, arg.Cursor, arg.PageLimit) if err != nil { return nil, err } diff --git a/internal/events/impl.go b/internal/events/impl.go index 521c49a622..e7fd8d87aa 100644 --- a/internal/events/impl.go +++ b/internal/events/impl.go @@ -553,26 +553,29 @@ func (s *Service) CopyRows(ctx context.Context, projectID string, interval int) return tx.Commit(ctx) } -// ExportRecords exports events to a writer as a JSON array -// It processes records in batches to avoid memory issues with large datasets -func (s *Service) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { - // Count total exportable events (with empty cursor for initial count) - count, err := s.repo.CountExportedEvents(ctx, repo.CountExportedEventsParams{ - ProjectID: common.StringToPgTextNullable(projectID), - CreatedAt: common.TimeToPgTimestamptz(createdAt), - Cursor: common.StringToPgText(""), // Use Filter to keep Valid=true for SQL comparison +// ExportRecords exports events to a writer as JSONL (one JSON object per line). +// It uses a REPEATABLE READ transaction for snapshot consistency across batches. +func (s *Service) ExportRecords(ctx context.Context, start, end time.Time, w io.Writer) (int64, error) { + // Begin REPEATABLE READ transaction for snapshot consistency + tx, err := s.db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return 0, fmt.Errorf("begin snapshot tx: %w", err) + } + defer tx.Rollback(ctx) //nolint:errcheck + + txRepo := repo.New(tx) + + // Count total exportable events in the window [start, end) + count, err := txRepo.CountExportedEvents(ctx, repo.CountExportedEventsParams{ + CreatedAtEnd: common.TimeToPgTimestamptz(end), + CreatedAtStart: common.TimeToPgTimestamptz(start), }) if err != nil { return 0, err } count64 := common.PgInt8ToInt64(count) - - if count64 == 0 { // nothing to export, write empty JSON array - _, err = w.Write([]byte(`[]`)) - if err != nil { - return 0, err - } + if count64 == 0 { return 0, nil } @@ -583,58 +586,32 @@ func (s *Service) ExportRecords(ctx context.Context, projectID string, createdAt lastID string ) - // Write opening bracket for JSON array - _, err = w.Write([]byte(`[`)) - if err != nil { - return 0, err - } - - isFirstRecord := true - for i := 0; i < numBatches; i++ { - // Fetch batch of events as JSON params := repo.ExportEventsParams{ - ProjectID: common.StringToPgTextNullable(projectID), - CreatedAt: common.TimeToPgTimestamptz(createdAt), - Cursor: common.StringToPgText(lastID), // Use Filter to keep Valid=true for SQL comparison - PageLimit: pgtype.Int8{Int64: int64(batchSize), Valid: true}, + CreatedAtEnd: common.TimeToPgTimestamptz(end), + CreatedAtStart: common.TimeToPgTimestamptz(start), + Cursor: common.StringToPgText(lastID), + PageLimit: pgtype.Int8{Int64: int64(batchSize), Valid: true}, } - rows, exportErr := s.repo.ExportEvents(ctx, params) + rows, exportErr := txRepo.ExportEvents(ctx, params) if exportErr != nil { return 0, fmt.Errorf("failed to query batch %d: %w", i, exportErr) } - // Write each JSON record to the writer for _, row := range rows { - // Add a comma separator between records (not before the first record) - if !isFirstRecord { - _, writeErr := w.Write([]byte(`,`)) - if writeErr != nil { - return 0, writeErr - } + if _, writeErr := w.Write(row.JsonOutput); writeErr != nil { + return 0, writeErr } - isFirstRecord = false - - // Write the JSON record - _, writeErr := w.Write(row.JsonOutput) - if writeErr != nil { + if _, writeErr := w.Write([]byte("\n")); writeErr != nil { return 0, writeErr } numDocs++ - - // Use the ID directly for cursor pagination (no JSON parsing needed) lastID = row.ID } } - // Write a closing bracket for JSON array - _, err = w.Write([]byte(`]`)) - if err != nil { - return 0, err - } - return numDocs, nil } diff --git a/internal/events/impl_test.go b/internal/events/impl_test.go index 6a14e7be55..1a9cec359d 100644 --- a/internal/events/impl_test.go +++ b/internal/events/impl_test.go @@ -1,6 +1,7 @@ package events import ( + "bytes" "context" "encoding/json" "fmt" @@ -208,6 +209,74 @@ func defaultSearchParams() datastore.SearchParams { } } +func TestExportRecords(t *testing.T) { + service, db := setupTestDB(t) + ctx := context.Background() + + project := seedTestProject(t, db) + endpoint := seedTestEndpoint(t, db, project.UID) + source := seedTestSource(t, db, project.UID) + + t.Run("Success", func(t *testing.T) { + for range 5 { + event := createTestEvent(t, project.UID, []string{endpoint.UID}, source.UID) + require.NoError(t, service.CreateEvent(ctx, event)) + } + + var buf bytes.Buffer + epoch := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + count, err := service.ExportRecords(ctx, epoch, time.Now().Add(1*time.Hour), &buf) + require.NoError(t, err) + require.GreaterOrEqual(t, count, int64(5)) + + // Verify valid JSONL + lines := bytes.Split(bytes.TrimSpace(buf.Bytes()), []byte("\n")) + require.GreaterOrEqual(t, len(lines), 5) + for _, line := range lines { + var record map[string]any + err = json.Unmarshal(line, &record) + require.NoError(t, err) + // id should be renamed to uid + require.NotEmpty(t, record["uid"], "each record should have uid") + require.Nil(t, record["id"], "id should be renamed to uid") + } + }) + + t.Run("Empty_with_past_cutoff", func(t *testing.T) { + var buf bytes.Buffer + epoch := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + count, err := service.ExportRecords(ctx, epoch, time.Now().Add(-24*time.Hour), &buf) + require.NoError(t, err) + require.Equal(t, int64(0), count) + require.Empty(t, buf.String()) + }) + + t.Run("TimeWindow", func(t *testing.T) { + // Create more events + for range 3 { + event := createTestEvent(t, project.UID, []string{endpoint.UID}, source.UID) + require.NoError(t, service.CreateEvent(ctx, event)) + } + + // Export with window [1h ago, now+1h) — should include all recently created + var buf bytes.Buffer + start := time.Now().Add(-1 * time.Hour) + end := time.Now().Add(1 * time.Hour) + count, err := service.ExportRecords(ctx, start, end, &buf) + require.NoError(t, err) + require.GreaterOrEqual(t, count, int64(3)) + + lines := bytes.Split(bytes.TrimSpace(buf.Bytes()), []byte("\n")) + require.GreaterOrEqual(t, len(lines), 3) + + // Export with window that excludes all: [2h ago, 1h ago) + buf.Reset() + count, err = service.ExportRecords(ctx, time.Now().Add(-2*time.Hour), time.Now().Add(-1*time.Hour), &buf) + require.NoError(t, err) + require.Equal(t, int64(0), count) + }) +} + func TestCreateEvent(t *testing.T) { service, db := setupTestDB(t) ctx := context.Background() diff --git a/internal/events/queries.sql b/internal/events/queries.sql index b4870a1429..6c1c076f0a 100644 --- a/internal/events/queries.sql +++ b/internal/events/queries.sql @@ -414,19 +414,16 @@ SELECT convoy.copy_rows(@project_id, @batch_size); SELECT ed.id, TO_JSONB(ed) - 'id' || JSONB_BUILD_OBJECT('uid', ed.id) AS json_output FROM convoy.events AS ed -WHERE project_id = @project_id - AND created_at < @created_at +WHERE created_at < @created_at_end + AND created_at >= @created_at_start AND (id > @cursor OR @cursor = '') - AND deleted_at IS NULL ORDER BY id LIMIT @page_limit; -- name: CountExportedEvents :one SELECT COUNT(*) as count FROM convoy.events -WHERE project_id = @project_id - AND created_at < @created_at - AND (id > @cursor OR @cursor = '') - AND deleted_at IS NULL; +WHERE created_at < @created_at_end + AND created_at >= @created_at_start; -- ============================================================================ -- Group 5: Partition Management (4 queries) diff --git a/internal/events/repo/queries.sql.go b/internal/events/repo/queries.sql.go index 227147788e..b68d6284c1 100644 --- a/internal/events/repo/queries.sql.go +++ b/internal/events/repo/queries.sql.go @@ -65,20 +65,17 @@ func (q *Queries) CountEvents(ctx context.Context, arg CountEventsParams) (pgtyp const countExportedEvents = `-- name: CountExportedEvents :one SELECT COUNT(*) as count FROM convoy.events -WHERE project_id = $1 - AND created_at < $2 - AND (id > $3 OR $3 = '') - AND deleted_at IS NULL +WHERE created_at < $1 + AND created_at >= $2 ` type CountExportedEventsParams struct { - ProjectID pgtype.Text - CreatedAt pgtype.Timestamptz - Cursor pgtype.Text + CreatedAtEnd pgtype.Timestamptz + CreatedAtStart pgtype.Timestamptz } func (q *Queries) CountExportedEvents(ctx context.Context, arg CountExportedEventsParams) (pgtype.Int8, error) { - row := q.db.QueryRow(ctx, countExportedEvents, arg.ProjectID, arg.CreatedAt, arg.Cursor) + row := q.db.QueryRow(ctx, countExportedEvents, arg.CreatedAtEnd, arg.CreatedAtStart) var count pgtype.Int8 err := row.Scan(&count) return count, err @@ -316,19 +313,18 @@ const exportEvents = `-- name: ExportEvents :many SELECT ed.id, TO_JSONB(ed) - 'id' || JSONB_BUILD_OBJECT('uid', ed.id) AS json_output FROM convoy.events AS ed -WHERE project_id = $1 - AND created_at < $2 +WHERE created_at < $1 + AND created_at >= $2 AND (id > $3 OR $3 = '') - AND deleted_at IS NULL ORDER BY id LIMIT $4 ` type ExportEventsParams struct { - ProjectID pgtype.Text - CreatedAt pgtype.Timestamptz - Cursor pgtype.Text - PageLimit pgtype.Int8 + CreatedAtEnd pgtype.Timestamptz + CreatedAtStart pgtype.Timestamptz + Cursor pgtype.Text + PageLimit pgtype.Int8 } type ExportEventsRow struct { @@ -337,12 +333,7 @@ type ExportEventsRow struct { } func (q *Queries) ExportEvents(ctx context.Context, arg ExportEventsParams) ([]ExportEventsRow, error) { - rows, err := q.db.Query(ctx, exportEvents, - arg.ProjectID, - arg.CreatedAt, - arg.Cursor, - arg.PageLimit, - ) + rows, err := q.db.Query(ctx, exportEvents, arg.CreatedAtEnd, arg.CreatedAtStart, arg.Cursor, arg.PageLimit) if err != nil { return nil, err } diff --git a/internal/pkg/backup_collector/buffer.go b/internal/pkg/backup_collector/buffer.go new file mode 100644 index 0000000000..f4d807bb4a --- /dev/null +++ b/internal/pkg/backup_collector/buffer.go @@ -0,0 +1,66 @@ +package backup_collector + +import ( + "sync" + + "github.com/jackc/pglogrepl" +) + +// BufferEntry holds a single WAL INSERT record. +type BufferEntry struct { + Values map[string]string +} + +// Buffer is a thread-safe accumulator for WAL insert records, keyed by table name. +// The streamLoop goroutine appends records; the flushLoop goroutine +// periodically swaps the buffer and processes the old contents. +type Buffer struct { + mu sync.Mutex + records map[string][]BufferEntry // key: table name (e.g. "events") + maxLSN pglogrepl.LSN +} + +// NewBuffer creates an empty buffer. +func NewBuffer() *Buffer { + return &Buffer{ + records: make(map[string][]BufferEntry), + } +} + +// Append adds a record to the buffer. Thread-safe. +func (b *Buffer) Append(tableName string, values map[string]string, lsn pglogrepl.LSN) { + b.mu.Lock() + defer b.mu.Unlock() + + b.records[tableName] = append(b.records[tableName], BufferEntry{Values: values}) + if lsn > b.maxLSN { + b.maxLSN = lsn + } +} + +// Swap atomically replaces the buffer with an empty one and returns the +// old contents along with the highest LSN seen. +func (b *Buffer) Swap() (map[string][]BufferEntry, pglogrepl.LSN) { + b.mu.Lock() + defer b.mu.Unlock() + + records := b.records + maxLSN := b.maxLSN + + b.records = make(map[string][]BufferEntry) + b.maxLSN = 0 + + return records, maxLSN +} + +// Len returns the total number of buffered records. Thread-safe. +func (b *Buffer) Len() int { + b.mu.Lock() + defer b.mu.Unlock() + + n := 0 + for _, entries := range b.records { + n += len(entries) + } + return n +} diff --git a/internal/pkg/backup_collector/buffer_test.go b/internal/pkg/backup_collector/buffer_test.go new file mode 100644 index 0000000000..8b3bc4821c --- /dev/null +++ b/internal/pkg/backup_collector/buffer_test.go @@ -0,0 +1,83 @@ +package backup_collector + +import ( + "fmt" + "sync" + "testing" + + "github.com/jackc/pglogrepl" + "github.com/stretchr/testify/require" +) + +func TestBuffer_Append(t *testing.T) { + buf := NewBuffer() + + buf.Append("events", map[string]string{"id": "1", "project_id": "p1"}, 100) + buf.Append("events", map[string]string{"id": "2", "project_id": "p1"}, 200) + buf.Append("event_deliveries", map[string]string{"id": "3", "project_id": "p1"}, 300) + + require.Equal(t, 3, buf.Len()) + + records, lsn := buf.Swap() + require.Len(t, records["events"], 2) + require.Len(t, records["event_deliveries"], 1) + require.Equal(t, pglogrepl.LSN(300), lsn) +} + +func TestBuffer_Append_Concurrent(t *testing.T) { + buf := NewBuffer() + var wg sync.WaitGroup + + for i := range 100 { + wg.Add(1) + go func(n int) { + defer wg.Done() + buf.Append("events", map[string]string{ + "id": fmt.Sprintf("e%d", n), + "project_id": "p1", + }, pglogrepl.LSN(n)) + }(i) + } + + wg.Wait() + require.Equal(t, 100, buf.Len()) +} + +func TestBuffer_Swap(t *testing.T) { + buf := NewBuffer() + + buf.Append("events", map[string]string{"id": "1"}, 100) + buf.Append("events", map[string]string{"id": "2"}, 200) + + records, lsn := buf.Swap() + require.Len(t, records["events"], 2) + require.Equal(t, pglogrepl.LSN(200), lsn) + + // After swap, buffer is empty + require.Equal(t, 0, buf.Len()) + + // Second swap returns nothing + records2, lsn2 := buf.Swap() + require.Len(t, records2, 0) + require.Equal(t, pglogrepl.LSN(0), lsn2) +} + +func TestBuffer_Swap_Empty(t *testing.T) { + buf := NewBuffer() + + records, lsn := buf.Swap() + require.Len(t, records, 0) + require.Equal(t, pglogrepl.LSN(0), lsn) +} + +func TestBuffer_LSN_Tracking(t *testing.T) { + buf := NewBuffer() + + // LSN should track the maximum + buf.Append("events", map[string]string{"id": "1"}, 500) + buf.Append("events", map[string]string{"id": "2"}, 100) // lower LSN + buf.Append("events", map[string]string{"id": "3"}, 999) // highest + + _, lsn := buf.Swap() + require.Equal(t, pglogrepl.LSN(999), lsn) +} diff --git a/internal/pkg/backup_collector/collector.go b/internal/pkg/backup_collector/collector.go new file mode 100644 index 0000000000..a3d93be6a0 --- /dev/null +++ b/internal/pkg/backup_collector/collector.go @@ -0,0 +1,317 @@ +package backup_collector + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgproto3" + "github.com/jackc/pgx/v5/pgxpool" + + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" + log "github.com/frain-dev/convoy/pkg/logger" +) + +const ( + defaultSlotName = "convoy_backup" + defaultPublication = "convoy_backup" + standbyStatusInterval = 10 * time.Second + receiveTimeout = 5 * time.Second +) + +// BackupCollector streams WAL changes from PostgreSQL for the backup tables +// and periodically flushes them as gzip-compressed JSONL to a BlobStore. +type BackupCollector struct { + pool *pgxpool.Pool + dsn string + slotName string + publication string + + replConn *pgconn.PgConn + relations map[uint32]*pglogrepl.RelationMessage + clientLSN pglogrepl.LSN + flushedLSN atomic.Uint64 + + buffer *Buffer + store blobstore.BlobStore + + flushInterval time.Duration + + cancel context.CancelFunc + wg sync.WaitGroup + logger log.Logger +} + +// NewBackupCollector creates a new CDC-based backup collector. +func NewBackupCollector( + pool *pgxpool.Pool, + dsn string, + store blobstore.BlobStore, + flushInterval time.Duration, + logger log.Logger, +) *BackupCollector { + return &BackupCollector{ + pool: pool, + dsn: dsn, + slotName: defaultSlotName, + publication: defaultPublication, + relations: make(map[uint32]*pglogrepl.RelationMessage), + buffer: NewBuffer(), + store: store, + flushInterval: flushInterval, + logger: logger, + } +} + +// Start initialises the replication slot (if needed) and begins streaming +// WAL changes + periodic flushing in background goroutines. +func (c *BackupCollector) Start(ctx context.Context) error { + replConn, err := c.connectReplication(ctx) + if err != nil { + return fmt.Errorf("replication connect: %w", err) + } + c.replConn = replConn + + // Check if slot already exists (restart case) + var startLSN pglogrepl.LSN + var restartLSNStr *string + + err = c.pool.QueryRow(ctx, + "SELECT restart_lsn::TEXT FROM pg_replication_slots WHERE slot_name = $1", + c.slotName, + ).Scan(&restartLSNStr) + + switch { + case err == nil && restartLSNStr != nil: + // Slot exists — resume from restart LSN + startLSN, err = pglogrepl.ParseLSN(*restartLSNStr) + if err != nil { + c.replConn.Close(ctx) + return fmt.Errorf("parse restart LSN: %w", err) + } + c.logger.Info(fmt.Sprintf("resuming from existing slot %q at LSN %s", c.slotName, startLSN)) + case err != nil && !errors.Is(err, pgx.ErrNoRows): + // Unexpected error (network, permissions, etc.) + c.replConn.Close(ctx) + return fmt.Errorf("check replication slot: %w", err) + default: + // Slot does not exist — create it + result, createErr := pglogrepl.CreateReplicationSlot( + ctx, c.replConn, c.slotName, "pgoutput", + pglogrepl.CreateReplicationSlotOptions{ + Temporary: false, + SnapshotAction: "EXPORT_SNAPSHOT", + }, + ) + if createErr != nil { + c.replConn.Close(ctx) + return fmt.Errorf("create replication slot: %w", createErr) + } + + startLSN, err = pglogrepl.ParseLSN(result.ConsistentPoint) + if err != nil { + c.replConn.Close(ctx) + return fmt.Errorf("parse consistent point LSN: %w", err) + } + + c.logger.Info(fmt.Sprintf("created replication slot %q at LSN %s (snapshot: %s)", + result.SlotName, startLSN, result.SnapshotName)) + } + + c.clientLSN = startLSN + c.flushedLSN.Store(uint64(startLSN)) + + err = pglogrepl.StartReplication( + ctx, c.replConn, c.slotName, startLSN, + pglogrepl.StartReplicationOptions{ + PluginArgs: []string{ + "proto_version '1'", + fmt.Sprintf("publication_names '%s'", c.publication), + }, + }, + ) + if err != nil { + c.replConn.Close(ctx) + return fmt.Errorf("start replication: %w", err) + } + + streamCtx, cancel := context.WithCancel(ctx) + c.cancel = cancel + + c.wg.Add(2) + go c.streamLoop(streamCtx) + go c.flushLoop(streamCtx) + + c.logger.Info("backup collector started — streaming WAL changes") + return nil +} + +// Stop cancels the streaming goroutines and closes the replication connection. +func (c *BackupCollector) Stop(ctx context.Context) { + if c.cancel != nil { + c.cancel() + } + c.wg.Wait() + + if c.replConn != nil { + if err := c.replConn.Close(ctx); err != nil { + c.logger.Warn(fmt.Sprintf("close replication connection: %v", err)) + } + } + c.logger.Info("backup collector stopped") +} + +// connectReplication opens a pgconn connection with the replication protocol. +func (c *BackupCollector) connectReplication(ctx context.Context) (*pgconn.PgConn, error) { + cfg, err := pgconn.ParseConfig(c.dsn) + if err != nil { + return nil, fmt.Errorf("parse dsn: %w", err) + } + cfg.RuntimeParams["replication"] = "database" + return pgconn.ConnectConfig(ctx, cfg) +} + +// streamLoop receives WAL messages and buffers INSERT records. +func (c *BackupCollector) streamLoop(ctx context.Context) { + defer c.wg.Done() + + nextStandbyDeadline := time.Now().Add(standbyStatusInterval) + + for { + if ctx.Err() != nil { + return + } + + if time.Now().After(nextStandbyDeadline) { + if err := c.sendStandbyStatus(ctx); err != nil { + c.logger.Warn(fmt.Sprintf("send standby status: %v", err)) + } + nextStandbyDeadline = time.Now().Add(standbyStatusInterval) + } + + recvCtx, cancel := context.WithDeadline(ctx, time.Now().Add(receiveTimeout)) + rawMsg, err := c.replConn.ReceiveMessage(recvCtx) + cancel() + + if err != nil { + if ctx.Err() != nil { + return + } + if pgconn.Timeout(err) || recvCtx.Err() != nil { + continue + } + c.logger.Error(fmt.Sprintf("receive WAL message: %v", err)) + c.cancel() // signal flushLoop to do final flush and exit + return + } + + if errMsg, ok := rawMsg.(*pgproto3.ErrorResponse); ok { + c.logger.Error(fmt.Sprintf("WAL stream error: severity=%s message=%s code=%s", + errMsg.Severity, errMsg.Message, string(errMsg.Code))) + c.cancel() // signal flushLoop to do final flush and exit + return + } + + msg, ok := rawMsg.(*pgproto3.CopyData) + if !ok { + continue + } + + switch msg.Data[0] { + case pglogrepl.XLogDataByteID: + xld, parseErr := pglogrepl.ParseXLogData(msg.Data[1:]) + if parseErr != nil { + c.logger.Error(fmt.Sprintf("parse XLogData: %v", parseErr)) + continue + } + c.handleXLogData(xld) + + case pglogrepl.PrimaryKeepaliveMessageByteID: + pkm, parseErr := pglogrepl.ParsePrimaryKeepaliveMessage(msg.Data[1:]) + if parseErr != nil { + c.logger.Error(fmt.Sprintf("parse keepalive: %v", parseErr)) + continue + } + if pkm.ServerWALEnd > c.clientLSN { + c.clientLSN = pkm.ServerWALEnd + } + if pkm.ReplyRequested { + if statusErr := c.sendStandbyStatus(ctx); statusErr != nil { + c.logger.Warn(fmt.Sprintf("send standby status (reply requested): %v", statusErr)) + } + nextStandbyDeadline = time.Now().Add(standbyStatusInterval) + } + } + } +} + +func (c *BackupCollector) sendStandbyStatus(ctx context.Context) error { + return pglogrepl.SendStandbyStatusUpdate(ctx, c.replConn, pglogrepl.StandbyStatusUpdate{ + WALWritePosition: pglogrepl.LSN(c.flushedLSN.Load()), + }) +} + +func (c *BackupCollector) handleXLogData(xld pglogrepl.XLogData) { + logicalMsg, err := pglogrepl.Parse(xld.WALData) + if err != nil { + c.logger.Warn(fmt.Sprintf("parse logical message: %v", err)) + return + } + + switch m := logicalMsg.(type) { + case *pglogrepl.RelationMessage: + c.relations[m.RelationID] = m + c.logger.Info(fmt.Sprintf("CDC relation: %s.%s (id=%d, cols=%d)", m.Namespace, m.RelationName, m.RelationID, len(m.Columns))) + + case *pglogrepl.InsertMessage: + rel, ok := c.relations[m.RelationID] + if !ok { + c.logger.Warn(fmt.Sprintf("CDC insert for unknown relation %d", m.RelationID)) + return + } + values := tupleToMap(rel, m.Tuple) + if values != nil { + c.buffer.Append(rel.RelationName, values, xld.WALStart+pglogrepl.LSN(len(xld.WALData))) + } + + case *pglogrepl.BeginMessage, *pglogrepl.CommitMessage, + *pglogrepl.UpdateMessage, *pglogrepl.DeleteMessage: + // Ignored — we only back up INSERTs + } + + if xld.WALStart > 0 { + newLSN := xld.WALStart + pglogrepl.LSN(len(xld.WALData)) + if newLSN > c.clientLSN { + c.clientLSN = newLSN + } + } +} + +// tupleToMap converts a pglogrepl TupleData into a map of column name → text value. +func tupleToMap(rel *pglogrepl.RelationMessage, tuple *pglogrepl.TupleData) map[string]string { + if tuple == nil { + return nil + } + values := make(map[string]string, len(rel.Columns)) + for i, col := range rel.Columns { + if i >= len(tuple.Columns) { + break + } + tc := tuple.Columns[i] + switch tc.DataType { + case pglogrepl.TupleDataTypeText: + values[col.Name] = string(tc.Data) + case pglogrepl.TupleDataTypeNull: + // null — skip + case pglogrepl.TupleDataTypeToast: + // unchanged toast — skip (shouldn't happen for INSERTs) + } + } + return values +} diff --git a/internal/pkg/backup_collector/collector_test.go b/internal/pkg/backup_collector/collector_test.go new file mode 100644 index 0000000000..4c5b869528 --- /dev/null +++ b/internal/pkg/backup_collector/collector_test.go @@ -0,0 +1,291 @@ +package backup_collector + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" + + "github.com/frain-dev/convoy/api/testdb" + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/database/hooks" + "github.com/frain-dev/convoy/database/postgres" + "github.com/frain-dev/convoy/datastore" + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" + "github.com/frain-dev/convoy/internal/pkg/keys" + log "github.com/frain-dev/convoy/pkg/logger" + "github.com/frain-dev/convoy/testenv" +) + +var infra *testenv.Environment + +func TestMain(m *testing.M) { + res, cleanup, err := testenv.Launch( + context.Background(), + testenv.WithMinIO(), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to launch test infrastructure: %v\n", err) + os.Exit(1) + } + + infra = res + code := m.Run() + + if err := cleanup(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to cleanup: %v\n", err) + } + + os.Exit(code) +} + +func setupTestDB(t *testing.T) (*pgxpool.Pool, string) { + t.Helper() + + err := config.LoadConfig("") + require.NoError(t, err) + + pool, err := infra.CloneTestDatabase(t, "convoy") + require.NoError(t, err) + + dbHooks := hooks.Init() + dbHooks.RegisterHook(datastore.EndpointCreated, func(_ context.Context, _ any, _ any) {}) + + km, err := keys.NewLocalKeyManager("test") + require.NoError(t, err) + err = keys.Set(km) + require.NoError(t, err) + + // Build DSN from pool config for the replication connection + cfg := pool.Config().ConnConfig + dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", + cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database) + + // Create the publication in this cloned DB (template may not have it) + _, err = pool.Exec(context.Background(), ` + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'convoy_backup') THEN + CREATE PUBLICATION convoy_backup FOR TABLE + convoy.events, convoy.event_deliveries, convoy.delivery_attempts; + END IF; + END $$; + `) + require.NoError(t, err) + + return pool, dsn +} + +// seedProject creates a project with all FK dependencies using testdb helpers. +func seedProject(t *testing.T, pool *pgxpool.Pool) string { + t.Helper() + + db := postgres.NewFromConnection(pool) + + user, err := testdb.SeedDefaultUser(db) + require.NoError(t, err) + + org, err := testdb.SeedDefaultOrganisation(db, user) + require.NoError(t, err) + + project, err := testdb.SeedDefaultProject(db, org.UID) + require.NoError(t, err) + + return project.UID +} + +func TestCollector_StartStop(t *testing.T) { + pool, dsn := setupTestDB(t) + tmpDir := t.TempDir() + + logger := log.New("test", log.LevelInfo) + store, err := blobstore.NewOnPremClient(blobstore.BlobStoreOptions{OnPremStorageDir: tmpDir}, logger) + require.NoError(t, err) + + collector := NewBackupCollector(pool, dsn, store, 10*time.Second, logger) + + ctx := context.Background() + err = collector.Start(ctx) + require.NoError(t, err) + + // Verify slot exists + var slotName string + err = pool.QueryRow(ctx, "SELECT slot_name FROM pg_replication_slots WHERE slot_name = $1", defaultSlotName).Scan(&slotName) + require.NoError(t, err) + require.Equal(t, defaultSlotName, slotName) + + // Stop + collector.Stop(ctx) + + // Slot should still exist (permanent) + err = pool.QueryRow(ctx, "SELECT slot_name FROM pg_replication_slots WHERE slot_name = $1", defaultSlotName).Scan(&slotName) + require.NoError(t, err) + require.Equal(t, defaultSlotName, slotName) + + // Cleanup slot + _, _ = pool.Exec(ctx, "SELECT pg_drop_replication_slot($1)", defaultSlotName) +} + +func TestCollector_CaptureInserts(t *testing.T) { + pool, dsn := setupTestDB(t) + tmpDir := t.TempDir() + + logger := log.New("test", log.LevelInfo) + store, err := blobstore.NewOnPremClient(blobstore.BlobStoreOptions{OnPremStorageDir: tmpDir}, logger) + require.NoError(t, err) + + // Use short flush interval for tests + collector := NewBackupCollector(pool, dsn, store, 3*time.Second, logger) + + ctx := context.Background() + err = collector.Start(ctx) + require.NoError(t, err) + + defer func() { + collector.Stop(ctx) + _, _ = pool.Exec(ctx, "SELECT pg_drop_replication_slot($1)", defaultSlotName) + }() + + // Seed FK dependencies + projectID := seedProject(t, pool) + + // Insert events + for i := range 5 { + _, err = pool.Exec(ctx, ` + INSERT INTO convoy.events (id, event_type, endpoints, project_id, headers, raw, data, status, created_at, updated_at) + VALUES ($1, 'test.event', '{}', $2, '{}', '{}', '\x7b7d', 'Success', NOW(), NOW()) + `, fmt.Sprintf("evt_%d_%d", i, time.Now().UnixNano()), projectID) + require.NoError(t, err) + } + + // Wait for flush + time.Sleep(5 * time.Second) + + // Verify files exist + files := findGzipFiles(t, tmpDir) + require.NotEmpty(t, files, "should have backup files after flush") + + // Find events file and count records + var totalEvents int + for _, f := range files { + if containsPath(f, "events") { + records := readJSONLFile(t, f) + totalEvents += len(records) + } + } + + require.GreaterOrEqual(t, totalEvents, 5, "should have captured at least 5 events") +} + +func TestCollector_IgnoreUpdatesDeletes(t *testing.T) { + pool, dsn := setupTestDB(t) + tmpDir := t.TempDir() + + logger := log.New("test", log.LevelInfo) + store, err := blobstore.NewOnPremClient(blobstore.BlobStoreOptions{OnPremStorageDir: tmpDir}, logger) + require.NoError(t, err) + + collector := NewBackupCollector(pool, dsn, store, 3*time.Second, logger) + + ctx := context.Background() + err = collector.Start(ctx) + require.NoError(t, err) + + defer func() { + collector.Stop(ctx) + _, _ = pool.Exec(ctx, "SELECT pg_drop_replication_slot($1)", defaultSlotName) + }() + + projectID := seedProject(t, pool) + + // Insert 3 events + eventIDs := make([]string, 3) + for i := range 3 { + eventIDs[i] = fmt.Sprintf("evt_ud_%d_%d", i, time.Now().UnixNano()) + _, err = pool.Exec(ctx, ` + INSERT INTO convoy.events (id, event_type, endpoints, project_id, headers, raw, data, status, created_at, updated_at) + VALUES ($1, 'test.event', '{}', $2, '{}', '{}', '\x7b7d', 'Success', NOW(), NOW()) + `, eventIDs[i], projectID) + require.NoError(t, err) + } + + // Update one + _, err = pool.Exec(ctx, `UPDATE convoy.events SET status = 'Failed' WHERE id = $1`, eventIDs[0]) + require.NoError(t, err) + + // Soft-delete one + _, err = pool.Exec(ctx, `UPDATE convoy.events SET deleted_at = NOW() WHERE id = $1`, eventIDs[1]) + require.NoError(t, err) + + // Wait for flush + time.Sleep(5 * time.Second) + + // Count all exported events — should be exactly 3 (only INSERTs) + files := findGzipFiles(t, tmpDir) + var totalEvents int + for _, f := range files { + if containsPath(f, "events") { + records := readJSONLFile(t, f) + totalEvents += len(records) + } + } + + require.Equal(t, 3, totalEvents, "should have exactly 3 events (inserts only, no updates/deletes)") +} + +// --- Test helpers --- + +func findGzipFiles(t *testing.T, dir string) []string { + t.Helper() + var files []string + entries, err := os.ReadDir(dir) + require.NoError(t, err) + for _, e := range entries { + if e.IsDir() { + sub := findGzipFiles(t, dir+"/"+e.Name()) + files = append(files, sub...) + } else if len(e.Name()) > 3 && e.Name()[len(e.Name())-3:] == ".gz" { + files = append(files, dir+"/"+e.Name()) + } + } + return files +} + +func readJSONLFile(t *testing.T, path string) []map[string]any { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + + gr, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + defer gr.Close() + + raw, err := io.ReadAll(gr) + require.NoError(t, err) + + var results []map[string]any + scanner := bufio.NewScanner(bytes.NewReader(raw)) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var record map[string]any + err = json.Unmarshal(line, &record) + require.NoError(t, err) + results = append(results, record) + } + return results +} + +func containsPath(path, segment string) bool { + return len(path) > 0 && bytes.Contains([]byte(path), []byte("/"+segment+"/")) +} diff --git a/internal/pkg/backup_collector/flush.go b/internal/pkg/backup_collector/flush.go new file mode 100644 index 0000000000..24db3169f1 --- /dev/null +++ b/internal/pkg/backup_collector/flush.go @@ -0,0 +1,179 @@ +package backup_collector + +import ( + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "time" +) + +// tableToKeySegment maps WAL table names to blob key path segments. +var tableToKeySegment = map[string]string{ + "events": "events", + "event_deliveries": "eventdeliveries", + "delivery_attempts": "deliveryattempts", +} + +// flushLoop runs on a ticker, swapping the buffer and uploading to blob storage. +func (c *BackupCollector) flushLoop(ctx context.Context) { + defer c.wg.Done() + + ticker := time.NewTicker(c.flushInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + // Final flush — use fresh context since ctx is cancelled + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + c.doFlush(shutdownCtx) + shutdownCancel() + return + case <-ticker.C: + c.doFlush(ctx) + } + } +} + +// doFlush swaps the buffer and uploads each table's entries to blob storage. +// At-least-once semantics: if any table fails, the LSN is NOT advanced. +// On restart, the WAL replays from the last good LSN, re-exporting all tables +// (including ones that succeeded). This is simpler and safer than re-queuing +// failed entries, which risks unbounded memory growth. +func (c *BackupCollector) doFlush(ctx context.Context) { + records, swapLSN := c.buffer.Swap() + if len(records) == 0 { + return + } + + total := 0 + for _, entries := range records { + total += len(entries) + } + c.logger.Info(fmt.Sprintf("flushing %d records across %d tables (LSN: %s)", total, len(records), swapLSN)) + + allOK := true + for tableName, entries := range records { + if err := c.flushTable(ctx, tableName, entries); err != nil { + c.logger.Error(fmt.Sprintf("flush failed for %s: %v", tableName, err)) + allOK = false + } + } + + if allOK && swapLSN > 0 { + c.flushedLSN.Store(uint64(swapLSN)) + c.logger.Info(fmt.Sprintf("advanced flushed LSN to %s", swapLSN)) + } +} + +func (c *BackupCollector) flushTable(ctx context.Context, tableName string, entries []BufferEntry) error { + if len(entries) == 0 { + return nil + } + + segment, ok := tableToKeySegment[tableName] + if !ok { + return fmt.Errorf("unknown table: %s", tableName) + } + + now := time.Now().UTC() + date := now.Format("2006-01-02") + ts := now.Format(time.RFC3339) + blobKey := fmt.Sprintf("backup/%s/%s/%s.jsonl.gz", date, segment, ts) + + pr, pw := io.Pipe() + errCh := make(chan error, 1) + + go func() { + var writeErr error + gzw := gzip.NewWriter(pw) + enc := json.NewEncoder(gzw) + + for _, entry := range entries { + record := recordToJSON(tableName, entry.Values) + if writeErr = enc.Encode(record); writeErr != nil { + break + } + } + + // Always close gzip first to flush the trailer, then close the pipe + if closeErr := gzw.Close(); writeErr == nil { + writeErr = closeErr + } + if writeErr != nil { + pw.CloseWithError(writeErr) + } else { + pw.Close() + } + errCh <- writeErr + }() + + // Wait for goroutine to finish FIRST by reading errCh, + // but Upload blocks on pr — so we must read both. + // Upload returns when pw is closed (by goroutine). + uploadErr := c.store.Upload(ctx, blobKey, pr) + encodeErr := <-errCh + + if encodeErr != nil { + return fmt.Errorf("encode: %w", encodeErr) + } + if uploadErr != nil { + return fmt.Errorf("upload: %w", uploadErr) + } + + c.logger.Info(fmt.Sprintf("uploaded %d records to %s", len(entries), blobKey)) + return nil +} + +// recordToJSON converts WAL column values to a JSON-compatible map. +// Renames "id" to "uid" to match the existing export format. +func recordToJSON(tableName string, values map[string]string) map[string]any { + result := make(map[string]any, len(values)) + + for k, v := range values { + if k == "id" { + result["uid"] = v + continue + } + + if isJSONColumn(tableName, k) && len(v) > 0 && (v[0] == '{' || v[0] == '[' || v[0] == '"') { + result[k] = json.RawMessage(v) + continue + } + + result[k] = v + } + + return result +} + +// isJSONColumn returns true if the column stores JSON/JSONB data. +func isJSONColumn(tableName, column string) bool { + jsonColumns := map[string]map[string]bool{ + "events": { + "headers": true, + // "data" is bytea, not jsonb — WAL sends it as hex (\x...) + // "raw" is text, not jsonb + "url_query_params": true, + "metadata": true, + }, + "event_deliveries": { + "headers": true, + "metadata": true, + "cli_metadata": true, + "attempts": true, + }, + "delivery_attempts": { + "request_http_header": true, + "response_http_header": true, + }, + } + + cols, ok := jsonColumns[tableName] + if !ok { + return false + } + return cols[column] +} diff --git a/internal/pkg/backup_collector/flush_test.go b/internal/pkg/backup_collector/flush_test.go new file mode 100644 index 0000000000..2bc493b9be --- /dev/null +++ b/internal/pkg/backup_collector/flush_test.go @@ -0,0 +1,104 @@ +package backup_collector + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRecordToJSON(t *testing.T) { + values := map[string]string{ + "id": "abc123", + "project_id": "proj1", + "event_type": "user.created", + "headers": `{"Content-Type":["application/json"]}`, + "status": "Success", + } + + result := recordToJSON("events", values) + + // id renamed to uid + require.Equal(t, "abc123", result["uid"]) + require.Nil(t, result["id"]) + + // plain strings preserved + require.Equal(t, "proj1", result["project_id"]) + require.Equal(t, "user.created", result["event_type"]) + require.Equal(t, "Success", result["status"]) + + // JSONB column parsed as RawMessage + headers, ok := result["headers"].(json.RawMessage) + require.True(t, ok, "headers should be json.RawMessage") + require.JSONEq(t, `{"Content-Type":["application/json"]}`, string(headers)) +} + +func TestRecordToJSON_EmptyValues(t *testing.T) { + values := map[string]string{ + "id": "abc", + "url_query_params": "", // empty string — should NOT be treated as JSON + "metadata": "", // empty string + "headers": "{}", // valid JSON + } + + result := recordToJSON("events", values) + + // Empty strings stay as plain strings, not json.RawMessage + require.Equal(t, "", result["url_query_params"]) + require.Equal(t, "", result["metadata"]) + + // Valid JSON still parsed + _, ok := result["headers"].(json.RawMessage) + require.True(t, ok) + + // Can marshal without error + _, err := json.Marshal(result) + require.NoError(t, err) +} + +func TestRecordToJSON_ByteaColumn(t *testing.T) { + values := map[string]string{ + "id": "abc", + "data": `\x7b226e616d65223a2274657374227d`, // hex-encoded bytea + "raw": "some raw text", + } + + result := recordToJSON("events", values) + + // bytea data should be a plain string, not json.RawMessage + dataVal, ok := result["data"].(string) + require.True(t, ok, "data should be a plain string") + require.Contains(t, dataVal, `\x`) + + // raw is text, also plain string + require.Equal(t, "some raw text", result["raw"]) + + // Must marshal without error + _, err := json.Marshal(result) + require.NoError(t, err) +} + +func TestIsJSONColumn(t *testing.T) { + // Events + require.True(t, isJSONColumn("events", "headers")) + require.True(t, isJSONColumn("events", "metadata")) + require.True(t, isJSONColumn("events", "url_query_params")) + require.False(t, isJSONColumn("events", "data")) // bytea + require.False(t, isJSONColumn("events", "raw")) // text + require.False(t, isJSONColumn("events", "event_type")) // text + + // Event deliveries + require.True(t, isJSONColumn("event_deliveries", "headers")) + require.True(t, isJSONColumn("event_deliveries", "metadata")) + require.True(t, isJSONColumn("event_deliveries", "cli_metadata")) + require.True(t, isJSONColumn("event_deliveries", "attempts")) + require.False(t, isJSONColumn("event_deliveries", "status")) + + // Delivery attempts + require.True(t, isJSONColumn("delivery_attempts", "request_http_header")) + require.True(t, isJSONColumn("delivery_attempts", "response_http_header")) + require.False(t, isJSONColumn("delivery_attempts", "url")) + + // Unknown table + require.False(t, isJSONColumn("unknown_table", "headers")) +} diff --git a/internal/pkg/blob-store/azure.go b/internal/pkg/blob-store/azure.go new file mode 100644 index 0000000000..7fe5536361 --- /dev/null +++ b/internal/pkg/blob-store/azure.go @@ -0,0 +1,88 @@ +package blobstore + +import ( + "context" + "fmt" + "io" + "strings" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + + log "github.com/frain-dev/convoy/pkg/logger" +) + +// AzureBlobClient implements BlobStore for Azure Blob Storage. +type AzureBlobClient struct { + client *azblob.Client + containerName string + prefix string + logger log.Logger + + containerOnce sync.Once + containerErr error +} + +// NewAzureBlobClient creates a new Azure Blob Storage BlobStore. +func NewAzureBlobClient(opts BlobStoreOptions, logger log.Logger) (BlobStore, error) { + serviceURL := opts.AzureEndpoint + if serviceURL == "" { + serviceURL = fmt.Sprintf("https://%s.blob.core.windows.net", opts.AzureAccountName) + } + + cred, err := azblob.NewSharedKeyCredential(opts.AzureAccountName, opts.AzureAccountKey) + if err != nil { + return nil, fmt.Errorf("azure credentials: %w", err) + } + + client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) + if err != nil { + return nil, fmt.Errorf("azure client: %w", err) + } + + return &AzureBlobClient{ + client: client, + containerName: opts.AzureContainerName, + prefix: opts.Prefix, + logger: logger, + }, nil +} + +// ensureContainer creates the blob container if it doesn't already exist. +// Safe to call concurrently — only runs once. +func (a *AzureBlobClient) ensureContainer(ctx context.Context) error { + a.containerOnce.Do(func() { + _, a.containerErr = a.client.CreateContainer(ctx, a.containerName, nil) + if a.containerErr != nil && strings.Contains(a.containerErr.Error(), "ContainerAlreadyExists") { + a.containerErr = nil // already exists is fine + } + if a.containerErr == nil { + a.logger.Info(fmt.Sprintf("ensured azure container %q exists", a.containerName)) + } + }) + return a.containerErr +} + +// Upload streams data to Azure Blob Storage. +func (a *AzureBlobClient) Upload(ctx context.Context, key string, r io.Reader) error { + if err := a.ensureContainer(ctx); err != nil { + return fmt.Errorf("ensure container: %w", err) + } + + blobName := key + if a.prefix != "" { + blobName = a.prefix + "/" + key + } + + _, err := a.client.UploadStream(ctx, a.containerName, blobName, r, + &azblob.UploadStreamOptions{ + BlockSize: 8 * 1024 * 1024, // 8MB per block + Concurrency: 3, + }) + if err != nil { + return fmt.Errorf("azure upload %q: %w", blobName, err) + } + + a.logger.Info(fmt.Sprintf("uploaded %q to azure container %q", blobName, a.containerName)) + return nil +} diff --git a/internal/pkg/blob-store/blobstore.go b/internal/pkg/blob-store/blobstore.go new file mode 100644 index 0000000000..7f1bb7fa85 --- /dev/null +++ b/internal/pkg/blob-store/blobstore.go @@ -0,0 +1,73 @@ +package blobstore + +import ( + "context" + "errors" + "io" + + "github.com/frain-dev/convoy/datastore" + log "github.com/frain-dev/convoy/pkg/logger" +) + +// BlobStore defines the interface for uploading export data to a storage backend. +type BlobStore interface { + Upload(ctx context.Context, key string, r io.Reader) error +} + +// BlobStoreOptions holds configuration for connecting to a blob storage backend. +type BlobStoreOptions struct { + Prefix string + Bucket string + AccessKey string + SecretKey string + Region string + Endpoint string + SessionToken string + + OnPremStorageDir string + + AzureAccountName string + AzureAccountKey string + AzureContainerName string + AzureEndpoint string +} + +// NewBlobStoreClient creates a BlobStore from the given storage policy configuration. +func NewBlobStoreClient(storage *datastore.StoragePolicyConfiguration, logger log.Logger) (BlobStore, error) { + if storage == nil { + return nil, errors.New("storage policy configuration is nil") + } + + switch storage.Type { + case datastore.S3: + opts := BlobStoreOptions{ + Prefix: storage.S3.Prefix.ValueOrZero(), + Bucket: storage.S3.Bucket.ValueOrZero(), + Endpoint: storage.S3.Endpoint.ValueOrZero(), + AccessKey: storage.S3.AccessKey.ValueOrZero(), + SecretKey: storage.S3.SecretKey.ValueOrZero(), + SessionToken: storage.S3.SessionToken.ValueOrZero(), + Region: storage.S3.Region.ValueOrZero(), + } + return NewS3Client(opts, logger) + + case datastore.OnPrem: + opts := BlobStoreOptions{ + OnPremStorageDir: storage.OnPrem.Path.String, + } + return NewOnPremClient(opts, logger) + + case datastore.AzureBlob: + opts := BlobStoreOptions{ + Prefix: storage.AzureBlob.Prefix.ValueOrZero(), + AzureAccountName: storage.AzureBlob.AccountName.ValueOrZero(), + AzureAccountKey: storage.AzureBlob.AccountKey.ValueOrZero(), + AzureContainerName: storage.AzureBlob.ContainerName.ValueOrZero(), + AzureEndpoint: storage.AzureBlob.Endpoint.ValueOrZero(), + } + return NewAzureBlobClient(opts, logger) + + default: + return nil, errors.New("invalid storage policy") + } +} diff --git a/internal/pkg/blob-store/blobstore_test.go b/internal/pkg/blob-store/blobstore_test.go new file mode 100644 index 0000000000..673f0c1ef7 --- /dev/null +++ b/internal/pkg/blob-store/blobstore_test.go @@ -0,0 +1,305 @@ +package blobstore + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" + "gopkg.in/guregu/null.v4" + + "github.com/frain-dev/convoy/datastore" + log "github.com/frain-dev/convoy/pkg/logger" + "github.com/frain-dev/convoy/testenv" +) + +var infra *testenv.Environment + +func TestMain(m *testing.M) { + res, cleanup, err := testenv.Launch( + context.Background(), + testenv.WithoutPostgres(), + testenv.WithoutRedis(), + testenv.WithMinIO(), + testenv.WithAzurite(), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to launch test infrastructure: %v\n", err) + os.Exit(1) + } + + infra = res + code := m.Run() + + if err := cleanup(); err != nil { + fmt.Fprintf(os.Stderr, "Failed to cleanup: %v\n", err) + } + + os.Exit(code) +} + +// ============================================================================ +// OnPrem Tests +// ============================================================================ + +func TestOnPremClient_Upload(t *testing.T) { + tmpDir := t.TempDir() + logger := log.New("test", log.LevelInfo) + + client, err := NewOnPremClient(BlobStoreOptions{OnPremStorageDir: tmpDir}, logger) + require.NoError(t, err) + + content := "hello world\n" + err = client.Upload(context.Background(), "test/file.txt", strings.NewReader(content)) + require.NoError(t, err) + + // Verify file exists with correct content + data, err := os.ReadFile(filepath.Join(tmpDir, "test/file.txt")) + require.NoError(t, err) + require.Equal(t, content, string(data)) +} + +func TestOnPremClient_Upload_CreatesDirectories(t *testing.T) { + tmpDir := t.TempDir() + logger := log.New("test", log.LevelInfo) + + client, err := NewOnPremClient(BlobStoreOptions{OnPremStorageDir: tmpDir}, logger) + require.NoError(t, err) + + err = client.Upload(context.Background(), "deeply/nested/path/file.jsonl.gz", strings.NewReader("data")) + require.NoError(t, err) + + _, err = os.Stat(filepath.Join(tmpDir, "deeply/nested/path/file.jsonl.gz")) + require.NoError(t, err) +} + +// ============================================================================ +// S3 Tests (via MinIO) +// ============================================================================ + +func TestS3Client_Upload(t *testing.T) { + if infra.NewMinIOClient == nil { + t.Skip("MinIO not available") + } + + minioClient, endpoint, err := (*infra.NewMinIOClient)(t) + require.NoError(t, err) + + logger := log.New("test", log.LevelInfo) + s3Client, err := NewS3Client(BlobStoreOptions{ + Bucket: "convoy-test-exports", + AccessKey: "minioadmin", + SecretKey: "minioadmin", + Region: "us-east-1", + Endpoint: "http://" + endpoint, + }, logger) + require.NoError(t, err) + + content := `{"uid":"123","event_type":"test"}` + "\n" + err = s3Client.Upload(context.Background(), "test/events.jsonl", strings.NewReader(content)) + require.NoError(t, err) + + // Download and verify via MinIO client + obj, err := minioClient.GetObject(context.Background(), "convoy-test-exports", "test/events.jsonl", minio.GetObjectOptions{}) + require.NoError(t, err) + defer obj.Close() + + data, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, content, string(data)) +} + +func TestS3Client_Upload_WithPrefix(t *testing.T) { + if infra.NewMinIOClient == nil { + t.Skip("MinIO not available") + } + + _, endpoint, err := (*infra.NewMinIOClient)(t) + require.NoError(t, err) + + logger := log.New("test", log.LevelInfo) + s3Client, err := NewS3Client(BlobStoreOptions{ + Bucket: "convoy-test-exports", + AccessKey: "minioadmin", + SecretKey: "minioadmin", + Region: "us-east-1", + Endpoint: "http://" + endpoint, + Prefix: "backups", + }, logger) + require.NoError(t, err) + + err = s3Client.Upload(context.Background(), "events.jsonl", strings.NewReader("data")) + require.NoError(t, err) + + // The object should be stored at "backups/events.jsonl" + minioClient, _, err := (*infra.NewMinIOClient)(t) + require.NoError(t, err) + + _, err = minioClient.StatObject(context.Background(), "convoy-test-exports", "backups/events.jsonl", minio.StatObjectOptions{}) + require.NoError(t, err) +} + +// ============================================================================ +// Azure Tests (via Azurite) +// ============================================================================ + +func TestAzureBlobClient_Upload(t *testing.T) { + if infra.NewAzuriteClient == nil { + t.Skip("Azurite not available") + } + + azClient, endpoint, err := (*infra.NewAzuriteClient)(t) + require.NoError(t, err) + + logger := log.New("test", log.LevelInfo) + blobClient, err := NewAzureBlobClient(BlobStoreOptions{ + AzureAccountName: "devstoreaccount1", + AzureAccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + AzureContainerName: "convoy-test-exports", + AzureEndpoint: endpoint, + }, logger) + require.NoError(t, err) + + content := `{"uid":"456","event_type":"azure.test"}` + "\n" + err = blobClient.Upload(context.Background(), "test/events.jsonl", strings.NewReader(content)) + require.NoError(t, err) + + // Download and verify via Azure client + resp, err := azClient.DownloadStream(context.Background(), "convoy-test-exports", "test/events.jsonl", nil) + require.NoError(t, err) + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, content, string(data)) +} + +func TestAzureBlobClient_Upload_WithPrefix(t *testing.T) { + if infra.NewAzuriteClient == nil { + t.Skip("Azurite not available") + } + + azClient, endpoint, err := (*infra.NewAzuriteClient)(t) + require.NoError(t, err) + + logger := log.New("test", log.LevelInfo) + blobClient, err := NewAzureBlobClient(BlobStoreOptions{ + AzureAccountName: "devstoreaccount1", + AzureAccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + AzureContainerName: "convoy-test-exports", + AzureEndpoint: endpoint, + Prefix: "prefixed", + }, logger) + require.NoError(t, err) + + err = blobClient.Upload(context.Background(), "data.jsonl", strings.NewReader("prefixed data")) + require.NoError(t, err) + + // Verify stored at "prefixed/data.jsonl" + resp, err := azClient.DownloadStream(context.Background(), "convoy-test-exports", "prefixed/data.jsonl", nil) + require.NoError(t, err) + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "prefixed data", string(data)) +} + +// ============================================================================ +// Factory Tests +// ============================================================================ + +func TestNewBlobStoreClient_S3(t *testing.T) { + if infra.NewMinIOClient == nil { + t.Skip("MinIO not available") + } + + _, endpoint, err := (*infra.NewMinIOClient)(t) + require.NoError(t, err) + + logger := log.New("test", log.LevelInfo) + client, err := NewBlobStoreClient(&datastore.StoragePolicyConfiguration{ + Type: datastore.S3, + S3: &datastore.S3Storage{ + Bucket: null.NewString("convoy-test-exports", true), + AccessKey: null.NewString("minioadmin", true), + SecretKey: null.NewString("minioadmin", true), + Region: null.NewString("us-east-1", true), + Endpoint: null.NewString("http://"+endpoint, true), + }, + }, logger) + require.NoError(t, err) + require.NotNil(t, client) + + // Verify it works + err = client.Upload(context.Background(), "factory-test/s3.txt", strings.NewReader("factory s3")) + require.NoError(t, err) +} + +func TestNewBlobStoreClient_OnPrem(t *testing.T) { + tmpDir := t.TempDir() + logger := log.New("test", log.LevelInfo) + + client, err := NewBlobStoreClient(&datastore.StoragePolicyConfiguration{ + Type: datastore.OnPrem, + OnPrem: &datastore.OnPremStorage{ + Path: null.NewString(tmpDir, true), + }, + }, logger) + require.NoError(t, err) + require.NotNil(t, client) + + err = client.Upload(context.Background(), "factory-test.txt", strings.NewReader("factory onprem")) + require.NoError(t, err) + + data, err := os.ReadFile(filepath.Join(tmpDir, "factory-test.txt")) + require.NoError(t, err) + require.Equal(t, "factory onprem", string(data)) +} + +func TestNewBlobStoreClient_Azure(t *testing.T) { + if infra.NewAzuriteClient == nil { + t.Skip("Azurite not available") + } + + _, endpoint, err := (*infra.NewAzuriteClient)(t) + require.NoError(t, err) + + logger := log.New("test", log.LevelInfo) + client, err := NewBlobStoreClient(&datastore.StoragePolicyConfiguration{ + Type: datastore.AzureBlob, + AzureBlob: &datastore.AzureBlobStorage{ + AccountName: null.NewString("devstoreaccount1", true), + AccountKey: null.NewString("Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", true), + ContainerName: null.NewString("convoy-test-exports", true), + Endpoint: null.NewString(endpoint, true), + }, + }, logger) + require.NoError(t, err) + require.NotNil(t, client) + + err = client.Upload(context.Background(), "factory-test/azure.txt", strings.NewReader("factory azure")) + require.NoError(t, err) +} + +func TestNewBlobStoreClient_InvalidType(t *testing.T) { + logger := log.New("test", log.LevelInfo) + _, err := NewBlobStoreClient(&datastore.StoragePolicyConfiguration{ + Type: "invalid", + }, logger) + require.Error(t, err) +} + +// Ensure imports are used +var _ = (*minio.Client)(nil) +var _ = (*azblob.Client)(nil) +var _ = (*bytes.Buffer)(nil) +var _ = credentials.NewStaticV4 diff --git a/internal/pkg/blob-store/onprem.go b/internal/pkg/blob-store/onprem.go new file mode 100644 index 0000000000..8ce5e92f8d --- /dev/null +++ b/internal/pkg/blob-store/onprem.go @@ -0,0 +1,70 @@ +package blobstore + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + log "github.com/frain-dev/convoy/pkg/logger" +) + +// OnPremClient implements BlobStore for local filesystem storage. +type OnPremClient struct { + opts BlobStoreOptions + logger log.Logger +} + +// NewOnPremClient creates a new on-prem filesystem BlobStore. +func NewOnPremClient(opts BlobStoreOptions, logger log.Logger) (BlobStore, error) { + return &OnPremClient{ + opts: opts, + logger: logger, + }, nil +} + +// Upload writes the stream to the local filesystem at the given key path. +func (o *OnPremClient) Upload(ctx context.Context, key string, r io.Reader) error { + baseDir := filepath.Clean(o.opts.OnPremStorageDir) + fullPath := filepath.Join(baseDir, filepath.Clean(key)) + + // Guard against path traversal (e.g. key = "../../etc/passwd") + if !strings.HasPrefix(fullPath, baseDir+string(filepath.Separator)) && fullPath != baseDir { + return fmt.Errorf("path traversal detected: %q resolves outside base directory", key) + } + + if err := os.MkdirAll(filepath.Dir(fullPath), 0o750); err != nil { + return fmt.Errorf("create directory for %q: %w", fullPath, err) + } + + f, err := os.Create(fullPath) + if err != nil { + return fmt.Errorf("create file %q: %w", fullPath, err) + } + defer f.Close() + + // Context-aware copy: check for cancellation during write + buf := make([]byte, 32*1024) + for { + if ctx.Err() != nil { + return ctx.Err() + } + n, readErr := r.Read(buf) + if n > 0 { + if _, writeErr := f.Write(buf[:n]); writeErr != nil { + return fmt.Errorf("write to %q: %w", fullPath, writeErr) + } + } + if readErr == io.EOF { + break + } + if readErr != nil { + return fmt.Errorf("read for %q: %w", fullPath, readErr) + } + } + + o.logger.Info(fmt.Sprintf("saved %q", fullPath)) + return nil +} diff --git a/internal/pkg/blob-store/s3.go b/internal/pkg/blob-store/s3.go new file mode 100644 index 0000000000..86dfb53ad2 --- /dev/null +++ b/internal/pkg/blob-store/s3.go @@ -0,0 +1,62 @@ +package blobstore + +import ( + "context" + "fmt" + "io" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + + log "github.com/frain-dev/convoy/pkg/logger" +) + +// S3Client implements BlobStore for AWS S3 (and S3-compatible) backends. +type S3Client struct { + session *session.Session + opts BlobStoreOptions + logger log.Logger +} + +// NewS3Client creates a new S3-backed BlobStore. +func NewS3Client(opts BlobStoreOptions, logger log.Logger) (BlobStore, error) { + sess, err := session.NewSession(&aws.Config{ + S3ForcePathStyle: aws.Bool(true), + Region: aws.String(opts.Region), + Endpoint: aws.String(opts.Endpoint), + Credentials: credentials.NewStaticCredentials(opts.AccessKey, opts.SecretKey, opts.SessionToken), + }) + if err != nil { + return nil, err + } + + return &S3Client{ + session: sess, + opts: opts, + logger: logger, + }, nil +} + +// Upload streams data directly to S3 via multipart upload. +func (s3c *S3Client) Upload(ctx context.Context, key string, r io.Reader) error { + name := key + if s3c.opts.Prefix != "" { + name = s3c.opts.Prefix + "/" + key + } + + uploader := s3manager.NewUploader(s3c.session) + _, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(s3c.opts.Bucket), + Key: aws.String(name), + Body: r, + }) + if err != nil { + s3c.logger.Error(fmt.Sprintf("failed to upload %q to %q: %v", name, s3c.opts.Bucket, err)) + return err + } + + s3c.logger.Info(fmt.Sprintf("uploaded %q to %q", name, s3c.opts.Bucket)) + return nil +} diff --git a/internal/pkg/exporter/exporter.go b/internal/pkg/exporter/exporter.go index 0e29ac3f88..82d1985752 100644 --- a/internal/pkg/exporter/exporter.go +++ b/internal/pkg/exporter/exporter.go @@ -1,6 +1,7 @@ package exporter import ( + "compress/gzip" "context" "errors" "fmt" @@ -10,7 +11,9 @@ import ( "time" "github.com/frain-dev/convoy" + "github.com/frain-dev/convoy/config" "github.com/frain-dev/convoy/datastore" + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" log "github.com/frain-dev/convoy/pkg/logger" ) @@ -19,12 +22,12 @@ var ( ErrInvalidExportDir = errors.New("invalid export directory") ) -type tablename string +type tableName string const ( - eventsTable tablename = "convoy.events" - eventDeliveriesTable tablename = "convoy.event_deliveries" - deliveryAttemptsTable tablename = "convoy.delivery_attempts" + eventsTable tableName = "convoy.events" + eventDeliveriesTable tableName = "convoy.event_deliveries" + deliveryAttemptsTable tableName = "convoy.delivery_attempts" ) // order is important here, @@ -32,16 +35,23 @@ const ( // event_deliveries references event id, // so delivery_attempts must be deleted first, // then event_deliveries then events. -var tables = []tablename{deliveryAttemptsTable, eventDeliveriesTable, eventsTable} +var tables = []tableName{deliveryAttemptsTable, eventDeliveriesTable, eventsTable} -var tableToFileMapping = map[tablename]string{ - eventsTable: "%s/orgs/%s/projects/%s/events/%s.json", - eventDeliveriesTable: "%s/orgs/%s/projects/%s/eventdeliveries/%s.json", - deliveryAttemptsTable: "%s/orgs/%s/projects/%s/deliveryattempts/%s.json", +var tableToBlobKeyMapping = map[tableName]string{ + eventsTable: "backup/%s/events/%s.jsonl.gz", + eventDeliveriesTable: "backup/%s/eventdeliveries/%s.jsonl.gz", + deliveryAttemptsTable: "backup/%s/deliveryattempts/%s.jsonl.gz", +} + +// tableToFileMapping is used by the disk-based Export() path. +var tableToFileMapping = map[tableName]string{ + eventsTable: "%s/backup/%s/events/%s.jsonl.gz", + eventDeliveriesTable: "%s/backup/%s/eventdeliveries/%s.jsonl.gz", + deliveryAttemptsTable: "%s/backup/%s/deliveryattempts/%s.jsonl.gz", } type ( - ExportResult map[tablename]ExportTableResult + ExportResult map[tableName]ExportTableResult ExportTableResult struct { NumDocs int64 ExportFile string @@ -49,50 +59,83 @@ type ( ) type Exporter struct { - config *datastore.Configuration - project *datastore.Project + config *datastore.Configuration - expDate time.Time - result ExportResult + expStart time.Time + expEnd time.Time + result ExportResult // repositories eventRepo datastore.EventRepository - projectRepo datastore.ProjectRepository eventDeliveryRepo datastore.EventDeliveryRepository deliveryAttemptsRepo datastore.DeliveryAttemptsRepository logger log.Logger } -func NewExporter(projectRepo datastore.ProjectRepository, +func NewExporter( eventRepo datastore.EventRepository, eventDeliveryRepo datastore.EventDeliveryRepository, - p *datastore.Project, c *datastore.Configuration, + c *datastore.Configuration, attemptsRepo datastore.DeliveryAttemptsRepository, logger log.Logger, ) (*Exporter, error) { + // Derive the look back duration from CONVOY_BACKUP_INTERVAL (defaults to 1h) + lookBackDur := DefaultBackupInterval + if cfg, err := config.Get(); err == nil { + lookBackDur = ParseBackupInterval(cfg.RetentionPolicy.BackupInterval) + } + return &Exporter{ - config: c, - project: p, - result: ExportResult{}, - expDate: time.Now().Add(-time.Hour * 24), + config: c, + result: ExportResult{}, + expEnd: time.Now().UTC(), + expStart: time.Now().UTC().Add(-lookBackDur), eventRepo: eventRepo, - projectRepo: projectRepo, deliveryAttemptsRepo: attemptsRepo, eventDeliveryRepo: eventDeliveryRepo, logger: logger, }, nil } +// NewExporterWithWindow creates an Exporter with an explicit time window, +// bypassing the config-derived backup interval. Used by manual/ad-hoc backups. +func NewExporterWithWindow( + eventRepo datastore.EventRepository, + eventDeliveryRepo datastore.EventDeliveryRepository, + c *datastore.Configuration, + attemptsRepo datastore.DeliveryAttemptsRepository, + start, end time.Time, + logger log.Logger, +) (*Exporter, error) { + if !start.Before(end) { + return nil, fmt.Errorf("invalid export window: start (%s) must be before end (%s)", + start.Format(time.RFC3339), end.Format(time.RFC3339)) + } + + return &Exporter{ + config: c, + result: ExportResult{}, + expEnd: end, + expStart: start, + + eventRepo: eventRepo, + deliveryAttemptsRepo: attemptsRepo, + eventDeliveryRepo: eventDeliveryRepo, + logger: logger, + }, nil +} + +// Export writes gzip-compressed JSONL files to disk. Used by the legacy +// file-based backup flow and E2E tests. func (ex *Exporter) Export(ctx context.Context) (ExportResult, error) { if !ex.config.RetentionPolicy.IsRetentionPolicyEnabled { return nil, nil } - // export tables for _, table := range tables { - result, err := ex.exportTable(ctx, table, ex.expDate) + result, err := ex.exportTableToDisk(ctx, table, ex.expStart, ex.expEnd) if err != nil { return nil, err } @@ -104,7 +147,85 @@ func (ex *Exporter) Export(ctx context.Context) (ExportResult, error) { return ex.result, nil } -func (ex *Exporter) exportTable(ctx context.Context, table tablename, expDate time.Time) (*ExportTableResult, error) { +// StreamExport exports all tables and streams gzip-compressed JSONL directly to +// the given BlobStore via io.Pipe, avoiding any local disk writes. +func (ex *Exporter) StreamExport(ctx context.Context, store blobstore.BlobStore) (ExportResult, error) { + if !ex.config.RetentionPolicy.IsRetentionPolicyEnabled { + return nil, nil + } + + result := ExportResult{} + + for _, table := range tables { + tableResult, err := ex.streamExportTable(ctx, store, table, ex.expStart, ex.expEnd) + if err != nil { + return nil, err + } + + result[table] = *tableResult + ex.logger.Info(fmt.Sprintf("streamed %v record(s) from %v", tableResult.NumDocs, table)) + } + + return result, nil +} + +// streamExportTable pipes ExportRecords → gzip → BlobStore.Upload without touching disk. +func (ex *Exporter) streamExportTable(ctx context.Context, store blobstore.BlobStore, table tableName, expStart, expEnd time.Time) (*ExportTableResult, error) { + keyFormat, ok := tableToBlobKeyMapping[table] + if !ok { + return nil, ErrInvalidTable + } + + now := time.Now().UTC() + date := now.Format("2006-01-02") + ts := now.Format(time.RFC3339) + key := fmt.Sprintf(keyFormat, date, ts) + + exportRepo, err := ex.getRepo(table) + if err != nil { + return nil, err + } + + pr, pw := io.Pipe() + exportCtx, cancelExport := context.WithCancel(ctx) + defer cancelExport() + + var numDocs int64 + errCh := make(chan error, 1) + + go func() { + gzw := gzip.NewWriter(pw) + + n, exportErr := exportRepo.ExportRecords(exportCtx, expStart, expEnd, gzw) + numDocs = n + + // MUST close gzip before pipe — flush trailer (checksum + size) + if closeErr := gzw.Close(); closeErr != nil && exportErr == nil { + exportErr = closeErr + } + if exportErr != nil { + pw.CloseWithError(exportErr) + } else { + pw.Close() + } + errCh <- exportErr + }() + + uploadErr := store.Upload(ctx, key, pr) + exportErr := <-errCh + + if uploadErr != nil { + return nil, fmt.Errorf("upload %q: %w", key, uploadErr) + } + if exportErr != nil { + return nil, fmt.Errorf("export %q: %w", key, exportErr) + } + + return &ExportTableResult{NumDocs: numDocs, ExportFile: key}, nil +} + +// exportTableToDisk writes gzip-compressed JSONL to a local file (legacy path). +func (ex *Exporter) exportTableToDisk(ctx context.Context, table tableName, expStart, expEnd time.Time) (*ExportTableResult, error) { result := &ExportTableResult{} exportFileFormat, ok := tableToFileMapping[table] if !ok { @@ -116,26 +237,34 @@ func (ex *Exporter) exportTable(ctx context.Context, table tablename, expDate ti return result, err } - now := time.Now().UTC().Format(time.RFC3339) - exportFile := fmt.Sprintf(exportFileFormat, exportDir, ex.project.OrganisationID, ex.project.UID, now) + now := time.Now().UTC() + date := now.Format("2006-01-02") + ts := now.Format(time.RFC3339) + exportFile := fmt.Sprintf(exportFileFormat, exportDir, date, ts) - writer, err := getOutputWriter(exportFile) + fileWriter, err := getOutputWriter(exportFile) if err != nil { return result, err } + defer func(fileWriter io.WriteCloser) { + if err = fileWriter.Close(); err != nil { + ex.logger.Error("failed to close file writer", "error", err) + } + }(fileWriter) - if writer == nil { - writer = os.Stdout - } else { - defer writer.Close() - } + gzw := gzip.NewWriter(fileWriter) + defer func(gzw *gzip.Writer) { + if err = gzw.Close(); err != nil { + ex.logger.Error("failed to close gzip writer", "error", err) + } + }(gzw) - repo, err := ex.getRepo(table) + exportRepo, err := ex.getRepo(table) if err != nil { return result, err } - numDocs, err := repo.ExportRecords(ctx, ex.project.UID, expDate, writer) + numDocs, err := exportRepo.ExportRecords(ctx, expStart, expEnd, gzw) if err != nil { ex.logger.Error("failed to export records", "error", err) return result, err @@ -162,7 +291,7 @@ func (ex *Exporter) getExportDir() (string, error) { } } -func (ex *Exporter) getRepo(table tablename) (datastore.ExportRepository, error) { +func (ex *Exporter) getRepo(table tableName) (datastore.ExportRepository, error) { switch table { case eventsTable: return ex.eventRepo, nil @@ -175,42 +304,17 @@ func (ex *Exporter) getRepo(table tablename) (datastore.ExportRepository, error) } } -// GetOutputWriter opens and returns an io.WriteCloser for the output -// options or nil if none is set. The caller is responsible for closing it. func getOutputWriter(out string) (io.WriteCloser, error) { - // If the directory in which the output file is to be - // written does not exist, create it fileDir := filepath.Dir(out) err := os.MkdirAll(fileDir, 0o750) if err != nil { return nil, err } - file, err := os.Create(toUniversalPath(out)) + file, err := os.Create(filepath.FromSlash(out)) if err != nil { return nil, err } return file, err } - -// type compressWriter struct { -// gw *gzip.Writer -// file *os.File -// } -// -// func (c compressWriter) Write(b []byte) (int, error) { -// return c.gw.Write(b) -// } -// -// func (c compressWriter) Close() error { -// if err := c.gw.Close(); err != nil { -// return err -// } -// -// return c.file.Close() -// } - -func toUniversalPath(path string) string { - return filepath.FromSlash(path) -} diff --git a/internal/pkg/exporter/interval.go b/internal/pkg/exporter/interval.go new file mode 100644 index 0000000000..c69a9855ca --- /dev/null +++ b/internal/pkg/exporter/interval.go @@ -0,0 +1,52 @@ +package exporter + +import ( + "fmt" + "time" +) + +const DefaultBackupInterval = time.Hour + +// ParseBackupInterval parses a duration string into a time.Duration, +// falling back to DefaultBackupInterval on error or empty input. +func ParseBackupInterval(s string) time.Duration { + if s == "" { + return DefaultBackupInterval + } + d, err := time.ParseDuration(s) + if err != nil || d <= 0 { + return DefaultBackupInterval + } + return d +} + +// DurationToCron converts a time.Duration into a cron spec string. +// Sub-hour durations produce minute-level cron (e.g. */5 * * * *). +// Hour-or-above durations produce hour-level cron (e.g. 0 */6 * * *). +func DurationToCron(d time.Duration) string { + return durationToCronWithOffset(d, 0) +} + +// DurationToCronOffset returns a cron spec offset by the given minutes. +// Used to stagger tasks that depend on each other (e.g. enqueue at :00, process at :01). +func DurationToCronOffset(d time.Duration, offsetMinutes int) string { + return durationToCronWithOffset(d, offsetMinutes) +} + +func durationToCronWithOffset(d time.Duration, offset int) string { + minutes := int(d.Minutes()) + switch { + case minutes <= 0: + return fmt.Sprintf("%d * * * *", 5+offset) // fallback: hourly + case minutes < 60: + return fmt.Sprintf("%d-59/%d * * * *", offset, minutes) + case minutes == 60: + return fmt.Sprintf("%d * * * *", offset) // hourly + default: + hours := int(d.Hours()) + if hours <= 0 { + hours = 1 + } + return fmt.Sprintf("%d */%d * * *", offset, hours) + } +} diff --git a/internal/pkg/object-store/objectstore.go b/internal/pkg/object-store/objectstore.go deleted file mode 100644 index 71dcd83250..0000000000 --- a/internal/pkg/object-store/objectstore.go +++ /dev/null @@ -1,57 +0,0 @@ -package objectstore - -import ( - "errors" - - "github.com/frain-dev/convoy/datastore" - log "github.com/frain-dev/convoy/pkg/logger" -) - -type ObjectStore interface { - Save(string) error -} - -type ObjectStoreOptions struct { - Prefix string - Bucket string - AccessKey string - SecretKey string - Region string - Endpoint string - SessionToken string - OnPremStorageDir string -} - -func NewObjectStoreClient(storage *datastore.StoragePolicyConfiguration, logger log.Logger) (ObjectStore, error) { - switch storage.Type { - case datastore.S3: - objectStoreOpts := ObjectStoreOptions{ - Prefix: storage.S3.Prefix.ValueOrZero(), - Bucket: storage.S3.Bucket.ValueOrZero(), - Endpoint: storage.S3.Endpoint.ValueOrZero(), - AccessKey: storage.S3.AccessKey.ValueOrZero(), - SecretKey: storage.S3.SecretKey.ValueOrZero(), - SessionToken: storage.S3.SessionToken.ValueOrZero(), - Region: storage.S3.Region.ValueOrZero(), - } - - objectStoreClient, err := NewS3Client(objectStoreOpts, logger) - if err != nil { - return nil, err - } - return objectStoreClient, nil - - case datastore.OnPrem: - exportDir := storage.OnPrem.Path - objectStoreOpts := ObjectStoreOptions{ - OnPremStorageDir: exportDir.String, - } - objectStoreClient, err := NewOnPremClient(objectStoreOpts, logger) - if err != nil { - return nil, err - } - return objectStoreClient, nil - default: - return nil, errors.New("invalid storage policy") - } -} diff --git a/internal/pkg/object-store/onprem.go b/internal/pkg/object-store/onprem.go deleted file mode 100644 index 40def9255d..0000000000 --- a/internal/pkg/object-store/onprem.go +++ /dev/null @@ -1,29 +0,0 @@ -package objectstore - -import ( - "fmt" - "os" - - log "github.com/frain-dev/convoy/pkg/logger" -) - -type OnPremClient struct { - opts ObjectStoreOptions - logger log.Logger -} - -func NewOnPremClient(opts ObjectStoreOptions, logger log.Logger) (ObjectStore, error) { - client := &OnPremClient{ - opts: opts, - logger: logger, - } - return client, nil -} - -func (o *OnPremClient) Save(filename string) error { - if _, err := os.Stat(filename); err != nil { - return err - } - o.logger.Info(fmt.Sprintf("Successfully saved %q \n", filename)) - return nil -} diff --git a/internal/pkg/object-store/s3.go b/internal/pkg/object-store/s3.go deleted file mode 100644 index 7fb3b274b3..0000000000 --- a/internal/pkg/object-store/s3.go +++ /dev/null @@ -1,77 +0,0 @@ -package objectstore - -import ( - "fmt" - "os" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - - log "github.com/frain-dev/convoy/pkg/logger" - "github.com/frain-dev/convoy/util" -) - -type S3Client struct { - session *session.Session - opts ObjectStoreOptions - logger log.Logger -} - -func NewS3Client(opts ObjectStoreOptions, logger log.Logger) (ObjectStore, error) { - sess, err := session.NewSession(&aws.Config{ - S3ForcePathStyle: aws.Bool(true), - Region: aws.String(opts.Region), - Endpoint: aws.String(opts.Endpoint), - Credentials: credentials.NewStaticCredentials(opts.AccessKey, opts.SecretKey, opts.SessionToken), - }) - if err != nil { - return nil, err - } - - client := &S3Client{ - session: sess, - opts: opts, - logger: logger, - } - - return client, nil -} - -func (s3 *S3Client) Save(filename string) error { - file, err := os.Open(filename) - if err != nil { - s3.logger.Error(fmt.Sprintf("Unable to open file %q, %v: %v", filename, err, err)) - return err - } - - defer file.Close() - - name := filename - - if util.IsStringEmpty(s3.opts.Prefix) { - names := strings.Split(filename, "/tmp/") - if len(names) > 1 { - name = names[1] - } - } else { - name = strings.Replace(filename, "/tmp", s3.opts.Prefix, 1) - } - - uploader := s3manager.NewUploader(s3.session) - _, err = uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(s3.opts.Bucket), - Key: aws.String(name), - Body: file, - }) - - if err != nil { - s3.logger.Error(fmt.Sprintf("Unable to save %q to %q, %v: %v", filename, s3.opts.Bucket, err, err)) - return err - } - - s3.logger.Info(fmt.Sprintf("Successfully saved %q to %q\n", filename, s3.opts.Bucket)) - return nil -} diff --git a/internal/projects/repo/queries.sql.go b/internal/projects/repo/queries.sql.go index 6447447916..a6e16bbce1 100644 --- a/internal/projects/repo/queries.sql.go +++ b/internal/projects/repo/queries.sql.go @@ -658,14 +658,14 @@ type UpdateProjectEndpointStatusParams struct { type UpdateProjectEndpointStatusRow struct { ID string - Name string + Name pgtype.Text Status string OwnerID pgtype.Text - Url string + Url pgtype.Text Description pgtype.Text - HttpTimeout int32 + HttpTimeout pgtype.Int4 RateLimit int32 - RateLimitDuration int32 + RateLimitDuration pgtype.Int4 AdvancedSignatures bool SlackWebhookUrl pgtype.Text SupportEmail pgtype.Text diff --git a/mise.toml b/mise.toml index a176c58ed5..384268058c 100644 --- a/mise.toml +++ b/mise.toml @@ -8,7 +8,7 @@ go = "1.25.7" golangci-lint = "2.4.0" nodejs = "24.10.0" -# postgres = "15.1" + postgres = "18" "asdf:jonathanmorley/asdf-pre-commit" = "latest" "aqua:pvolok/mprocs" = "0.7.2" diff --git a/mocks/repository.go b/mocks/repository.go index 9aa397fa7c..dee213278d 100644 --- a/mocks/repository.go +++ b/mocks/repository.go @@ -118,18 +118,18 @@ func (mr *MockEventDeliveryRepositoryMockRecorder) DeleteProjectEventDeliveries( } // ExportRecords mocks base method. -func (m *MockEventDeliveryRepository) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { +func (m *MockEventDeliveryRepository) ExportRecords(ctx context.Context, start time.Time, end time.Time, w io.Writer) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExportRecords", ctx, projectID, createdAt, w) + ret := m.ctrl.Call(m, "ExportRecords", ctx, start, end, w) ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } // ExportRecords indicates an expected call of ExportRecords. -func (mr *MockEventDeliveryRepositoryMockRecorder) ExportRecords(ctx, projectID, createdAt, w any) *gomock.Call { +func (mr *MockEventDeliveryRepositoryMockRecorder) ExportRecords(ctx, start, end, w any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockEventDeliveryRepository)(nil).ExportRecords), ctx, projectID, createdAt, w) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockEventDeliveryRepository)(nil).ExportRecords), ctx, start, end, w) } // FindDiscardedEventDeliveries mocks base method. @@ -434,18 +434,18 @@ func (mr *MockEventRepositoryMockRecorder) DeleteProjectTokenizedEvents(ctx, pro } // ExportRecords mocks base method. -func (m *MockEventRepository) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { +func (m *MockEventRepository) ExportRecords(ctx context.Context, start time.Time, end time.Time, w io.Writer) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExportRecords", ctx, projectID, createdAt, w) + ret := m.ctrl.Call(m, "ExportRecords", ctx, start, end, w) ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } // ExportRecords indicates an expected call of ExportRecords. -func (mr *MockEventRepositoryMockRecorder) ExportRecords(ctx, projectID, createdAt, w any) *gomock.Call { +func (mr *MockEventRepositoryMockRecorder) ExportRecords(ctx, start, end, w any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockEventRepository)(nil).ExportRecords), ctx, projectID, createdAt, w) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockEventRepository)(nil).ExportRecords), ctx, start, end, w) } // FindEventByID mocks base method. @@ -2826,18 +2826,18 @@ func (m *MockExportRepository) EXPECT() *MockExportRepositoryMockRecorder { } // ExportRecords mocks base method. -func (m *MockExportRepository) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { +func (m *MockExportRepository) ExportRecords(ctx context.Context, start time.Time, end time.Time, w io.Writer) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExportRecords", ctx, projectID, createdAt, w) + ret := m.ctrl.Call(m, "ExportRecords", ctx, start, end, w) ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } // ExportRecords indicates an expected call of ExportRecords. -func (mr *MockExportRepositoryMockRecorder) ExportRecords(ctx, projectID, createdAt, w any) *gomock.Call { +func (mr *MockExportRepositoryMockRecorder) ExportRecords(ctx, start, end, w any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockExportRepository)(nil).ExportRecords), ctx, projectID, createdAt, w) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockExportRepository)(nil).ExportRecords), ctx, start, end, w) } // MockDeliveryAttemptsRepository is a mock of DeliveryAttemptsRepository interface. @@ -2893,18 +2893,18 @@ func (mr *MockDeliveryAttemptsRepositoryMockRecorder) DeleteProjectDeliveriesAtt } // ExportRecords mocks base method. -func (m *MockDeliveryAttemptsRepository) ExportRecords(ctx context.Context, projectID string, createdAt time.Time, w io.Writer) (int64, error) { +func (m *MockDeliveryAttemptsRepository) ExportRecords(ctx context.Context, start time.Time, end time.Time, w io.Writer) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ExportRecords", ctx, projectID, createdAt, w) + ret := m.ctrl.Call(m, "ExportRecords", ctx, start, end, w) ret0, _ := ret[0].(int64) ret1, _ := ret[1].(error) return ret0, ret1 } // ExportRecords indicates an expected call of ExportRecords. -func (mr *MockDeliveryAttemptsRepositoryMockRecorder) ExportRecords(ctx, projectID, createdAt, w any) *gomock.Call { +func (mr *MockDeliveryAttemptsRepositoryMockRecorder) ExportRecords(ctx, start, end, w any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockDeliveryAttemptsRepository)(nil).ExportRecords), ctx, projectID, createdAt, w) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExportRecords", reflect.TypeOf((*MockDeliveryAttemptsRepository)(nil).ExportRecords), ctx, start, end, w) } // FindDeliveryAttemptById mocks base method. diff --git a/pkg/cachedrepo/cachedrepo.go b/pkg/cachedrepo/cachedrepo.go new file mode 100644 index 0000000000..60ca48d501 --- /dev/null +++ b/pkg/cachedrepo/cachedrepo.go @@ -0,0 +1,149 @@ +package cachedrepo + +import ( + "context" + "time" +) + +// Cache is a minimal cache interface. +// Get must return nil (not an error) on cache miss. +type Cache interface { + Set(ctx context.Context, key string, data interface{}, ttl time.Duration) error + Get(ctx context.Context, key string, data interface{}) error + Delete(ctx context.Context, key string) error +} + +// Logger is a minimal structured logger interface. +type Logger interface { + Error(args ...any) +} + +// sliceWrapper distinguishes "empty result cached" from "cache miss" for slice types. +type sliceWrapper[T any] struct { + Items []T +} + +// foundWrapper distinguishes "cache miss" from "cached not-found" for single-entity lookups. +type foundWrapper[T any] struct { + Value *T + Found bool +} + +// FetchOne performs read-through caching for single-entity lookups. +// hitCheck returns true if the cached value is populated (e.g., entity.UID != ""). +// On cache miss, calls fetch, caches the result, and returns it. +// Cache errors are logged but never propagated. +func FetchOne[T any]( + ctx context.Context, ca Cache, logger Logger, + key string, ttl time.Duration, + hitCheck func(*T) bool, + fetch func() (*T, error), +) (*T, error) { + var result T + err := ca.Get(ctx, key, &result) + if err != nil { + logger.Error("cache get error", "key", key, "error", err) + } + + if hitCheck(&result) { + return &result, nil + } + + val, err := fetch() + if err != nil { + return nil, err + } + + if setErr := ca.Set(ctx, key, val, ttl); setErr != nil { + logger.Error("cache set error", "key", key, "error", setErr) + } + + return val, nil +} + +// FetchSlice performs read-through caching for slice lookups. +// Internally wraps the slice to distinguish "empty result cached" from "cache miss". +// Cache errors are logged but never propagated. +func FetchSlice[T any]( + ctx context.Context, ca Cache, logger Logger, + key string, ttl time.Duration, + fetch func() ([]T, error), +) ([]T, error) { + var cached sliceWrapper[T] + err := ca.Get(ctx, key, &cached) + if err != nil { + logger.Error("cache get error", "key", key, "error", err) + } + + if cached.Items != nil { + return cached.Items, nil + } + + items, err := fetch() + if err != nil { + return nil, err + } + + toCache := sliceWrapper[T]{Items: items} + if setErr := ca.Set(ctx, key, &toCache, ttl); setErr != nil { + logger.Error("cache set error", "key", key, "error", setErr) + } + + return items, nil +} + +// FetchWithNotFound is like FetchOne but also caches not-found results. +// When fetch returns an error where isNotFound(err) is true, the not-found +// is cached so subsequent calls skip the DB. Returns (nil, original error) on not-found. +// Cache errors are logged but never propagated. +func FetchWithNotFound[T any]( + ctx context.Context, ca Cache, logger Logger, + key string, ttl time.Duration, + fetch func() (*T, error), + isNotFound func(error) bool, + notFoundErr error, +) (*T, error) { + var cached foundWrapper[T] + err := ca.Get(ctx, key, &cached) + if err != nil { + logger.Error("cache get error", "key", key, "error", err) + } + + if cached.Found { + if cached.Value == nil { + return nil, notFoundErr + } + return cached.Value, nil + } + + val, err := fetch() + if err != nil { + if isNotFound(err) { + toCache := foundWrapper[T]{Value: nil, Found: true} + if setErr := ca.Set(ctx, key, &toCache, ttl); setErr != nil { + logger.Error("cache set error", "key", key, "error", setErr) + } + } + return nil, err + } + + toCache := foundWrapper[T]{Value: val, Found: true} + if setErr := ca.Set(ctx, key, &toCache, ttl); setErr != nil { + logger.Error("cache set error", "key", key, "error", setErr) + } + + return val, nil +} + +// Invalidate deletes one or more cache keys. Empty keys are skipped. +// Errors are logged but never propagated. +func Invalidate(ctx context.Context, ca Cache, logger Logger, keys ...string) { + for _, key := range keys { + if key == "" { + continue + } + if err := ca.Delete(ctx, key); err != nil { + logger.Error("cache delete error", "key", key, "error", err) + } + } +} diff --git a/pkg/cachedrepo/cachedrepo_test.go b/pkg/cachedrepo/cachedrepo_test.go new file mode 100644 index 0000000000..79c1626fbd --- /dev/null +++ b/pkg/cachedrepo/cachedrepo_test.go @@ -0,0 +1,341 @@ +package cachedrepo + +import ( + "context" + "errors" + "testing" + "time" +) + +// --- test mocks --- + +type mockLogger struct{} + +func (m *mockLogger) Error(args ...any) {} + +type mockCache struct { + data map[string]interface{} + getErr error + setErr error + delErr error + + getCalled int + setCalled int + deleteCalled int + setFunc func(key string, data interface{}) // optional hook +} + +func newMockCache() *mockCache { + return &mockCache{data: make(map[string]interface{})} +} + +func (m *mockCache) Get(_ context.Context, key string, data interface{}) error { + m.getCalled++ + if m.getErr != nil { + return m.getErr + } + // no-op: data stays zero-valued (simulates cache miss) + return nil +} + +func (m *mockCache) Set(_ context.Context, key string, data interface{}, _ time.Duration) error { + m.setCalled++ + if m.setFunc != nil { + m.setFunc(key, data) + } + if m.setErr != nil { + return m.setErr + } + m.data[key] = data + return nil +} + +func (m *mockCache) Delete(_ context.Context, key string) error { + m.deleteCalled++ + if m.delErr != nil { + return m.delErr + } + delete(m.data, key) + return nil +} + +// --- test entity --- + +type testEntity struct { + UID string + Name string +} + +// --- FetchOne tests --- + +func TestFetchOne_CacheMiss(t *testing.T) { + ca := newMockCache() + logger := &mockLogger{} + entity := &testEntity{UID: "123", Name: "test"} + + result, err := FetchOne(context.Background(), ca, logger, "key:123", time.Minute, + func(e *testEntity) bool { return e.UID != "" }, + func() (*testEntity, error) { return entity, nil }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.UID != "123" { + t.Fatalf("expected UID 123, got %s", result.UID) + } + if ca.setCalled != 1 { + t.Fatalf("expected Set to be called once, got %d", ca.setCalled) + } +} + +func TestFetchOne_CacheHit(t *testing.T) { + ca := &hitCache{val: testEntity{UID: "123", Name: "cached"}} + logger := &mockLogger{} + fetchCalled := false + + result, err := FetchOne(context.Background(), ca, logger, "key:123", time.Minute, + func(e *testEntity) bool { return e.UID != "" }, + func() (*testEntity, error) { + fetchCalled = true + return nil, errors.New("should not be called") + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.UID != "123" || result.Name != "cached" { + t.Fatalf("unexpected result: %+v", result) + } + if fetchCalled { + t.Fatal("fetch should not have been called on cache hit") + } +} + +func TestFetchOne_CacheError_FallsThrough(t *testing.T) { + ca := newMockCache() + ca.getErr = errors.New("redis down") + logger := &mockLogger{} + entity := &testEntity{UID: "123"} + + result, err := FetchOne(context.Background(), ca, logger, "key:123", time.Minute, + func(e *testEntity) bool { return e.UID != "" }, + func() (*testEntity, error) { return entity, nil }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.UID != "123" { + t.Fatalf("expected UID 123, got %s", result.UID) + } +} + +func TestFetchOne_DBError(t *testing.T) { + ca := newMockCache() + logger := &mockLogger{} + + result, err := FetchOne(context.Background(), ca, logger, "key:123", time.Minute, + func(e *testEntity) bool { return e.UID != "" }, + func() (*testEntity, error) { return nil, errors.New("db error") }) + + if err == nil { + t.Fatal("expected error") + } + if result != nil { + t.Fatal("expected nil result on DB error") + } + if ca.setCalled != 0 { + t.Fatal("Set should not be called on DB error") + } +} + +// --- FetchSlice tests --- + +func TestFetchSlice_CacheMiss(t *testing.T) { + ca := newMockCache() + logger := &mockLogger{} + items := []testEntity{{UID: "1"}, {UID: "2"}} + + result, err := FetchSlice(context.Background(), ca, logger, "key:list", time.Minute, + func() ([]testEntity, error) { return items, nil }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 items, got %d", len(result)) + } + if ca.setCalled != 1 { + t.Fatalf("expected Set called once, got %d", ca.setCalled) + } +} + +func TestFetchSlice_CacheHit(t *testing.T) { + ca := &hitCache{val: sliceWrapper[testEntity]{Items: []testEntity{{UID: "cached"}}}} + logger := &mockLogger{} + fetchCalled := false + + result, err := FetchSlice(context.Background(), ca, logger, "key:list", time.Minute, + func() ([]testEntity, error) { + fetchCalled = true + return nil, errors.New("should not be called") + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result) != 1 || result[0].UID != "cached" { + t.Fatalf("unexpected result: %+v", result) + } + if fetchCalled { + t.Fatal("fetch should not be called on cache hit") + } +} + +func TestFetchSlice_CacheHitEmptySlice(t *testing.T) { + ca := &hitCache{val: sliceWrapper[testEntity]{Items: []testEntity{}}} + logger := &mockLogger{} + + result, err := FetchSlice(context.Background(), ca, logger, "key:list", time.Minute, + func() ([]testEntity, error) { return nil, errors.New("should not be called") }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result) != 0 { + t.Fatalf("expected empty slice, got %d items", len(result)) + } +} + +// --- FetchWithNotFound tests --- + +var errNotFound = errors.New("not found") + +func TestFetchWithNotFound_CacheMiss_Found(t *testing.T) { + ca := newMockCache() + logger := &mockLogger{} + entity := &testEntity{UID: "123"} + + result, err := FetchWithNotFound(context.Background(), ca, logger, "key:123", time.Minute, + func() (*testEntity, error) { return entity, nil }, + func(err error) bool { return errors.Is(err, errNotFound) }, + errNotFound) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.UID != "123" { + t.Fatalf("expected UID 123, got %s", result.UID) + } + if ca.setCalled != 1 { + t.Fatalf("expected Set called once, got %d", ca.setCalled) + } +} + +func TestFetchWithNotFound_CacheMiss_NotFound_Cached(t *testing.T) { + ca := newMockCache() + logger := &mockLogger{} + + result, err := FetchWithNotFound(context.Background(), ca, logger, "key:123", time.Minute, + func() (*testEntity, error) { return nil, errNotFound }, + func(err error) bool { return errors.Is(err, errNotFound) }, + errNotFound) + + if !errors.Is(err, errNotFound) { + t.Fatalf("expected errNotFound, got %v", err) + } + if result != nil { + t.Fatal("expected nil result") + } + if ca.setCalled != 1 { + t.Fatalf("expected Set called once (caching not-found), got %d", ca.setCalled) + } +} + +func TestFetchWithNotFound_CacheHit_Found(t *testing.T) { + entity := &testEntity{UID: "123"} + ca := &hitCache{val: foundWrapper[testEntity]{Value: entity, Found: true}} + logger := &mockLogger{} + + result, err := FetchWithNotFound(context.Background(), ca, logger, "key:123", time.Minute, + func() (*testEntity, error) { return nil, errors.New("should not be called") }, + func(err error) bool { return false }, + errNotFound) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.UID != "123" { + t.Fatalf("expected UID 123, got %s", result.UID) + } +} + +func TestFetchWithNotFound_CacheHit_CachedNotFound(t *testing.T) { + ca := &hitCache{val: foundWrapper[testEntity]{Value: nil, Found: true}} + logger := &mockLogger{} + + result, err := FetchWithNotFound(context.Background(), ca, logger, "key:123", time.Minute, + func() (*testEntity, error) { return nil, errors.New("should not be called") }, + func(err error) bool { return false }, + errNotFound) + + if !errors.Is(err, errNotFound) { + t.Fatalf("expected errNotFound, got %v", err) + } + if result != nil { + t.Fatal("expected nil result for cached not-found") + } +} + +// --- Invalidate tests --- + +func TestInvalidate(t *testing.T) { + ca := newMockCache() + logger := &mockLogger{} + + Invalidate(context.Background(), ca, logger, "key:1", "key:2", "", "key:3") + + if ca.deleteCalled != 3 { + t.Fatalf("expected 3 Delete calls (skipping empty), got %d", ca.deleteCalled) + } +} + +func TestInvalidate_ErrorLogged(t *testing.T) { + ca := newMockCache() + ca.delErr = errors.New("delete failed") + logger := &mockLogger{} + + // Should not panic or return error + Invalidate(context.Background(), ca, logger, "key:1") + + if ca.deleteCalled != 1 { + t.Fatalf("expected 1 Delete call, got %d", ca.deleteCalled) + } +} + +// --- hitCache simulates a cache that returns a populated value on Get --- + +type hitCache struct { + val interface{} +} + +func (h *hitCache) Get(_ context.Context, _ string, data interface{}) error { + // Use type switch to populate the data pointer with the stored value. + switch d := data.(type) { + case *testEntity: + if v, ok := h.val.(testEntity); ok { + *d = v + } + case *sliceWrapper[testEntity]: + if v, ok := h.val.(sliceWrapper[testEntity]); ok { + *d = v + } + case *foundWrapper[testEntity]: + if v, ok := h.val.(foundWrapper[testEntity]); ok { + *d = v + } + } + return nil +} + +func (h *hitCache) Set(context.Context, string, interface{}, time.Duration) error { return nil } +func (h *hitCache) Delete(context.Context, string) error { return nil } diff --git a/scripts/stress-test-backup.sh b/scripts/stress-test-backup.sh new file mode 100755 index 0000000000..ca7b716509 --- /dev/null +++ b/scripts/stress-test-backup.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# +# Stress Test: 1M Events + 5-Minute Backup to 3 Backends +# +# Usage: +# 1. Start infrastructure: ./scripts/stress-test-backup.sh infra-up +# 2. Set backend env vars (see below), start Convoy, then: +# 3. Generate events: ./scripts/stress-test-backup.sh generate +# 4. Wait 15-20 minutes for backup cycles +# 5. Verify results: ./scripts/stress-test-backup.sh verify +# 6. Tear down: ./scripts/stress-test-backup.sh infra-down +# +# Required env vars for event generation: +# CONVOY_BASE_URL - Convoy API URL (default: http://localhost:5005) +# CONVOY_PROJECT_ID - Target project ID +# CONVOY_API_KEY - API key with project admin access +# CONVOY_ENDPOINT_ID - Target endpoint ID +# +# Backend env vars (set ONE group before starting Convoy): +# +# --- MinIO (S3) --- +# CONVOY_STORAGE_POLICY_TYPE=s3 +# CONVOY_STORAGE_AWS_BUCKET=convoy-stress-test +# CONVOY_STORAGE_AWS_ACCESS_KEY=minioadmin +# CONVOY_STORAGE_AWS_SECRET_KEY=minioadmin +# CONVOY_STORAGE_AWS_REGION=us-east-1 +# CONVOY_STORAGE_AWS_ENDPOINT=http://localhost:9000 +# +# --- On-Prem --- +# CONVOY_STORAGE_POLICY_TYPE=on_prem +# CONVOY_STORAGE_PREM_PATH=/tmp/convoy-stress-test +# +# --- Azure Blob --- +# CONVOY_STORAGE_POLICY_TYPE=azure_blob +# CONVOY_STORAGE_AZURE_ACCOUNT_NAME= +# CONVOY_STORAGE_AZURE_ACCOUNT_KEY= +# CONVOY_STORAGE_AZURE_CONTAINER_NAME= +# CONVOY_STORAGE_AZURE_ENDPOINT= +# +# Always set these for backup: +# CONVOY_RETENTION_POLICY_ENABLED=true +# CONVOY_RETENTION_POLICY=720h +# CONVOY_BACKUP_INTERVAL=5m + +set -euo pipefail + +CONVOY_BASE_URL="${CONVOY_BASE_URL:-http://localhost:5005}" +BENCH_DIR="${BENCH_DIR:-/Users/rtukpe/Documents/dev/frain/convoy-bench}" +MINIO_CONTAINER="convoy-stress-minio" +AZURITE_CONTAINER="convoy-stress-azurite" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log() { echo -e "${GREEN}[stress-test]${NC} $*"; } +warn() { echo -e "${YELLOW}[stress-test]${NC} $*"; } +err() { echo -e "${RED}[stress-test]${NC} $*" >&2; } + +cmd_infra_up() { + log "Starting MinIO..." + docker run -d --name "$MINIO_CONTAINER" \ + -p 9000:9000 -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio:RELEASE.2024-01-16T16-07-38Z server /data --console-address ":9001" \ + 2>/dev/null || warn "MinIO container already exists" + + log "Waiting for MinIO to be ready..." + sleep 3 + + log "Creating MinIO bucket..." + docker run --rm --network host \ + --entrypoint /bin/sh minio/mc -c \ + "mc alias set local http://localhost:9000 minioadmin minioadmin && mc mb --ignore-existing local/convoy-stress-test" \ + 2>/dev/null || true + + log "Starting Azurite..." + docker run -d --name "$AZURITE_CONTAINER" \ + -p 10000:10000 \ + mcr.microsoft.com/azure-storage/azurite:3.31.0 \ + azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck \ + 2>/dev/null || warn "Azurite container already exists" + + log "Creating on-prem directory..." + mkdir -p /tmp/convoy-stress-test + + log "Infrastructure ready." + log " MinIO: http://localhost:9000 (console: http://localhost:9001)" + log " Azurite: http://localhost:10000" + log " On-Prem: /tmp/convoy-stress-test" +} + +cmd_infra_down() { + log "Stopping containers..." + docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true + docker rm -f "$AZURITE_CONTAINER" 2>/dev/null || true + rm -rf /tmp/convoy-stress-test + log "Infrastructure torn down." +} + +cmd_generate() { + if [ -z "${CONVOY_PROJECT_ID:-}" ] || [ -z "${CONVOY_API_KEY:-}" ] || [ -z "${CONVOY_ENDPOINT_ID:-}" ]; then + err "Missing required env vars: CONVOY_PROJECT_ID, CONVOY_API_KEY, CONVOY_ENDPOINT_ID" + exit 1 + fi + + if [ ! -f "$BENCH_DIR/bench.sh" ]; then + err "convoy-bench not found at $BENCH_DIR" + exit 1 + fi + + log "Generating 1M events via convoy-bench..." + log " Target: $CONVOY_BASE_URL" + log " Project: $CONVOY_PROJECT_ID" + log " Endpoint: $CONVOY_ENDPOINT_ID" + log " Rate: 5000 req/s for 200s = ~1M events" + + cd "$BENCH_DIR" + ./bench.sh -p http -t outgoing \ + -u "$CONVOY_BASE_URL" \ + -v 100 -r 5000 -d 200s \ + --endpoint-id "$CONVOY_ENDPOINT_ID" \ + --project-id "$CONVOY_PROJECT_ID" \ + --api-key "$CONVOY_API_KEY" + + log "Event generation complete." +} + +cmd_verify() { + local backend="${1:-all}" + + log "Verifying backup results..." + + if [ "$backend" = "all" ] || [ "$backend" = "s3" ] || [ "$backend" = "minio" ]; then + log "" + log "=== MinIO (S3) ===" + if docker ps --format '{{.Names}}' | grep -q "$MINIO_CONTAINER"; then + docker run --rm --network host \ + --entrypoint /bin/sh minio/mc -c \ + "mc alias set local http://localhost:9000 minioadmin minioadmin && mc ls --recursive local/convoy-stress-test/" \ + 2>/dev/null || warn "Failed to list MinIO objects" + else + warn "MinIO container not running" + fi + fi + + if [ "$backend" = "all" ] || [ "$backend" = "onprem" ]; then + log "" + log "=== On-Prem ===" + if [ -d /tmp/convoy-stress-test ]; then + local count + count=$(find /tmp/convoy-stress-test -name "*.jsonl.gz" 2>/dev/null | wc -l | tr -d ' ') + log "Export files found: $count" + find /tmp/convoy-stress-test -name "*.jsonl.gz" -exec ls -lh {} \; 2>/dev/null | head -10 + if [ "$count" -gt 0 ]; then + local first_file + first_file=$(find /tmp/convoy-stress-test -name "*.jsonl.gz" | head -1) + local lines + lines=$(gunzip -c "$first_file" | wc -l | tr -d ' ') + log "Records in first file: $lines" + fi + else + warn "On-prem directory not found" + fi + fi + + if [ "$backend" = "all" ] || [ "$backend" = "azure" ]; then + log "" + log "=== Azure Blob ===" + if [ -n "${CONVOY_STORAGE_AZURE_ACCOUNT_NAME:-}" ]; then + warn "Azure verification requires 'az' CLI. Run manually:" + log " az storage blob list --container-name --account-name --output table" + else + warn "Azure env vars not set. Skipping." + fi + fi + + log "" + log "=== Backup Jobs (DB) ===" + warn "Check backup_jobs table manually:" + log " SELECT id, project_id, status, record_counts, created_at FROM convoy.backup_jobs ORDER BY created_at DESC LIMIT 20;" +} + +cmd_help() { + echo "Usage: $0 [args]" + echo "" + echo "Commands:" + echo " infra-up Start MinIO + Azurite + create on-prem dir" + echo " infra-down Stop containers + remove on-prem dir" + echo " generate Generate 1M events via convoy-bench" + echo " verify [backend] Verify backup results (s3|onprem|azure|all)" + echo " help Show this help" + echo "" + echo "Workflow:" + echo " 1. $0 infra-up" + echo " 2. Set backend env vars + CONVOY_BACKUP_INTERVAL=5m" + echo " 3. Start Convoy (server + worker)" + echo " 4. $0 generate" + echo " 5. Wait 15-20 minutes" + echo " 6. $0 verify" + echo " 7. $0 infra-down" +} + +case "${1:-help}" in + infra-up) cmd_infra_up ;; + infra-down) cmd_infra_down ;; + generate) cmd_generate ;; + verify) cmd_verify "${2:-all}" ;; + help|*) cmd_help ;; +esac diff --git a/sql/1774901839.sql b/sql/1774901839.sql new file mode 100644 index 0000000000..4e7f8e1250 --- /dev/null +++ b/sql/1774901839.sql @@ -0,0 +1,21 @@ +-- +migrate Up +CREATE TABLE IF NOT EXISTS convoy.backup_jobs ( + id VARCHAR PRIMARY KEY DEFAULT convoy.generate_ulid(), + hour_start TIMESTAMPTZ NOT NULL, + hour_end TIMESTAMPTZ NOT NULL, + status VARCHAR NOT NULL DEFAULT 'pending', + worker_id VARCHAR, + claimed_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + error TEXT, + record_counts JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- +migrate Up notransaction +CREATE INDEX idx_backup_jobs_pending ON convoy.backup_jobs(status, created_at) + WHERE status IN ('pending', 'claimed'); + +-- +migrate Down +DROP TABLE IF EXISTS convoy.backup_jobs; diff --git a/sql/1774904763.sql b/sql/1774904763.sql new file mode 100644 index 0000000000..2e67f0b075 --- /dev/null +++ b/sql/1774904763.sql @@ -0,0 +1,13 @@ +-- +migrate Up +ALTER TABLE convoy.configurations ADD COLUMN IF NOT EXISTS azure_account_name VARCHAR; +ALTER TABLE convoy.configurations ADD COLUMN IF NOT EXISTS azure_account_key VARCHAR; +ALTER TABLE convoy.configurations ADD COLUMN IF NOT EXISTS azure_container_name VARCHAR; +ALTER TABLE convoy.configurations ADD COLUMN IF NOT EXISTS azure_endpoint VARCHAR; +ALTER TABLE convoy.configurations ADD COLUMN IF NOT EXISTS azure_prefix VARCHAR; + +-- +migrate Down +ALTER TABLE convoy.configurations DROP COLUMN IF EXISTS azure_account_name; +ALTER TABLE convoy.configurations DROP COLUMN IF EXISTS azure_account_key; +ALTER TABLE convoy.configurations DROP COLUMN IF EXISTS azure_container_name; +ALTER TABLE convoy.configurations DROP COLUMN IF EXISTS azure_endpoint; +ALTER TABLE convoy.configurations DROP COLUMN IF EXISTS azure_prefix; diff --git a/sql/1774976436.sql b/sql/1774976436.sql new file mode 100644 index 0000000000..0ae6481fd0 --- /dev/null +++ b/sql/1774976436.sql @@ -0,0 +1,13 @@ +-- +migrate Up +-- +migrate StatementBegin +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'convoy_backup') THEN + EXECUTE 'CREATE PUBLICATION convoy_backup FOR TABLE convoy.events, convoy.event_deliveries, convoy.delivery_attempts'; + END IF; +END; +$$; +-- +migrate StatementEnd + +-- +migrate Down +DROP PUBLICATION IF EXISTS convoy_backup; diff --git a/sqlc.yaml b/sqlc.yaml index 041f25841f..1c104529a4 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -202,3 +202,13 @@ sql: overrides: - column: "convoy.event_deliveries.delivery_mode" go_type: "string" + - queries: ./internal/backup_jobs/queries.sql + engine: postgresql + database: *db_config + gen: + go: + package: "repo" + out: "./internal/backup_jobs/repo" + sql_package: "pgx/v5" + omit_unused_structs: true + emit_interface: true diff --git a/testenv/azurite.go b/testenv/azurite.go new file mode 100644 index 0000000000..aafe876552 --- /dev/null +++ b/testenv/azurite.go @@ -0,0 +1,83 @@ +package testenv + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + // Azurite well-known development credentials + azuriteAccountName = "devstoreaccount1" + azuriteAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + azuriteContainer = "convoy-test-exports" + azuriteBlobPort = "10000/tcp" + azuriteDockerImage = "mcr.microsoft.com/azure-storage/azurite:3.31.0" +) + +// AzuriteClientFunc is a factory function that creates an Azure Blob client for tests. +// It returns the client and the blob service endpoint URL. +type AzuriteClientFunc func(t *testing.T) (*azblob.Client, string, error) + +// NewTestAzurite creates a new Azurite container and returns a factory function +// for creating Azure Blob clients in tests. +func NewTestAzurite(ctx context.Context) (testcontainers.Container, AzuriteClientFunc, error) { + req := testcontainers.ContainerRequest{ + Image: azuriteDockerImage, + ExposedPorts: []string{azuriteBlobPort}, + Cmd: []string{"azurite-blob", "--blobHost", "0.0.0.0", "--blobPort", "10000", "--skipApiVersionCheck"}, + WaitingFor: wait.ForListeningPort(azuriteBlobPort), + } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, nil, fmt.Errorf("failed to start azurite container: %w", err) + } + + return container, newAzuriteClientFunc(container), nil +} + +func newAzuriteClientFunc(container testcontainers.Container) AzuriteClientFunc { + return func(t *testing.T) (*azblob.Client, string, error) { + t.Helper() + + host, err := container.Host(t.Context()) + if err != nil { + return nil, "", fmt.Errorf("failed to get azurite host: %w", err) + } + + mappedPort, err := container.MappedPort(t.Context(), azuriteBlobPort) + if err != nil { + return nil, "", fmt.Errorf("failed to get azurite port: %w", err) + } + + endpoint := fmt.Sprintf("http://%s:%s/%s", host, mappedPort.Port(), azuriteAccountName) + + cred, err := azblob.NewSharedKeyCredential(azuriteAccountName, azuriteAccountKey) + if err != nil { + return nil, "", fmt.Errorf("failed to create azurite credentials: %w", err) + } + + client, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil) + if err != nil { + return nil, "", fmt.Errorf("failed to create azurite client: %w", err) + } + + // Create default container for tests (ignore "already exists" errors) + ctx := t.Context() + _, err = client.CreateContainer(ctx, azuriteContainer, nil) + if err != nil && !strings.Contains(err.Error(), "ContainerAlreadyExists") { + return nil, "", fmt.Errorf("failed to create container: %w", err) + } + + return client, endpoint, nil + } +} diff --git a/testenv/launch.go b/testenv/launch.go index c471ed7d77..52728d09e3 100644 --- a/testenv/launch.go +++ b/testenv/launch.go @@ -24,6 +24,7 @@ type launchConfig struct { enablePostgres bool enableRedis bool enableMinIO bool + enableAzurite bool enableRabbitMQ bool enableLocalStack bool enableKafka bool @@ -64,6 +65,13 @@ func WithMinIO() LaunchOption { } } +// WithAzurite enables Azurite (Azure Blob Storage emulator) container +func WithAzurite() LaunchOption { + return func(c *launchConfig) { + c.enableAzurite = true + } +} + // WithRabbitMQ enables RabbitMQ container func WithRabbitMQ() LaunchOption { return func(c *launchConfig) { @@ -97,6 +105,7 @@ type launchState struct { pgcontainer testcontainers.Container rediscontainer testcontainers.Container miniocontainer testcontainers.Container + azuritecontainer testcontainers.Container rabbitmqcontainer *RabbitMQContainer localstackcontainer testcontainers.Container kafkacontainer testcontainers.Container @@ -111,6 +120,7 @@ type Environment struct { // Optional dependencies (nil if not started) NewMinIOClient *MinIOClientFunc + NewAzuriteClient *AzuriteClientFunc NewRabbitMQConnect *RabbitMQConnectionFunc NewLocalStackConnect *LocalStackConnectionFunc NewKafkaConnect *KafkaConnectionFunc @@ -170,6 +180,18 @@ func Launch(ctx context.Context, opts ...LaunchOption) (*Environment, func() err minioFactory = &factoryCopy } + // Start Azurite if enabled + var azuriteFactory *AzuriteClientFunc + if config.enableAzurite { + azuritecontainer, factory, err := NewTestAzurite(ctx) + if err != nil { + return nil, nil, fmt.Errorf("start azurite container: %w", err) + } + state.azuritecontainer = azuritecontainer + factoryCopy := factory + azuriteFactory = &factoryCopy + } + // Start RabbitMQ if enabled var rmqFactory *RabbitMQConnectionFunc var restartRabbitMQ *RabbitMQRestartFunc @@ -226,6 +248,7 @@ func Launch(ctx context.Context, opts ...LaunchOption) (*Environment, func() err NewRedisClient: rcFactory, NewQueueInspector: inspectorFactory, NewMinIOClient: minioFactory, + NewAzuriteClient: azuriteFactory, NewRabbitMQConnect: rmqFactory, NewLocalStackConnect: localstackFactory, NewKafkaConnect: kafkaFactory, @@ -272,6 +295,18 @@ func Launch(ctx context.Context, opts ...LaunchOption) (*Environment, func() err }) } + // Terminate Azurite if it was started + if state.azuritecontainer != nil { + eg.Go(func() error { + c, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if termErr := state.azuritecontainer.Terminate(c); termErr != nil { + slog.Info(fmt.Sprintf("terminate azurite container: %v", termErr)) + } + return nil + }) + } + // Terminate RabbitMQ if it was started if state.rabbitmqcontainer != nil { eg.Go(func() error { diff --git a/testenv/postgresql.go b/testenv/postgresql.go index 44af5b03b3..fa6b41c607 100644 --- a/testenv/postgresql.go +++ b/testenv/postgresql.go @@ -44,6 +44,8 @@ func NewTestPostgres(ctx context.Context) (*postgres.PostgresContainer, Postgres testcontainers.WithTmpfs(map[string]string{"/var/lib/postgresql/data": "rw"}), testcontainers.WithEnv(map[string]string{"PGDATA": "/var/lib/postgresql/data"}), testcontainers.WithLogger(log.New("postgres", log.LevelDebug)), + // Enable logical WAL for CDC backup tests (superset of replica, no cost to non-CDC tests) + testcontainers.WithCmdArgs("-c", "wal_level=logical"), ) if err != nil { return nil, nil, fmt.Errorf("start postgres container: %w", err) diff --git a/type.go b/type.go index 2cc6e56c30..e8cce68077 100644 --- a/type.go +++ b/type.go @@ -107,7 +107,9 @@ const ( StreamCliEventsProcessor TaskName = "StreamCliEventsProcessor" MonitorTwitterSources TaskName = "MonitorTwitterSources" RetentionPolicies TaskName = "RetentionPolicies" - BackupProjectData TaskName = "BackupProjectData" + EnqueueBackupJobs TaskName = "EnqueueBackupJobs" + ProcessBackupJob TaskName = "ProcessBackupJob" + ManualBackupJob TaskName = "ManualBackupJob" EmailProcessor TaskName = "EmailProcessor" ExpireSecretsProcessor TaskName = "ExpireSecretsProcessor" DeleteArchivedTasksProcessor TaskName = "DeleteArchivedTasksProcessor" diff --git a/util/validator.go b/util/validator.go index 8757153adf..e524bcbaff 100644 --- a/util/validator.go +++ b/util/validator.go @@ -85,8 +85,9 @@ func init() { govalidator.TagMap["supported_storage"] = func(encoder string) bool { encoders := map[string]bool{ - string(datastore.S3): true, - string(datastore.OnPrem): true, + string(datastore.S3): true, + string(datastore.OnPrem): true, + string(datastore.AzureBlob): true, } if _, ok := encoders[encoder]; !ok { diff --git a/worker/task/backup_jobs.go b/worker/task/backup_jobs.go new file mode 100644 index 0000000000..f1c4aae367 --- /dev/null +++ b/worker/task/backup_jobs.go @@ -0,0 +1,183 @@ +package task + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/hibiken/asynq" + + "github.com/frain-dev/convoy/config" + "github.com/frain-dev/convoy/datastore" + blobstore "github.com/frain-dev/convoy/internal/pkg/blob-store" + "github.com/frain-dev/convoy/internal/pkg/exporter" + log "github.com/frain-dev/convoy/pkg/logger" +) + +// EnqueueBackupJobs runs hourly. It inserts a pending backup_job row for each +// (project, hour) pair and reclaims any stale claimed jobs. +func EnqueueBackupJobs( + configRepo datastore.ConfigurationRepository, + backupJobRepo datastore.BackupJobRepository, + logger log.Logger, +) func(context.Context, *asynq.Task) error { + return func(ctx context.Context, t *asynq.Task) error { + dbConfig, err := configRepo.LoadConfiguration(ctx) + if err != nil { + return err + } + + if !dbConfig.RetentionPolicy.IsRetentionPolicyEnabled { + return nil + } + + // Enqueue a global backup job only if no job is currently pending or claimed. + // Completed/failed jobs are kept for audit — they don't block new jobs. + end := time.Now().UTC() + backupInterval := exporter.DefaultBackupInterval + if envCfg, cfgErr := config.Get(); cfgErr == nil { + backupInterval = exporter.ParseBackupInterval(envCfg.RetentionPolicy.BackupInterval) + } + start := end.Add(-backupInterval) + if err = backupJobRepo.EnqueueBackupJobIfIdle(ctx, start, end); err != nil { + logger.Error(fmt.Sprintf("failed to enqueue backup job: %v", err)) + } + + // Reclaim jobs that have been stuck in 'claimed' for > 30 minutes + reclaimed, err := backupJobRepo.ReclaimStaleJobs(ctx, 30) + if err != nil { + logger.Error(fmt.Sprintf("failed to reclaim stale jobs: %v", err)) + } else if reclaimed > 0 { + logger.Info(fmt.Sprintf("reclaimed %d stale backup jobs", reclaimed)) + } + + return nil + } +} + +// ProcessBackupJob claims a pending backup job and streams the export to blob +// storage. Each worker instance calls this independently — SELECT FOR UPDATE +// SKIP LOCKED ensures exactly-once processing. +func ProcessBackupJob( + configRepo datastore.ConfigurationRepository, + eventRepo datastore.EventRepository, + eventDeliveryRepo datastore.EventDeliveryRepository, + attemptsRepo datastore.DeliveryAttemptsRepository, + backupJobRepo datastore.BackupJobRepository, + logger log.Logger, +) func(context.Context, *asynq.Task) error { + return func(ctx context.Context, t *asynq.Task) error { + dbConfig, err := configRepo.LoadConfiguration(ctx) + if err != nil { + return err + } + + if !dbConfig.RetentionPolicy.IsRetentionPolicyEnabled { + return nil + } + + workerID := generateWorkerID() + + // Claim the next pending job (returns nil if none available) + job, err := backupJobRepo.ClaimBackupJob(ctx, workerID) + if err != nil { + return fmt.Errorf("claim backup job: %w", err) + } + if job == nil { + return nil // no work available + } + + logger.Info(fmt.Sprintf("processing backup job %s [%s, %s)", + job.ID, job.HourStart.Format(time.RFC3339), job.HourEnd.Format(time.RFC3339))) + + // Create blob store client + blobStoreClient, err := blobstore.NewBlobStoreClient(dbConfig.StoragePolicy, logger) + if err != nil { + _ = backupJobRepo.FailBackupJob(ctx, job.ID, fmt.Sprintf("create blob store: %v", err)) + return err + } + + // Create exporter with the job's stored time window + e, err := exporter.NewExporterWithWindow(eventRepo, eventDeliveryRepo, dbConfig, attemptsRepo, job.HourStart, job.HourEnd, logger) + if err != nil { + _ = backupJobRepo.FailBackupJob(ctx, job.ID, fmt.Sprintf("create exporter: %v", err)) + return err + } + + result, err := e.StreamExport(ctx, blobStoreClient) + if err != nil { + _ = backupJobRepo.FailBackupJob(ctx, job.ID, fmt.Sprintf("stream export: %v", err)) + return err + } + + // Collect record counts + counts := make(map[string]int64) + for table, r := range result { + counts[string(table)] = r.NumDocs + } + + if err := backupJobRepo.CompleteBackupJob(ctx, job.ID, counts); err != nil { + return fmt.Errorf("complete backup job: %w", err) + } + + logger.Info(fmt.Sprintf("completed backup job %s", job.ID)) + return nil + } +} + +// ManualBackup runs a one-time backup with an explicit time window. +// It always uses the cron-based Exporter, never CDC, regardless of config. +func ManualBackup( + configRepo datastore.ConfigurationRepository, + eventRepo datastore.EventRepository, + eventDeliveryRepo datastore.EventDeliveryRepository, + attemptsRepo datastore.DeliveryAttemptsRepository, + logger log.Logger, +) func(context.Context, *asynq.Task) error { + return func(ctx context.Context, t *asynq.Task) error { + var payload struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` + } + + if err := json.Unmarshal(t.Payload(), &payload); err != nil { + return fmt.Errorf("decode manual backup payload: %w", err) + } + + dbConfig, err := configRepo.LoadConfiguration(ctx) + if err != nil { + return fmt.Errorf("load configuration: %w", err) + } + + store, err := blobstore.NewBlobStoreClient(dbConfig.StoragePolicy, logger) + if err != nil { + return fmt.Errorf("create blob store: %w", err) + } + + exp, err := exporter.NewExporterWithWindow( + eventRepo, eventDeliveryRepo, dbConfig, attemptsRepo, + payload.Start, payload.End, logger, + ) + if err != nil { + return fmt.Errorf("create exporter: %w", err) + } + + result, err := exp.StreamExport(ctx, store) + if err != nil { + return fmt.Errorf("stream export: %w", err) + } + + for table, r := range result { + logger.Info(fmt.Sprintf("manual backup: %s — %d records → %s", table, r.NumDocs, r.ExportFile)) + } + + return nil + } +} + +func generateWorkerID() string { + hostname, _ := os.Hostname() + return fmt.Sprintf("%s-%d", hostname, os.Getpid()) +} diff --git a/worker/task/retention_policies.go b/worker/task/retention_policies.go index 72efb431fa..b016c814c8 100644 --- a/worker/task/retention_policies.go +++ b/worker/task/retention_policies.go @@ -2,7 +2,6 @@ package task import ( "context" - "errors" "fmt" "time" @@ -11,101 +10,19 @@ import ( "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" - "github.com/frain-dev/convoy/datastore" - "github.com/frain-dev/convoy/internal/pkg/exporter" - objectstore "github.com/frain-dev/convoy/internal/pkg/object-store" "github.com/frain-dev/convoy/internal/pkg/retention" log "github.com/frain-dev/convoy/pkg/logger" ) -func BackupProjectData(configRepo datastore.ConfigurationRepository, projectRepo datastore.ProjectRepository, - eventRepo datastore.EventRepository, eventDeliveryRepo datastore.EventDeliveryRepository, attemptsRepo datastore.DeliveryAttemptsRepository, rd redis.UniversalClient, logger log.Logger) func(context.Context, *asynq.Task) error { - pool := goredis.NewPool(rd) - rs := redsync.New(pool) - - return func(ctx context.Context, t *asynq.Task) error { - const mutexName = "convoy:backup-project-data:mutex" - mutex := rs.NewMutex(mutexName, redsync.WithExpiry(time.Second), redsync.WithTries(1)) - - innerCtx, cancel := context.WithTimeout(ctx, time.Second*2) - defer cancel() - - err := mutex.LockContext(innerCtx) - if err != nil { - return fmt.Errorf("failed to obtain lock: %v", err) - } - - defer func() { - _ctx, _cancel := context.WithTimeout(ctx, time.Second*2) - defer _cancel() - - ok, _err := mutex.UnlockContext(_ctx) - if !ok || _err != nil { - logger.Error("failed to release lock", "error", _err) - } - }() - - c := time.Now() - config, err := configRepo.LoadConfiguration(ctx) - if err != nil { - if errors.Is(err, datastore.ErrConfigNotFound) { - return nil - } - return err - } - - filter := &datastore.ProjectFilter{} - projects, err := projectRepo.LoadProjects(context.Background(), filter) - if err != nil { - return err - } - - if len(projects) == 0 { - logger.Warn("no existing projects, retention policy job exiting") - return nil - } - - for _, p := range projects { - e, innerErr := exporter.NewExporter(projectRepo, eventRepo, eventDeliveryRepo, p, config, attemptsRepo, logger) - if innerErr != nil { - return innerErr - } - - result, innerErr := e.Export(ctx) - if innerErr != nil { - logger.Error(fmt.Sprintf("Failed to archive project id's (%s) events : %v", p.UID, innerErr)) - } - - // upload to object storage. - objectStoreClient, innerErr := objectstore.NewObjectStoreClient(config.StoragePolicy, logger) - if innerErr != nil { - return innerErr - } - - for _, r := range result { - if r.NumDocs > 0 { // skip if no record was exported - innerErr = objectStoreClient.Save(r.ExportFile) - if innerErr != nil { - return innerErr - } - } - } - } - - logger.Info(fmt.Sprintf("Backup Project Data job took %f minutes to run", time.Since(c).Minutes())) - return nil - } -} - func RetentionPolicies(rd redis.UniversalClient, ret retention.Retentioner, logger log.Logger) func(context.Context, *asynq.Task) error { pool := goredis.NewPool(rd) rs := redsync.New(pool) return func(ctx context.Context, t *asynq.Task) error { const mutexName = "convoy:retention:mutex" - mutex := rs.NewMutex(mutexName, redsync.WithExpiry(time.Second), redsync.WithTries(1)) + mutex := rs.NewMutex(mutexName, redsync.WithExpiry(30*time.Minute), redsync.WithTries(1)) - lockCtx, cancel := context.WithTimeout(ctx, time.Second*2) + lockCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() err := mutex.LockContext(lockCtx) @@ -113,7 +30,25 @@ func RetentionPolicies(rd redis.UniversalClient, ret retention.Retentioner, logg return fmt.Errorf("failed to obtain lock: %v", err) } + // Renew lock periodically to prevent expiry during long-running retention + renewDone := make(chan struct{}) + go func() { + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + for { + select { + case <-renewDone: + return + case <-ticker.C: + renewCtx, renewCancel := context.WithTimeout(ctx, 10*time.Second) + _, _ = mutex.ExtendContext(renewCtx) + renewCancel() + } + } + }() + defer func() { + close(renewDone) _lockCtx, _cancel := context.WithTimeout(ctx, time.Second*2) defer _cancel() diff --git a/worker/task/retention_policies_test.go b/worker/task/retention_policies_test.go index df40e9041e..b7f5824457 100644 --- a/worker/task/retention_policies_test.go +++ b/worker/task/retention_policies_test.go @@ -197,18 +197,10 @@ func (r *RetentionPoliciesIntegrationTestSuite) Test_Should_Export_Two_Documents // call handler retentionTask := asynq.NewTask(string(convoy.RetentionPolicies), nil, asynq.Queue(string(convoy.ScheduleQueue))) - backUpTask := asynq.NewTask(string(convoy.BackupProjectData), nil, asynq.Queue(string(convoy.ScheduleQueue))) - clock.AdvanceTime(duration + time.Hour) - err = BackupProjectData( - r.ConvoyApp.configRepo, - r.ConvoyApp.projectRepo, - r.ConvoyApp.eventRepo, - r.ConvoyApp.eventDeliveryRepo, - r.ConvoyApp.deliveryRepo, - r.ConvoyApp.redis, r.ConvoyApp.logger)(context.Background(), backUpTask) - require.NoError(r.T(), err) + // Backup is now handled separately by CDC collector or ProcessBackupJob task. + // This test only validates retention (deletion) behavior. err = RetentionPolicies(r.ConvoyApp.redis, ret, r.ConvoyApp.logger)(context.Background(), retentionTask) require.NoError(r.T(), err)