diff --git a/api/projects/keys.go b/api/projects/keys.go index 74a887308..f1887d330 100644 --- a/api/projects/keys.go +++ b/api/projects/keys.go @@ -95,6 +95,7 @@ func (c *KeyController) AddKey(w http.ResponseWriter, r *http.Request) { // Plain cannot be passed via a request key.Plain = nil key.IgnorePlain = true + key.Synchronized = false //if err := key.Validate(true); err != nil { // helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{ @@ -138,9 +139,19 @@ func (c *KeyController) UpdateKey(w http.ResponseWriter, r *http.Request) { return } + if oldKey.Synchronized { + if key.Name != oldKey.Name || key.Type != oldKey.Type { + helpers.WriteJSON(w, http.StatusBadRequest, map[string]string{ + "error": "Name and type of synchronized key cannot be changed", + }) + return + } + } + // Plain cannot be passed via a request key.Plain = nil key.IgnorePlain = true + key.Synchronized = oldKey.Synchronized repos, err := helpers.Store(r).GetRepositories(*key.ProjectID, db.RetrieveQueryParams{}) if err != nil { diff --git a/api/projects/secret_storages.go b/api/projects/secret_storages.go index 1afcc7727..4b6f34ca0 100644 --- a/api/projects/secret_storages.go +++ b/api/projects/secret_storages.go @@ -204,7 +204,13 @@ func (c *SecretStorageController) SyncSecrets(w http.ResponseWriter, r *http.Req return } - err := c.secretStorageService.SyncSecrets(storage) + sync, err := helpers.Store(r).GetStorageSecretSync(storage.ID) + if err != nil { + helpers.WriteError(w, err) + return + } + + err = c.secretStorageService.SyncSecrets(sync) if err != nil { helpers.WriteError(w, err) return diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 8ece64100..c3361bbce 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -96,6 +96,7 @@ func runService() { ) accessKeyService := server.NewAccessKeyService(store, encryptionService, store) secretStorageService := server.NewSecretStorageService(store, store, accessKeyService, encryptionService) + secretStorageSyncScheduler := server.NewSecretStorageSyncScheduler(store, secretStorageService) environmentService := server.NewEnvironmentService(store, encryptionService) subscriptionService := proServer.NewSubscriptionService(store, store, store, terraformStore) logWriteService := proServer.NewLogWriteService() @@ -136,6 +137,7 @@ func runService() { if dedup := proHA.NewScheduleDeduplicator(); dedup != nil { schedulePool.SetDeduplicator(dedup) + secretStorageSyncScheduler.SetTickDeduplicator(dedup) } // Each process holds its own in-memory cron table. Schedule CRUD handlers only @@ -185,6 +187,9 @@ func runService() { go schedulePool.Run() go taskPool.Run() + secretStorageSyncScheduler.Start() + defer secretStorageSyncScheduler.Stop() + route := api.Route( store, terraformStore, diff --git a/db/AccessKey.go b/db/AccessKey.go index fecb74435..599721188 100644 --- a/db/AccessKey.go +++ b/db/AccessKey.go @@ -67,6 +67,8 @@ type AccessKey struct { // If SourceStorageID is nil, this field is references to an environment variable. SourceStorageKey *string `db:"source_storage_key" json:"source_storage_key,omitempty"` SourceStorageType *AccessKeySourceStorageType `db:"source_storage_type" json:"source_storage_type,omitempty"` + + Synchronized bool `db:"synchronized" json:"synchronized,omitempty"` } func (key *AccessKey) IsNativelyReadOnly() bool { diff --git a/db/Environment.go b/db/Environment.go index c6de1c136..f7322b69b 100644 --- a/db/Environment.go +++ b/db/Environment.go @@ -3,6 +3,7 @@ package db import ( "encoding/json" "errors" + "time" ) type EnvironmentSecretOperation string @@ -53,6 +54,13 @@ type Environment struct { SecretStorageID *int `db:"secret_storage_id" json:"secret_storage_id,omitempty" backup:"-"` SecretStorageKeyPrefix *string `db:"secret_storage_key_prefix" json:"secret_storage_key_prefix,omitempty"` + + // Sync fields are transfer-only; persisted in project__secret_sync. + SyncEnabled bool `db:"-" json:"sync_enabled"` + SyncInterval int `db:"-" json:"sync_interval"` + LastSyncedAt *time.Time `db:"-" json:"last_synced_at,omitempty"` + LastSyncFailedAt *time.Time `db:"-" json:"last_sync_failed_at,omitempty"` + SyncPaths []SecretSyncPath `db:"-" json:"sync_paths"` } func (s *EnvironmentSecret) Validate() error { diff --git a/db/Migration.go b/db/Migration.go index 6ce82ecf7..5a2e21f90 100644 --- a/db/Migration.go +++ b/db/Migration.go @@ -122,6 +122,7 @@ func GetMigrations(dialect string) []Migration { {Version: "2.17.15"}, {Version: "2.17.16"}, {Version: "2.17.17"}, + {Version: "2.18.1"}, } return append(initScripts, commonScripts...) diff --git a/db/SecretStorage.go b/db/SecretStorage.go index efcc50a9e..3eb747cda 100644 --- a/db/SecretStorage.go +++ b/db/SecretStorage.go @@ -1,11 +1,13 @@ package db +import "time" + type SecretStorageType string const ( - SecretStorageTypeLocal SecretStorageType = "local" - SecretStorageTypeVault SecretStorageType = "vault" - SecretStorageTypeDvls SecretStorageType = "dvls" + SecretStorageTypeLocal SecretStorageType = "local" + SecretStorageTypeVault SecretStorageType = "vault" + SecretStorageTypeDvls SecretStorageType = "dvls" SecretStorageTypeAwsSm SecretStorageType = "aws_sm" SecretStorageTypeAzureKv SecretStorageType = "azure_kv" ) @@ -18,6 +20,13 @@ type SecretStorage struct { Params MapStringAnyField `db:"params" json:"params"` ReadOnly bool `db:"readonly" json:"readonly"` + // Sync fields are transfer-only; persisted in project__secret_sync. + SyncEnabled bool `db:"-" json:"sync_enabled" backup:"sync_enabled"` + SyncInterval int `db:"-" json:"sync_interval" backup:"sync_interval"` + LastSyncedAt *time.Time `db:"-" json:"last_synced_at,omitempty" backup:"-"` + LastSyncFailedAt *time.Time `db:"-" json:"last_sync_failed_at,omitempty" backup:"-"` + SyncPaths []SecretSyncPath `db:"-" json:"sync_paths" backup:"-"` + SourceStorageType *AccessKeySourceStorageType `db:"-" json:"source_storage_type,omitempty" backup:"-"` // Secret is a source value: literal secret for local storage, // env var name for "env", or file path for "file". diff --git a/db/SecretSync.go b/db/SecretSync.go new file mode 100644 index 000000000..ec1960aa3 --- /dev/null +++ b/db/SecretSync.go @@ -0,0 +1,34 @@ +package db + +import "time" + +// SecretSync describes a unit of remote-storage synchronization. +// A row with EnvironmentID == nil represents storage-level sync (imports +// access keys). A row with EnvironmentID set represents env-scoped sync +// (imports environment variables for that variable group). +type SecretSync struct { + ID int `db:"id" json:"id" backup:"-"` + ProjectID int `db:"project_id" json:"project_id" backup:"-"` + StorageID int `db:"storage_id" json:"storage_id" backup:"-"` + EnvironmentID *int `db:"environment_id" json:"environment_id,omitempty" backup:"-"` + + SyncEnabled bool `db:"sync_enabled" json:"sync_enabled"` + // SyncInterval is the auto-sync period in minutes. Zero disables auto-sync. + SyncInterval int `db:"sync_interval" json:"sync_interval"` + LastSyncedAt *time.Time `db:"last_synced_at" json:"last_synced_at,omitempty"` + LastSyncFailedAt *time.Time `db:"last_sync_failed_at" json:"last_sync_failed_at,omitempty"` + + Paths []SecretSyncPath `db:"-" json:"paths"` +} + +type SecretSyncPath struct { + ID int `db:"id" json:"id" backup:"-"` + SyncID int `db:"sync_id" json:"sync_id" backup:"-"` + Path string `db:"path" json:"path"` + Prefix string `db:"prefix" json:"prefix"` + Separator string `db:"separator" json:"separator"` +} + +// SecretStorageSyncPath is retained as an alias for backward compatibility +// with callers that predate the SecretSync refactor. +type SecretStorageSyncPath = SecretSyncPath diff --git a/db/Store.go b/db/Store.go index b5a9e884c..e4bd287d5 100644 --- a/db/Store.go +++ b/db/Store.go @@ -478,6 +478,26 @@ type SecretStorageRepository interface { DeleteSecretStorage(projectID int, storageID int) error } +type SecretSyncRepository interface { + // GetSyncEnabledSecretSyncs returns every sync config (storage-level + // and env-scoped) that is enabled with a positive interval. + GetSyncEnabledSecretSyncs() ([]SecretSync, error) + MarkSecretSyncSynced(syncID int, success bool, at time.Time) error + + // GetStorageSecretSync returns the storage-level sync (EnvironmentID=nil) + // for the given storage, or ErrNotFound. + GetStorageSecretSync(storageID int) (SecretSync, error) + // GetEnvironmentSecretSync returns the env-scoped sync for the env, + // or ErrNotFound. + GetEnvironmentSecretSync(environmentID int) (SecretSync, error) + + // SaveSecretSync upserts a sync config (and its paths) identified by + // (StorageID, EnvironmentID) on the passed struct. When SyncEnabled + // is false, SyncInterval is zero, and Paths is empty, the row is + // deleted instead of being written. + SaveSecretSync(sync SecretSync) error +} + type RoleRepository interface { GetGlobalRoleBySlug(slug string) (Role, error) GetProjectOrGlobalRoleBySlug(projectID int, slug string) (Role, error) @@ -511,6 +531,7 @@ type Store interface { RunnerManager EventManager SecretStorageRepository + SecretSyncRepository RoleRepository } diff --git a/db/bolt/secret_storage.go b/db/bolt/secret_storage.go index c53b5f394..7b84894b2 100644 --- a/db/bolt/secret_storage.go +++ b/db/bolt/secret_storage.go @@ -1,6 +1,10 @@ package bolt -import "github.com/semaphoreui/semaphore/db" +import ( + "time" + + "github.com/semaphoreui/semaphore/db" +) func (d *BoltDb) GetSecretStorages(projectID int) ([]db.SecretStorage, error) { return []db.SecretStorage{}, nil @@ -28,3 +32,23 @@ func (d *BoltDb) UpdateSecretStorage(storage db.SecretStorage) error { func (d *BoltDb) GetSecretStorageRefs(projectID int, storageID int) (db.ObjectReferrers, error) { return d.getObjectRefs(projectID, db.SecretStorageProps, storageID) } + +func (d *BoltDb) GetSyncEnabledSecretSyncs() ([]db.SecretSync, error) { + return []db.SecretSync{}, nil +} + +func (d *BoltDb) MarkSecretSyncSynced(syncID int, success bool, at time.Time) error { + return nil +} + +func (d *BoltDb) GetStorageSecretSync(storageID int) (db.SecretSync, error) { + return db.SecretSync{}, db.ErrNotFound +} + +func (d *BoltDb) GetEnvironmentSecretSync(environmentID int) (db.SecretSync, error) { + return db.SecretSync{}, db.ErrNotFound +} + +func (d *BoltDb) SaveSecretSync(sync db.SecretSync) error { + return nil +} diff --git a/db/sql/access_key.go b/db/sql/access_key.go index b1d8da634..a8efdab38 100644 --- a/db/sql/access_key.go +++ b/db/sql/access_key.go @@ -111,8 +111,9 @@ func (d *SqlDb) CreateAccessKey(key db.AccessKey) (newKey db.AccessKey, err erro "storage_id, "+ "source_storage_id, "+ "source_storage_key, "+ - "source_storage_type) "+ - "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "source_storage_type, "+ + "synchronized) "+ + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", key.Name, key.Type, key.ProjectID, @@ -123,6 +124,7 @@ func (d *SqlDb) CreateAccessKey(key db.AccessKey) (newKey db.AccessKey, err erro key.SourceStorageID, key.SourceStorageKey, key.SourceStorageType, + key.Synchronized, ) } else { insertID, err = d.insert( @@ -138,8 +140,9 @@ func (d *SqlDb) CreateAccessKey(key db.AccessKey) (newKey db.AccessKey, err erro "storage_id, "+ "source_storage_id, "+ "source_storage_key, "+ - "source_storage_type) "+ - "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "source_storage_type, "+ + "synchronized) "+ + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", key.Name, key.Type, key.ProjectID, @@ -151,6 +154,7 @@ func (d *SqlDb) CreateAccessKey(key db.AccessKey) (newKey db.AccessKey, err erro key.SourceStorageID, key.SourceStorageKey, key.SourceStorageType, + key.Synchronized, ) } diff --git a/db/sql/environment.go b/db/sql/environment.go index abd9600ae..36ce35af5 100644 --- a/db/sql/environment.go +++ b/db/sql/environment.go @@ -6,6 +6,11 @@ import ( func (d *SqlDb) GetEnvironment(projectID int, environmentID int) (environment db.Environment, err error) { err = d.getObject(projectID, db.EnvironmentProps, environmentID, &environment) + if err != nil { + return + } + + err = d.fillEnvironmentSync(&environment) return } @@ -14,9 +19,19 @@ func (d *SqlDb) GetEnvironmentRefs(projectID int, environmentID int) (db.ObjectR } func (d *SqlDb) GetEnvironments(projectID int, params db.RetrieveQueryParams) ([]db.Environment, error) { - var environment []db.Environment - err := d.getObjects(projectID, db.EnvironmentProps, params, nil, &environment) - return environment, err + var environments []db.Environment + err := d.getObjects(projectID, db.EnvironmentProps, params, nil, &environments) + if err != nil { + return environments, err + } + + for i := range environments { + if err = d.fillEnvironmentSync(&environments[i]); err != nil { + return environments, err + } + } + + return environments, nil } func (d *SqlDb) UpdateEnvironment(env db.Environment) error { @@ -33,7 +48,12 @@ func (d *SqlDb) UpdateEnvironment(env db.Environment) error { env.ENV, env.Password, env.ID) - return err + + if err != nil { + return err + } + + return d.saveEnvironmentSync(env) } func (d *SqlDb) CreateEnvironment(env db.Environment) (newEnv db.Environment, err error) { @@ -62,6 +82,12 @@ func (d *SqlDb) CreateEnvironment(env db.Environment) (newEnv db.Environment, er newEnv = env newEnv.ID = insertID + + if err = d.saveEnvironmentSync(newEnv); err != nil { + return + } + + err = d.fillEnvironmentSync(&newEnv) return } @@ -90,3 +116,45 @@ func (d *SqlDb) GetEnvironmentSecrets(projectID int, environmentID int) (keys [] return } + +func (d *SqlDb) fillEnvironmentSync(env *db.Environment) error { + sync, err := d.GetEnvironmentSecretSync(env.ID) + if err == db.ErrNotFound { + env.SyncEnabled = false + env.SyncInterval = 0 + env.LastSyncedAt = nil + env.LastSyncFailedAt = nil + env.SyncPaths = []db.SecretSyncPath{} + return nil + } + if err != nil { + return err + } + env.SyncEnabled = sync.SyncEnabled + env.SyncInterval = sync.SyncInterval + env.LastSyncedAt = sync.LastSyncedAt + env.LastSyncFailedAt = sync.LastSyncFailedAt + env.SyncPaths = sync.Paths + if env.SyncPaths == nil { + env.SyncPaths = []db.SecretSyncPath{} + } + return nil +} + +// saveEnvironmentSync persists sync settings for an environment. Syncs +// require a linked SecretStorage; without one, any pending sync row is +// removed. +func (d *SqlDb) saveEnvironmentSync(env db.Environment) error { + envID := env.ID + sync := db.SecretSync{ + ProjectID: env.ProjectID, + EnvironmentID: &envID, + } + if env.SecretStorageID != nil { + sync.StorageID = *env.SecretStorageID + sync.SyncEnabled = env.SyncEnabled + sync.SyncInterval = env.SyncInterval + sync.Paths = env.SyncPaths + } + return d.SaveSecretSync(sync) +} diff --git a/db/sql/migrations/v2.18.1.err.sql b/db/sql/migrations/v2.18.1.err.sql new file mode 100644 index 000000000..fb0de6e33 --- /dev/null +++ b/db/sql/migrations/v2.18.1.err.sql @@ -0,0 +1,3 @@ +drop table `project__secret_sync_path`; +drop table `project__secret_sync`; +alter table `access_key` drop column `synchronized`; diff --git a/db/sql/migrations/v2.18.1.sql b/db/sql/migrations/v2.18.1.sql new file mode 100644 index 000000000..b5af72170 --- /dev/null +++ b/db/sql/migrations/v2.18.1.sql @@ -0,0 +1,28 @@ +create table `project__secret_sync` ( + `id` integer primary key autoincrement, + + `project_id` int not null, + `storage_id` int not null, + `environment_id` int, + `sync_enabled` boolean not null default false, + `sync_interval` int not null default 0, + `last_synced_at` datetime null, + `last_sync_failed_at` datetime null, + + foreign key (`project_id`) references `project`(`id`) on delete cascade, + foreign key (`storage_id`) references `project__secret_storage`(`id`) on delete cascade, + foreign key (`environment_id`) references `project__environment`(`id`) on delete cascade +); + +create table `project__secret_sync_path` ( + `id` integer primary key autoincrement, + + `sync_id` int not null, + `path` varchar(1000) not null default '', + `prefix` varchar(1000) not null default '', + `separator` varchar(20) not null default '', + + foreign key (`sync_id`) references `project__secret_sync`(`id`) on delete cascade +); + +alter table `access_key` add `synchronized` boolean not null default false; diff --git a/db/sql/secret_storage.go b/db/sql/secret_storage.go index 18297dbad..e1a4e21e5 100644 --- a/db/sql/secret_storage.go +++ b/db/sql/secret_storage.go @@ -1,6 +1,8 @@ package sql -import "github.com/semaphoreui/semaphore/db" +import ( + "github.com/semaphoreui/semaphore/db" +) func (d *SqlDb) GetSecretStorages(projectID int) (storages []db.SecretStorage, err error) { storages = make([]db.SecretStorage, 0) @@ -19,6 +21,16 @@ func (d *SqlDb) GetSecretStorages(projectID int) (storages []db.SecretStorage, e _, err = d.selectAll(&storages, query, args...) + if err != nil { + return + } + + for i := range storages { + if err = d.fillStorageSync(&storages[i]); err != nil { + return + } + } + return } @@ -39,13 +51,23 @@ func (d *SqlDb) CreateSecretStorage(storage db.SecretStorage) (newStorage db.Sec newStorage = storage newStorage.ID = insertID + + if err = d.SaveSecretSync(secretSyncFromStorage(newStorage)); err != nil { + return + } + + err = d.fillStorageSync(&newStorage) return } -func (d *SqlDb) GetSecretStorage(projectID int, storageID int) (key db.SecretStorage, err error) { +func (d *SqlDb) GetSecretStorage(projectID int, storageID int) (storage db.SecretStorage, err error) { - err = d.getObject(projectID, db.SecretStorageProps, storageID, &key) + err = d.getObject(projectID, db.SecretStorageProps, storageID, &storage) + if err != nil { + return + } + err = d.fillStorageSync(&storage) return } @@ -70,5 +92,48 @@ func (d *SqlDb) UpdateSecretStorage(storage db.SecretStorage) error { storage.ReadOnly, storage.ProjectID, storage.ID) - return err + + if err != nil { + return err + } + + return d.SaveSecretSync(secretSyncFromStorage(storage)) +} + +// secretSyncFromStorage projects a SecretStorage's transfer-only sync fields +// onto a SecretSync payload for persistence. +func secretSyncFromStorage(storage db.SecretStorage) db.SecretSync { + return db.SecretSync{ + ProjectID: storage.ProjectID, + StorageID: storage.ID, + SyncEnabled: storage.SyncEnabled, + SyncInterval: storage.SyncInterval, + LastSyncedAt: storage.LastSyncedAt, + LastSyncFailedAt: storage.LastSyncFailedAt, + Paths: storage.SyncPaths, + } +} + +func (d *SqlDb) fillStorageSync(storage *db.SecretStorage) error { + sync, err := d.GetStorageSecretSync(storage.ID) + if err == db.ErrNotFound { + storage.SyncEnabled = false + storage.SyncInterval = 0 + storage.LastSyncedAt = nil + storage.LastSyncFailedAt = nil + storage.SyncPaths = []db.SecretSyncPath{} + return nil + } + if err != nil { + return err + } + storage.SyncEnabled = sync.SyncEnabled + storage.SyncInterval = sync.SyncInterval + storage.LastSyncedAt = sync.LastSyncedAt + storage.LastSyncFailedAt = sync.LastSyncFailedAt + storage.SyncPaths = sync.Paths + if storage.SyncPaths == nil { + storage.SyncPaths = []db.SecretSyncPath{} + } + return nil } diff --git a/db/sql/secret_sync.go b/db/sql/secret_sync.go new file mode 100644 index 000000000..326e42861 --- /dev/null +++ b/db/sql/secret_sync.go @@ -0,0 +1,144 @@ +package sql + +import ( + "time" + + "github.com/semaphoreui/semaphore/db" +) + +func (d *SqlDb) GetSyncEnabledSecretSyncs() (syncs []db.SecretSync, err error) { + syncs = make([]db.SecretSync, 0) + _, err = d.selectAll( + &syncs, + "select * from project__secret_sync where sync_enabled=? and sync_interval>0", + true, + ) + if err != nil { + return + } + for i := range syncs { + syncs[i].Paths, err = d.getSecretSyncPaths(syncs[i].ID) + if err != nil { + return + } + } + return +} + +func (d *SqlDb) MarkSecretSyncSynced(syncID int, success bool, at time.Time) error { + var query string + if success { + query = "update project__secret_sync set last_synced_at=? where id=?" + } else { + query = "update project__secret_sync set last_sync_failed_at=? where id=?" + } + _, err := d.exec(query, at, syncID) + return err +} + +func (d *SqlDb) GetStorageSecretSync(storageID int) (sync db.SecretSync, err error) { + return d.getSecretSyncByOwner(storageID, nil) +} + +func (d *SqlDb) GetEnvironmentSecretSync(environmentID int) (sync db.SecretSync, err error) { + return d.getSecretSyncByOwner(0, &environmentID) +} + +func (d *SqlDb) getSecretSyncByOwner(storageID int, environmentID *int) (sync db.SecretSync, err error) { + var query string + var args []any + if environmentID == nil { + query = "select * from project__secret_sync where storage_id=? and environment_id is null" + args = []any{storageID} + } else { + query = "select * from project__secret_sync where environment_id=?" + args = []any{*environmentID} + } + + err = d.selectOne(&sync, query, args...) + if err != nil { + return + } + + sync.Paths, err = d.getSecretSyncPaths(sync.ID) + return +} + +func (d *SqlDb) SaveSecretSync(sync db.SecretSync) error { + // If the row can't carry any info (disabled with no paths), remove it + // entirely instead of keeping a blank row around. + if !sync.SyncEnabled && sync.SyncInterval == 0 && len(sync.Paths) == 0 { + return d.deleteSecretSync(sync.StorageID, sync.EnvironmentID) + } + + existing, err := d.getSecretSyncByOwner(sync.StorageID, sync.EnvironmentID) + + var syncID int + switch err { + case nil: + syncID = existing.ID + if _, err = d.exec( + "update project__secret_sync set sync_enabled=?, sync_interval=? where id=?", + sync.SyncEnabled, sync.SyncInterval, syncID, + ); err != nil { + return err + } + case db.ErrNotFound: + syncID, err = d.insert( + "id", + "insert into project__secret_sync "+ + "(project_id, storage_id, environment_id, sync_enabled, sync_interval) "+ + "values (?, ?, ?, ?, ?)", + sync.ProjectID, sync.StorageID, sync.EnvironmentID, + sync.SyncEnabled, sync.SyncInterval, + ) + if err != nil { + return err + } + default: + return err + } + + return d.replaceSecretSyncPaths(syncID, sync.Paths) +} + +func (d *SqlDb) deleteSecretSync(storageID int, environmentID *int) error { + var query string + var args []any + if environmentID == nil { + query = "delete from project__secret_sync where storage_id=? and environment_id is null" + args = []any{storageID} + } else { + query = "delete from project__secret_sync where environment_id=?" + args = []any{*environmentID} + } + _, err := d.exec(query, args...) + return err +} + +func (d *SqlDb) getSecretSyncPaths(syncID int) (paths []db.SecretSyncPath, err error) { + paths = make([]db.SecretSyncPath, 0) + _, err = d.selectAll( + &paths, + "select id, sync_id, path, prefix, `separator` "+ + "from project__secret_sync_path where sync_id=? order by id", + syncID, + ) + return +} + +func (d *SqlDb) replaceSecretSyncPaths(syncID int, paths []db.SecretSyncPath) error { + if _, err := d.exec("delete from project__secret_sync_path where sync_id=?", syncID); err != nil { + return err + } + for _, p := range paths { + if _, err := d.insert( + "id", + "insert into project__secret_sync_path (sync_id, path, prefix, `separator`) values (?, ?, ?, ?)", + syncID, p.Path, p.Prefix, p.Separator, + ); err != nil { + return err + } + } + return nil +} diff --git a/db_lib/AnsiblePlaybook.go b/db_lib/AnsiblePlaybook.go index e15bd3e06..18dc09aa8 100644 --- a/db_lib/AnsiblePlaybook.go +++ b/db_lib/AnsiblePlaybook.go @@ -37,7 +37,7 @@ func (p AnsiblePlaybook) makeCmd(command string, args []string, environmentVars cmd.Env = append(cmd.Env, environmentVars...) - cmd.SysProcAttr = util.Config.GetSysProcAttr() + cmd.SysProcAttr = util.Config.GetAppSysProcAttr() return cmd } diff --git a/db_lib/ShellApp.go b/db_lib/ShellApp.go index 248af817f..52c398aad 100644 --- a/db_lib/ShellApp.go +++ b/db_lib/ShellApp.go @@ -49,7 +49,7 @@ func (t *ShellApp) makeCmd(command string, args []string, environmentVars []stri cmd.Env = append(cmd.Env, fmt.Sprintf("PWD=%s", cmd.Dir)) cmd.Env = append(cmd.Env, environmentVars...) - cmd.SysProcAttr = util.Config.GetSysProcAttr() + cmd.SysProcAttr = util.Config.GetAppSysProcAttr() return cmd } diff --git a/db_lib/TerraformApp.go b/db_lib/TerraformApp.go index a1e59d938..66e474be5 100644 --- a/db_lib/TerraformApp.go +++ b/db_lib/TerraformApp.go @@ -105,7 +105,7 @@ func (t *TerraformApp) makeCmd(command string, args []string, environmentVars [] cmd.Env = append(cmd.Env, environmentVars...) } - cmd.SysProcAttr = util.Config.GetSysProcAttr() + cmd.SysProcAttr = util.Config.GetAppSysProcAttr() return cmd } diff --git a/pro/services/server/secret_storage_svc.go b/pro/services/server/secret_storage_svc.go index 31694edda..44d821c95 100644 --- a/pro/services/server/secret_storage_svc.go +++ b/pro/services/server/secret_storage_svc.go @@ -1,8 +1,6 @@ package server import ( - "fmt" - "github.com/semaphoreui/semaphore/db" ) @@ -12,18 +10,10 @@ func GetSecretStorages(repo db.SecretStorageRepository, projectID int) (storages } func SyncSecrets( - storage db.SecretStorage, + sync db.SecretSync, + storageRepo db.SecretStorageRepository, accessKeyRepo db.AccessKeyManager, decryptor DvlsStorageTokenDeserializer, ) error { - switch storage.Type { - case db.SecretStorageTypeDvls: - return nil - case db.SecretStorageTypeAwsSm: - return nil - case db.SecretStorageTypeAzureKv: - return nil - default: - return fmt.Errorf("sync is not supported for storage type %q", storage.Type) - } + return nil } diff --git a/services/project/backup_test.go b/services/project/backup_test.go index 16ac292c9..90959cb6a 100644 --- a/services/project/backup_test.go +++ b/services/project/backup_test.go @@ -144,6 +144,7 @@ func TestBackup_BackupSecretStorage(t *testing.T) { "name": "Test Key", "owner": "vault", "storage": "Test", + "synchronized": false, "type": "none" } ], @@ -161,6 +162,8 @@ func TestBackup_BackupSecretStorage(t *testing.T) { "name": "Test", "params": {}, "readonly": false, + "sync_enabled": false, + "sync_interval": 0, "type": "vault" } ], diff --git a/services/server/access_key_svc.go b/services/server/access_key_svc.go index d3cf87337..4ee9a635d 100644 --- a/services/server/access_key_svc.go +++ b/services/server/access_key_svc.go @@ -45,11 +45,14 @@ func (s *AccessKeyServiceImpl) Delete(projectID int, keyID int) (err error) { return } - if !storage.ReadOnly { - err = s.encryptionService.DeleteSecret(&key) - if err != nil { - return - } + if storage.ReadOnly { + err = common_errors.NewUserErrorS("cannot delete secret from read-only storage") + return + } + + err = s.encryptionService.DeleteSecret(&key) + if err != nil { + return } } diff --git a/services/server/environment_svc.go b/services/server/environment_svc.go index 4bed349fe..3acd19532 100644 --- a/services/server/environment_svc.go +++ b/services/server/environment_svc.go @@ -2,6 +2,7 @@ package server import ( "fmt" + "github.com/semaphoreui/semaphore/db" ) @@ -25,8 +26,6 @@ type EnvironmentServiceImpl struct { } func (s *EnvironmentServiceImpl) Delete(projectID int, environmentID int) (err error) { - // Implement the logic to delete an environment - // This is a placeholder implementation if projectID <= 0 || environmentID <= 0 { return fmt.Errorf("invalid project or environment ID") } diff --git a/services/server/secret_storage_svc.go b/services/server/secret_storage_svc.go index 6d2c6be20..7547e36de 100644 --- a/services/server/secret_storage_svc.go +++ b/services/server/secret_storage_svc.go @@ -15,7 +15,7 @@ type SecretStorageService interface { Delete(projectID int, storageID int) error GetSecretStorages(projectID int) ([]db.SecretStorage, error) Create(storage db.SecretStorage) (res db.SecretStorage, err error) - SyncSecrets(storage db.SecretStorage) error + SyncSecrets(sync db.SecretSync) error } func NewSecretStorageService( @@ -39,11 +39,33 @@ type SecretStorageServiceImpl struct { encryptionService AccessKeyEncryptionService } -func (s *SecretStorageServiceImpl) SyncSecrets(storage db.SecretStorage) error { - return pro.SyncSecrets(storage, s.accessKeyRepo, s.encryptionService) +func (s *SecretStorageServiceImpl) SyncSecrets(sync db.SecretSync) error { + return pro.SyncSecrets(sync, s.secretStorageRepo, s.accessKeyRepo, s.encryptionService) } func (s *SecretStorageServiceImpl) Delete(projectID int, storageID int) (err error) { + storage, err := s.secretStorageRepo.GetSecretStorage(projectID, storageID) + if err != nil { + return + } + + if storage.SyncEnabled { + var syncedKeys []db.AccessKey + syncedKeys, err = s.accessKeyRepo.GetAccessKeys(projectID, db.GetAccessKeyOptions{ + IgnoreOwner: true, + SourceStorageID: &storageID, + }, db.RetrieveQueryParams{}) + if err != nil { + return + } + + for _, key := range syncedKeys { + if err = s.accessKeyRepo.DeleteAccessKey(projectID, key.ID); err != nil { + return + } + } + } + err = s.secretStorageRepo.DeleteSecretStorage(projectID, storageID) if err != nil { return diff --git a/services/server/secret_storage_sync_scheduler.go b/services/server/secret_storage_sync_scheduler.go new file mode 100644 index 000000000..1ce355d20 --- /dev/null +++ b/services/server/secret_storage_sync_scheduler.go @@ -0,0 +1,123 @@ +package server + +import ( + "sync" + "time" + + "github.com/semaphoreui/semaphore/db" + "github.com/semaphoreui/semaphore/pkg/tz" + log "github.com/sirupsen/logrus" +) + +const secretStorageSyncTickInterval = 60 * time.Second + +// SecretStorageSyncScheduler walks every sync-enabled SecretSync row +// (storage-level and env-scoped) and runs SyncSecrets when the configured +// interval has elapsed. +type SecretStorageSyncScheduler struct { + secretSyncRepo db.SecretSyncRepository + secretStorageService SecretStorageService + tickDeduplicator interface{ TryLockExecution(scheduleID int) bool } + + stop chan struct{} + wg sync.WaitGroup +} + +func NewSecretStorageSyncScheduler( + secretSyncRepo db.SecretSyncRepository, + secretStorageService SecretStorageService, +) *SecretStorageSyncScheduler { + return &SecretStorageSyncScheduler{ + secretSyncRepo: secretSyncRepo, + secretStorageService: secretStorageService, + stop: make(chan struct{}), + } +} + +const secretStorageSyncTickLockID = 0 + +func (s *SecretStorageSyncScheduler) SetTickDeduplicator(d interface{ TryLockExecution(scheduleID int) bool }) { + s.tickDeduplicator = d +} + +func (s *SecretStorageSyncScheduler) Start() { + s.wg.Add(1) + go s.run() +} + +func (s *SecretStorageSyncScheduler) Stop() { + close(s.stop) + s.wg.Wait() +} + +func (s *SecretStorageSyncScheduler) run() { + defer s.wg.Done() + + ticker := time.NewTicker(secretStorageSyncTickInterval) + defer ticker.Stop() + + for { + select { + case <-s.stop: + return + case <-ticker.C: + s.tick() + } + } +} + +func (s *SecretStorageSyncScheduler) tick() { + if s.tickDeduplicator != nil && !s.tickDeduplicator.TryLockExecution(secretStorageSyncTickLockID) { + return + } + + syncs, err := s.secretSyncRepo.GetSyncEnabledSecretSyncs() + if err != nil { + log.WithError(err).Warn("secret sync: failed to list sync-enabled configs") + return + } + + now := tz.Now() + for _, sync := range syncs { + if !secretSyncDue(sync, now) { + continue + } + + syncErr := s.secretStorageService.SyncSecrets(sync) + markTime := tz.Now() + success := syncErr == nil + + if syncErr != nil { + log.WithError(syncErr). + WithField("sync_id", sync.ID). + WithField("storage_id", sync.StorageID). + Warn("secret sync failed") + } + + if err := s.secretSyncRepo.MarkSecretSyncSynced(sync.ID, success, markTime); err != nil { + log.WithError(err). + WithField("sync_id", sync.ID). + Warn("secret sync: failed to record sync timestamp") + } + } +} + +func secretSyncDue(sync db.SecretSync, now time.Time) bool { + if sync.SyncInterval <= 0 { + return false + } + + var lastAttempt time.Time + if sync.LastSyncedAt != nil { + lastAttempt = *sync.LastSyncedAt + } + if sync.LastSyncFailedAt != nil && sync.LastSyncFailedAt.After(lastAttempt) { + lastAttempt = *sync.LastSyncFailedAt + } + + if lastAttempt.IsZero() { + return true + } + + return now.Sub(lastAttempt) >= time.Duration(sync.SyncInterval)*time.Minute +} diff --git a/util/config.go b/util/config.go index 2ebf061b0..a0a0f3241 100644 --- a/util/config.go +++ b/util/config.go @@ -190,6 +190,27 @@ type ConfigProcess struct { Chroot string `json:"chroot,omitempty" env:"SEMAPHORE_PROCESS_CHROOT"` GID *uint32 `json:"gid,omitempty" env:"SEMAPHORE_PROCESS_GID"` NoNewPrivs bool `json:"no_new_privs,omitempty" env:"SEMAPHORE_PROCESS_NO_NEW_PRIVS"` + + // AppNamespaces controls Linux namespace isolation for child apps + // (ansible, terraform, shell templates). Git is never isolated — + // SSH agent forwarding and credential helpers need host access. + AppNamespaces ConfigAppNamespaces `json:"app_namespaces,omitempty"` +} + +// ConfigAppNamespaces mirrors the CLONE_NEW* flags applied to app runs. +// Each flag is a standard Linux namespace and is a no-op on non-Linux. +type ConfigAppNamespaces struct { + // User isolates UIDs/GIDs (CLONE_NEWUSER). Enables unprivileged use + // of the other namespaces. + User bool `json:"user,omitempty" env:"SEMAPHORE_PROCESS_APP_NS_USER"` + // Mount hides host mount points such as secret tmpfs (CLONE_NEWNS). + Mount bool `json:"mount,omitempty" env:"SEMAPHORE_PROCESS_APP_NS_MOUNT"` + // PID hides host processes from child apps (CLONE_NEWPID). + PID bool `json:"pid,omitempty" env:"SEMAPHORE_PROCESS_APP_NS_PID"` + // IPC isolates SysV IPC and POSIX message queues (CLONE_NEWIPC). + IPC bool `json:"ipc,omitempty" env:"SEMAPHORE_PROCESS_APP_NS_IPC"` + // UTS isolates hostname and domain (CLONE_NEWUTS). + UTS bool `json:"uts,omitempty" env:"SEMAPHORE_PROCESS_APP_NS_UTS"` } type ScheduleConfig struct { diff --git a/util/config_sysproc.go b/util/config_sysproc.go index 29a6f29b4..c393aa2b6 100644 --- a/util/config_sysproc.go +++ b/util/config_sysproc.go @@ -70,6 +70,7 @@ func (conf *ConfigType) GetSysProcAttr() (res *syscall.SysProcAttr) { return } + // ChownDir changes ownership of the directory to the process config user/group. // This is needed because directories are created by the main Semaphore process, // but child processes (git, ansible, etc.) run as the configured process user. diff --git a/util/config_sysproc_linux.go b/util/config_sysproc_linux.go new file mode 100644 index 000000000..94975a087 --- /dev/null +++ b/util/config_sysproc_linux.go @@ -0,0 +1,43 @@ +//go:build linux + +package util + +import "syscall" + +// GetAppSysProcAttr returns SysProcAttr for child apps (ansible, terraform, +// shell templates). It builds on top of GetSysProcAttr and adds Linux +// namespace isolation flags when configured. It is NOT used for git: git +// needs host access for SSH agents and credential helpers. +func (conf *ConfigType) GetAppSysProcAttr() *syscall.SysProcAttr { + res := conf.GetSysProcAttr() + + flags := conf.Process.AppNamespaces.cloneFlags() + if flags == 0 { + return res + } + + if res == nil { + res = &syscall.SysProcAttr{} + } + res.Cloneflags |= flags + return res +} + +func (ns ConfigAppNamespaces) cloneFlags() (flags uintptr) { + if ns.User { + flags |= syscall.CLONE_NEWUSER + } + if ns.Mount { + flags |= syscall.CLONE_NEWNS + } + if ns.PID { + flags |= syscall.CLONE_NEWPID + } + if ns.IPC { + flags |= syscall.CLONE_NEWIPC + } + if ns.UTS { + flags |= syscall.CLONE_NEWUTS + } + return +} diff --git a/util/config_sysproc_other.go b/util/config_sysproc_other.go new file mode 100644 index 000000000..1d3986f13 --- /dev/null +++ b/util/config_sysproc_other.go @@ -0,0 +1,12 @@ +//go:build !linux && !windows + +package util + +import "syscall" + +// GetAppSysProcAttr falls back to the base SysProcAttr on OSes that don't +// support Linux CLONE_NEW* namespaces. The flags are configurable but +// silently inert here. +func (conf *ConfigType) GetAppSysProcAttr() *syscall.SysProcAttr { + return conf.GetSysProcAttr() +} diff --git a/util/config_sysproc_windows.go b/util/config_sysproc_windows.go index ef25e82b1..e4ae06905 100644 --- a/util/config_sysproc_windows.go +++ b/util/config_sysproc_windows.go @@ -11,6 +11,13 @@ func (conf *ConfigType) GetSysProcAttr() (res *syscall.SysProcAttr) { return } +// GetAppSysProcAttr is a no-op on Windows. Linux namespace isolation is +// not available, so child apps run with the same attributes as the main +// process. +func (conf *ConfigType) GetAppSysProcAttr() *syscall.SysProcAttr { + return conf.GetSysProcAttr() +} + func ChownDir(path string) error { return nil } diff --git a/web/src/components/EnvironmentForm.vue b/web/src/components/EnvironmentForm.vue index 8cfa6c4e2..a78dbf5cf 100644 --- a/web/src/components/EnvironmentForm.vue +++ b/web/src/components/EnvironmentForm.vue @@ -193,18 +193,83 @@ -
+
{{ getIcon(secretStorage.type) }} {{ secretStorage.name }} 
Source path pattern: {{ item.secret_storage_key_prefix }}*
+ +
+ + + + mdi-cog-sync + Sync paths + + {{ (item.sync_paths || []).length }} + + +
-
+ + + Sync paths + + + + + + + + + {{ $t('close') }} + + + + + +
{{ $t('extraVariables') }} -