refactor/sync flag

cli/cmd/root.go

@@ -96,6 +96,7 @@ func runService() {
 	)
 	accessKeyService := server.NewAccessKeyService(store, encryptionService, store)
 	secretStorageService := server.NewSecretStorageService(store, store, accessKeyService, encryptionService)
+	secretStorageSyncScheduler := server.NewSecretStorageSyncScheduler(store, secretStorageService)
 	environmentService := server.NewEnvironmentService(store, encryptionService)
 	subscriptionService := proServer.NewSubscriptionService(store, store, store, terraformStore)
 	logWriteService := proServer.NewLogWriteService()
@@ -185,6 +186,9 @@ func runService() {
 	go schedulePool.Run()
 	go taskPool.Run()
 
+	secretStorageSyncScheduler.Start()
+	defer secretStorageSyncScheduler.Stop()
+
 	route := api.Route(
 		store,
 		terraformStore,

db/Migration.go

@@ -122,6 +122,7 @@ func GetMigrations(dialect string) []Migration {
 		{Version: "2.17.15"},
 		{Version: "2.17.16"},
 		{Version: "2.17.17"},
+		{Version: "2.18.1"},
 	}
 
 	return append(initScripts, commonScripts...)

db/SecretStorage.go

@@ -1,5 +1,7 @@
 package db
 
+import "time"
+
 type SecretStorageType string
 
 const (
@@ -10,13 +12,28 @@ const (
 	SecretStorageTypeAzureKv SecretStorageType = "azure_kv"
 )
 
+type SecretStorageSyncPath struct {
+	ID        int    `db:"id" json:"id" backup:"-"`
+	StorageID int    `db:"storage_id" json:"storage_id" backup:"-"`
+	Path      string `db:"path" json:"path"`
+	Prefix    string `db:"prefix" json:"prefix"`
+	Separator string `db:"separator" json:"separator"`
+}
+
 type SecretStorage struct {
-	ID        int               `db:"id" json:"id" backup:"-"`
-	ProjectID int               `db:"project_id" json:"project_id" backup:"-"`
-	Name      string            `db:"name" json:"name"`
-	Type      SecretStorageType `db:"type" json:"type"`
-	Params    MapStringAnyField `db:"params" json:"params"`
-	ReadOnly  bool              `db:"readonly" json:"readonly"`
+	ID          int               `db:"id" json:"id" backup:"-"`
+	ProjectID   int               `db:"project_id" json:"project_id" backup:"-"`
+	Name        string            `db:"name" json:"name"`
+	Type        SecretStorageType `db:"type" json:"type"`
+	Params      MapStringAnyField `db:"params" json:"params"`
+	ReadOnly    bool              `db:"readonly" json:"readonly"`
+	SyncEnabled bool              `db:"sync_enabled" json:"sync_enabled"`
+	// SyncInterval is the auto-sync period in minutes. Zero disables auto-sync.
+	SyncInterval     int        `db:"sync_interval" json:"sync_interval"`
+	LastSyncedAt     *time.Time `db:"last_synced_at" json:"last_synced_at,omitempty"`
+	LastSyncFailedAt *time.Time `db:"last_sync_failed_at" json:"last_sync_failed_at,omitempty"`
+
+	SyncPaths []SecretStorageSyncPath `db:"-" json:"sync_paths"`
 
 	SourceStorageType *AccessKeySourceStorageType `db:"-" json:"source_storage_type,omitempty" backup:"-"`
 	// Secret is a source value: literal secret for local storage,

db/Store.go

@@ -476,6 +476,12 @@ type SecretStorageRepository interface {
 	UpdateSecretStorage(storage SecretStorage) error
 	GetSecretStorageRefs(projectID int, storageID int) (ObjectReferrers, error)
 	DeleteSecretStorage(projectID int, storageID int) error
+
+	GetSecretStorageSyncPaths(storageID int) ([]SecretStorageSyncPath, error)
+	ReplaceSecretStorageSyncPaths(storageID int, paths []SecretStorageSyncPath) error
+
+	GetSyncEnabledSecretStorages() ([]SecretStorage, error)
+	MarkSecretStorageSynced(storageID int, success bool, at time.Time) error
 }
 
 type RoleRepository interface {

db/bolt/secret_storage.go

@@ -1,6 +1,10 @@
 package bolt
 
-import "github.com/semaphoreui/semaphore/db"
+import (
+	"time"
+
+	"github.com/semaphoreui/semaphore/db"
+)
 
 func (d *BoltDb) GetSecretStorages(projectID int) ([]db.SecretStorage, error) {
 	return []db.SecretStorage{}, nil
@@ -28,3 +32,19 @@ func (d *BoltDb) UpdateSecretStorage(storage db.SecretStorage) error {
 func (d *BoltDb) GetSecretStorageRefs(projectID int, storageID int) (db.ObjectReferrers, error) {
 	return d.getObjectRefs(projectID, db.SecretStorageProps, storageID)
 }
+
+func (d *BoltDb) GetSecretStorageSyncPaths(storageID int) ([]db.SecretStorageSyncPath, error) {
+	return []db.SecretStorageSyncPath{}, nil
+}
+
+func (d *BoltDb) ReplaceSecretStorageSyncPaths(storageID int, paths []db.SecretStorageSyncPath) error {
+	return nil
+}
+
+func (d *BoltDb) GetSyncEnabledSecretStorages() ([]db.SecretStorage, error) {
+	return []db.SecretStorage{}, nil
+}
+
+func (d *BoltDb) MarkSecretStorageSynced(storageID int, success bool, at time.Time) error {
+	return nil
+}

db/sql/migrations/v2.18.1.err.sql

@@ -0,0 +1,6 @@
+alter table `project__secret_storage` drop column `sync_enabled`;
+alter table `project__secret_storage` drop column `sync_interval`;
+alter table `project__secret_storage` drop column `last_synced_at`;
+alter table `project__secret_storage` drop column `last_sync_failed_at`;
+
+drop table `project__secret_storage__sync_path`;

db/sql/migrations/v2.18.1.sql

@@ -0,0 +1,17 @@
+alter table `project__secret_storage` add `sync_enabled` boolean not null default false;
+alter table `project__secret_storage` add `sync_interval` int not null default 0;
+alter table `project__secret_storage` add `last_synced_at` datetime null;
+alter table `project__secret_storage` add `last_sync_failed_at` datetime null;
+
+create table `project__secret_storage__sync_path` (
+  `id` integer primary key autoincrement,
+
+  storage_id    int             not null,
+  path          varchar(1000)   not null default '',
+  prefix        varchar(1000)   not null default '',
+  `separator`   varchar(20)     not null default '',
+
+  foreign key (`storage_id`) references `project__secret_storage`(`id`) on delete cascade
+);
+
+

db/sql/secret_storage.go

@@ -1,6 +1,10 @@
 package sql
 
-import "github.com/semaphoreui/semaphore/db"
+import (
+	"time"
+
+	"github.com/semaphoreui/semaphore/db"
+)
 
 func (d *SqlDb) GetSecretStorages(projectID int) (storages []db.SecretStorage, err error) {
 	storages = make([]db.SecretStorage, 0)
@@ -19,18 +23,31 @@ func (d *SqlDb) GetSecretStorages(projectID int) (storages []db.SecretStorage, e
 
 	_, err = d.selectAll(&storages, query, args...)
 
+	if err != nil {
+		return
+	}
+
+	for i := range storages {
+		storages[i].SyncPaths, err = d.GetSecretStorageSyncPaths(storages[i].ID)
+		if err != nil {
+			return
+		}
+	}
+
 	return
 }
 
 func (d *SqlDb) CreateSecretStorage(storage db.SecretStorage) (newStorage db.SecretStorage, err error) {
 	insertID, err := d.insert(
 		"id",
-		"insert into project__secret_storage (name, type, project_id, params, readonly) values (?, ?, ?, ?, ?)",
+		"insert into project__secret_storage (name, type, project_id, params, readonly, sync_enabled, sync_interval) values (?, ?, ?, ?, ?, ?, ?)",
 		storage.Name,
 		storage.Type,
 		storage.ProjectID,
 		storage.Params,
 		storage.ReadOnly,
+		storage.SyncEnabled,
+		storage.SyncInterval,
 	)
 
 	if err != nil {
@@ -39,13 +56,24 @@ func (d *SqlDb) CreateSecretStorage(storage db.SecretStorage) (newStorage db.Sec
 
 	newStorage = storage
 	newStorage.ID = insertID
+
+	err = d.ReplaceSecretStorageSyncPaths(newStorage.ID, storage.SyncPaths)
+	if err != nil {
+		return
+	}
+
+	newStorage.SyncPaths, err = d.GetSecretStorageSyncPaths(newStorage.ID)
 	return
 }
 
 func (d *SqlDb) GetSecretStorage(projectID int, storageID int) (key db.SecretStorage, err error) {
 
 	err = d.getObject(projectID, db.SecretStorageProps, storageID, &key)
+	if err != nil {
+		return
+	}
 
+	key.SyncPaths, err = d.GetSecretStorageSyncPaths(key.ID)
 	return
 }
 
@@ -62,13 +90,84 @@ func (d *SqlDb) UpdateSecretStorage(storage db.SecretStorage) error {
 		"name=?, "+
 		"type=?, "+
 		"params=?, "+
-		"readonly=? "+
+		"readonly=?, "+
+		"sync_enabled=?, "+
+		"sync_interval=? "+
 		"where project_id=? and id=?",
 		storage.Name,
 		storage.Type,
 		storage.Params,
 		storage.ReadOnly,
+		storage.SyncEnabled,
+		storage.SyncInterval,
 		storage.ProjectID,
 		storage.ID)
+
+	if err != nil {
+		return err
+	}
+
+	return d.ReplaceSecretStorageSyncPaths(storage.ID, storage.SyncPaths)
+}
+
+func (d *SqlDb) GetSyncEnabledSecretStorages() (storages []db.SecretStorage, err error) {
+	storages = make([]db.SecretStorage, 0)
+	_, err = d.selectAll(
+		&storages,
+		"select * from project__secret_storage where sync_enabled=? and sync_interval>0",
+		true,
+	)
+	if err != nil {
+		return
+	}
+	for i := range storages {
+		storages[i].SyncPaths, err = d.GetSecretStorageSyncPaths(storages[i].ID)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (d *SqlDb) MarkSecretStorageSynced(storageID int, success bool, at time.Time) error {
+	var query string
+	if success {
+		query = "update project__secret_storage set last_synced_at=? where id=?"
+	} else {
+		query = "update project__secret_storage set last_sync_failed_at=? where id=?"
+	}
+	_, err := d.exec(query, at, storageID)
 	return err
 }
+
+func (d *SqlDb) GetSecretStorageSyncPaths(storageID int) (paths []db.SecretStorageSyncPath, err error) {
+	paths = make([]db.SecretStorageSyncPath, 0)
+	_, err = d.selectAll(
+		&paths,
+		"select id, storage_id, path, prefix, `separator` "+
+			"from project__secret_storage__sync_path where storage_id=? order by id",
+		storageID,
+	)
+	return
+}
+
+func (d *SqlDb) ReplaceSecretStorageSyncPaths(storageID int, paths []db.SecretStorageSyncPath) error {
+	if _, err := d.exec("delete from project__secret_storage__sync_path where storage_id=?", storageID); err != nil {
+		return err
+	}
+
+	for _, p := range paths {
+		if _, err := d.insert(
+			"id",
+			"insert into project__secret_storage__sync_path (storage_id, path, prefix, `separator`) values (?, ?, ?, ?)",
+			storageID,
+			p.Path,
+			p.Prefix,
+			p.Separator,
+		); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

services/server/access_key_svc.go

@@ -45,11 +45,14 @@ func (s *AccessKeyServiceImpl) Delete(projectID int, keyID int) (err error) {
 			return
 		}
 
-		if !storage.ReadOnly {
-			err = s.encryptionService.DeleteSecret(&key)
-			if err != nil {
-				return
-			}
+		if storage.ReadOnly {
+			err = common_errors.NewUserErrorS("cannot delete secret from read-only storage")
+			return
+		}
+
+		err = s.encryptionService.DeleteSecret(&key)
+		if err != nil {
+			return
 		}
 	}
 

services/server/secret_storage_svc.go

@@ -44,6 +44,28 @@ func (s *SecretStorageServiceImpl) SyncSecrets(storage db.SecretStorage) error {
 }
 
 func (s *SecretStorageServiceImpl) Delete(projectID int, storageID int) (err error) {
+	storage, err := s.secretStorageRepo.GetSecretStorage(projectID, storageID)
+	if err != nil {
+		return
+	}
+
+	if storage.SyncEnabled {
+		var syncedKeys []db.AccessKey
+		syncedKeys, err = s.accessKeyRepo.GetAccessKeys(projectID, db.GetAccessKeyOptions{
+			IgnoreOwner:     true,
+			SourceStorageID: &storageID,
+		}, db.RetrieveQueryParams{})
+		if err != nil {
+			return
+		}
+
+		for _, key := range syncedKeys {
+			if err = s.accessKeyRepo.DeleteAccessKey(projectID, key.ID); err != nil {
+				return
+			}
+		}
+	}
+
 	err = s.secretStorageRepo.DeleteSecretStorage(projectID, storageID)
 	if err != nil {
 		return