diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..7ac30e42 --- /dev/null +++ b/.envrc @@ -0,0 +1,3 @@ +# .envrc +use flake + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..563fdf47 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +result +.direnv +vendor/ diff --git a/cmd/rollback.go b/cmd/rollback.go new file mode 100644 index 00000000..d97031e5 --- /dev/null +++ b/cmd/rollback.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "fmt" + "os" + "path" + "runtime" + + "github.com/nlewo/comin/internal/config" + "github.com/nlewo/comin/internal/deployer" + executorPkg "github.com/nlewo/comin/internal/executor" + "github.com/nlewo/comin/internal/protobuf" + storePkg "github.com/nlewo/comin/internal/store" + "github.com/sirupsen/logrus" + + "github.com/spf13/cobra" +) + +var deploymentUUID, generationUUID, commitID string + +var rollbackCmd = &cobra.Command{ + Use: "rollback", + Short: "Rollback to a previous deployment", + Long: `Rollback to a previous deployment. +If no flags are provided, it rolls back to the last successful deployment. +You can specify a deployment to roll back to using one of the following flags: +--deployment-uuid, --generation-uuid, or --commit-id.`, + Run: func(cmd *cobra.Command, args []string) { + var stateDir string + if configFilepath != "" { + cfg, err := config.Read(configFilepath) + if err != nil { + logrus.Error(err) + os.Exit(1) + } + stateDir = cfg.StateDir + } else { + stateDir = "/var/lib/comin" + } + + storeFilename := path.Join(stateDir, "store.json") + gcRootsDir := path.Join(stateDir, "gcroots") + store, err := storePkg.New(storeFilename, gcRootsDir, 10, 10) + if err != nil { + logrus.Error(err) + os.Exit(1) + } + if err := store.Load(); err != nil { + logrus.Errorf("Ignoring the state file %s because of the loading error: %s", storeFilename, err) + } + + var deploymentToRollback *protobuf.Deployment + // Ensure that only one flag is provided + if deploymentUUID != "" && generationUUID != "" || deploymentUUID != "" && commitID != "" || generationUUID != "" && commitID != "" { + fmt.Println("Error: only one of --deployment-uuid, --generation-uuid, or --commit-id can be provided") + os.Exit(1) + } + + if deploymentUUID != "" { + deploymentToRollback, err = store.GetDeploymentByUUID(deploymentUUID) + if err != nil { + logrus.Error(err) + os.Exit(1) + } + } else if generationUUID != "" { + deploymentToRollback, err = store.GetDeploymentByGenerationUUID(generationUUID) + if err != nil { + logrus.Error(err) + os.Exit(1) + } + } else if commitID != "" { + deploymentToRollback, err = store.GetDeploymentByCommitId(commitID) + if err != nil { + logrus.Error(err) + os.Exit(1) + } + } else { + deploymentToRollback, err = store.GetLastSuccessfulDeployment() + if err != nil { + logrus.Error(err) + os.Exit(1) + } + } + + executor, err := executorPkg.NewNixOS() + if runtime.GOOS == "darwin" { + executor, err = executorPkg.NewNixDarwin() + } + if err != nil { + logrus.Errorf("Failed to create the executor: %s", err) + return + } + + deployer := deployer.New(store, executor.Deploy, nil, "", "") + if err := deployer.Rollback(deploymentToRollback); err != nil { + logrus.Error(err) + os.Exit(1) + } + + fmt.Println("Rollback successful") + }, +} + +func init() { + rootCmd.AddCommand(rollbackCmd) + rollbackCmd.PersistentFlags().StringVarP(&configFilepath, "config", "", "", "the configuration file path") + rollbackCmd.Flags().StringVar(&deploymentUUID, "deployment-uuid", "", "The UUID of the deployment to roll back to") + rollbackCmd.Flags().StringVar(&generationUUID, "generation-uuid", "", "The UUID of the generation to roll back to") + rollbackCmd.Flags().StringVar(&commitID, "commit-id", "", "The commit ID of the deployment to roll back to") +} diff --git a/cmd/run.go b/cmd/run.go index 06acda1c..4c37834d 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -84,7 +84,7 @@ var runCmd = &cobra.Command{ // redeployment as well as non fast forward checkouts var mainCommitId string var lastDeployment *protobuf.Deployment - if ok, ld := store.LastDeployment(); ok { + if ld, err := store.GetLastSuccessfulDeployment(); err == nil { mainCommitId = ld.Generation.MainCommitId lastDeployment = ld metrics.SetDeploymentInfo(ld.Generation.SelectedCommitId, ld.Status) @@ -101,7 +101,7 @@ var runCmd = &cobra.Command{ sched.FetchRemotes(fetcher, cfg.Remotes) builder := builder.New(store, executor, gitConfig.Path, gitConfig.Dir, cfg.Hostname, 30*time.Minute, 30*time.Minute) - deployer := deployer.New(store, executor.Deploy, lastDeployment, cfg.PostDeploymentCommand) + deployer := deployer.New(store, executor.Deploy, lastDeployment, cfg.PostDeploymentCommand, cfg.LivelinessCheckCommand) manager := manager.New(store, metrics, sched, fetcher, builder, deployer, machineId, executor) diff --git a/cmd/status.go b/cmd/status.go index e8b9e4a3..5dfbecc5 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -86,14 +86,24 @@ func onelineStatus(status *pb.State) { } else if status.Deployer.Deployment != nil { switch status.Deployer.Deployment.Status { case store.StatusToString(store.Running): - fmt.Printf(" deploy %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName, - humanize.Time(status.Deployer.Deployment.EndedAt.AsTime())) + if status.Deployer.Deployment.Operation == "rollback" { + fmt.Printf(" rollback %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName, + humanize.Time(status.Deployer.Deployment.EndedAt.AsTime())) + } else { + fmt.Printf(" deploy %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName, + humanize.Time(status.Deployer.Deployment.EndedAt.AsTime())) + } case store.StatusToString(store.Failed): fmt.Printf(" %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName, humanize.Time(status.Deployer.Deployment.EndedAt.AsTime())) case store.StatusToString(store.Done): - fmt.Printf(" %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName, - humanize.Time(status.Deployer.Deployment.EndedAt.AsTime())) + if status.Deployer.Deployment.Operation == "rollback" { + fmt.Printf(" rollback %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName, + humanize.Time(status.Deployer.Deployment.EndedAt.AsTime())) + } else { + fmt.Printf(" deploy %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName, + humanize.Time(status.Deployer.Deployment.EndedAt.AsTime())) + } } } if status.NeedToReboot.GetValue() { diff --git a/docs/advanced-config.md b/docs/advanced-config.md index 6e530f66..ca9ccd4a 100644 --- a/docs/advanced-config.md +++ b/docs/advanced-config.md @@ -17,8 +17,11 @@ services.comin = { url = "/your/local/infra/repository"; # We don't want to deploy the local main branch on each commit branches.main.name = "main-tilia"; - # We want to fetch this remote every 2 seconds - poller.period = 2; + # We want to fetch this remote every 2 seconds with a random delay up to 10 seconds + poller = { + period = 2; + random_delay = 10; + }; } ]; machineId = "22823ba6c96947e78b006c51a56fd89c"; diff --git a/docs/generated-module-options.md b/docs/generated-module-options.md index 925eda6e..13ce82c7 100644 --- a/docs/generated-module-options.md +++ b/docs/generated-module-options.md @@ -180,6 +180,29 @@ string +## services\.comin\.livelinessCheckCommand + + + +A command to be executed after a successful deployment to check if the new generation is healthy\. If the command returns a non-zero exit code, the deployment is considered failed and comin will roll back to the previous generation\. + + + +*Type:* +null or string + + + +*Default:* +` null ` + + + +*Example:* +` "curl --fail http://localhost:8080/health" ` + + + ## services\.comin\.machineId @@ -425,6 +448,24 @@ signed integer +## services\.comin\.remotes\.\*\.poller\.random_delay + + + +The maximum random delay in seconds before fetching the remote\. + + + +*Type:* +signed integer + + + +*Default:* +` 0 ` + + + ## services\.comin\.remotes\.\*\.timeout diff --git a/docs/howtos.md b/docs/howtos.md index 234083db..45e5f99b 100644 --- a/docs/howtos.md +++ b/docs/howtos.md @@ -21,6 +21,20 @@ has to be on top of the `main` branch. To `nixos-rebuild switch` to this configuration, the `main` branch has to be rebased on the `testing` branch. +## How to ensure a deployment is healthy + +After a deployment, you might want to run some checks to ensure that the new generation is healthy. For example, you might want to check that a web server is responding or that a specific service is running. + +Comin provides the `services.comin.livelinessCheckCommand` option to run a command after a successful deployment. If the command returns a non-zero exit code, the deployment is considered failed and comin will roll back to the previous generation. + +Here is an example of how to use it: + +```nix +services.comin.livelinessCheckCommand = "curl --fail http://localhost:8080/health"; +``` + +In this example, comin will run `curl --fail http://localhost:8080/health` after each deployment. If the command fails, comin will roll back to the previous generation. + ## Iterate faster with local repository By default, comin polls remotes every 60 seconds. You could however @@ -77,3 +91,13 @@ When comin is running on a Darwin system, it automatically builds and deploys a configuration found in the flake output `darwinConfigurations.hostname`. So, you only need to set this flake output and run comin on the target machine. + +## How to rollback to a previous generation + +Comin allows you to manually rollback to the last successful deployment. To do so, you can use the `rollback` command: + +```bash +comin rollback --config /etc/comin/configuration.yaml +``` + +This command will find the last successful deployment and will redeploy it. diff --git a/go.mod b/go.mod index 0445d6c6..66c78fba 100644 --- a/go.mod +++ b/go.mod @@ -5,21 +5,26 @@ go 1.23.0 toolchain go1.24.1 require ( + github.com/Masterminds/sprig/v3 v3.3.0 github.com/ProtonMail/go-crypto v1.1.5 - github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df github.com/dustin/go-humanize v1.0.1 github.com/go-co-op/gocron/v2 v2.11.0 github.com/go-git/go-git/v5 v5.11.0 github.com/google/uuid v1.6.0 + github.com/mattn/go-shellwords v1.0.12 github.com/prometheus/client_golang v1.19.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.9 gopkg.in/yaml.v2 v2.4.0 ) require ( - dario.cat/mergo v1.0.0 // indirect + dario.cat/mergo v1.0.1 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -30,11 +35,13 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect + github.com/huandu/xstrings v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -42,7 +49,9 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sergi/go-diff v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/skeema/knownhosts v1.2.1 // indirect + github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect golang.org/x/crypto v0.39.0 // indirect @@ -54,8 +63,6 @@ require ( golang.org/x/text v0.26.0 // indirect golang.org/x/tools v0.33.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect - google.golang.org/grpc v1.75.0 // indirect - google.golang.org/protobuf v1.36.9 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9bfefb05..e0309fa6 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -9,12 +15,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= -github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= @@ -31,6 +33,8 @@ github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcej github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-co-op/gocron/v2 v2.11.0 h1:IOowNA6SzwdRFnD4/Ol3Kj6G2xKfsoiiGq2Jhhm9bvE= @@ -43,15 +47,20 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -67,6 +76,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= @@ -90,11 +105,15 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skeema/knownhosts v1.2.1 h1:SHWdIUa82uGZz+F+47k8SY4QhhI291cXCpopT1lK2AQ= github.com/skeema/knownhosts v1.2.1/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -107,26 +126,30 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -135,32 +158,23 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/deployer/deployer.go b/internal/deployer/deployer.go index 3a173736..6e9d325e 100644 --- a/internal/deployer/deployer.go +++ b/internal/deployer/deployer.go @@ -19,17 +19,18 @@ import ( type DeployFunc func(context.Context, string, string) (bool, string, error) type Deployer struct { - GenerationCh chan *protobuf.Generation - deployerFunc DeployFunc - DeploymentDoneCh chan *protobuf.Deployment - mu sync.Mutex - deployment atomic.Pointer[protobuf.Deployment] - previousDeployment atomic.Pointer[protobuf.Deployment] - isDeploying atomic.Bool + GenerationCh chan *protobuf.Generation + deployerFunc DeployFunc + DeploymentDoneCh chan *protobuf.Deployment + mu sync.Mutex + deployment atomic.Pointer[protobuf.Deployment] + previousDeployment atomic.Pointer[protobuf.Deployment] + isDeploying atomic.Bool // The next generation to deploy. nil when there is no new generation to deploy - GenerationToDeploy *protobuf.Generation - generationAvailableCh chan struct{} - postDeploymentCommand string + GenerationToDeploy *protobuf.Generation + generationAvailableCh chan struct{} + postDeploymentCommand string + livelinessCheckCommand string isSuspended atomic.Bool resumeCh chan struct{} @@ -73,11 +74,19 @@ func showDeployment(padding string, d *protobuf.Deployment) { fmt.Printf("%sDeployment is running since %s\n", padding, humanize.Time(d.StartedAt.AsTime())) fmt.Printf("%sOperation %s\n", padding, d.Operation) case store.StatusToString(store.Done): - fmt.Printf("%sDeployment succeeded %s\n", padding, humanize.Time(d.EndedAt.AsTime())) + if d.Operation == "rollback" { + fmt.Printf("%sRollback succeeded %s\n", padding, humanize.Time(d.EndedAt.AsTime())) + } else { + fmt.Printf("%sDeployment succeeded %s\n", padding, humanize.Time(d.EndedAt.AsTime())) + } fmt.Printf("%sOperation %s\n", padding, d.Operation) fmt.Printf("%sProfilePath %s\n", padding, d.ProfilePath) case store.StatusToString(store.Failed): - fmt.Printf("%sDeployment failed %s\n", padding, humanize.Time(d.EndedAt.AsTime())) + if d.Operation == "rollback" { + fmt.Printf("%sRollback failed %s\n", padding, humanize.Time(d.EndedAt.AsTime())) + } else { + fmt.Printf("%sDeployment failed %s\n", padding, humanize.Time(d.EndedAt.AsTime())) + } fmt.Printf("%sOperation %s\n", padding, d.Operation) fmt.Printf("%sProfilePath %s\n", padding, d.ProfilePath) } @@ -100,16 +109,17 @@ func Show(s *protobuf.Deployer, padding string) { showDeployment(padding, s.Deployment) } -func New(store *store.Store, deployFunc DeployFunc, previousDeployment *protobuf.Deployment, postDeploymentCommand string) *Deployer { +func New(store *store.Store, deployFunc DeployFunc, previousDeployment *protobuf.Deployment, postDeploymentCommand string, livelinessCheckCommand string) *Deployer { if previousDeployment != nil { logrus.Infof("deployer: initializing with previous deployment %s", previousDeployment.Uuid) } deployer := &Deployer{ - store: store, - DeploymentDoneCh: make(chan *protobuf.Deployment, 1), - deployerFunc: deployFunc, - generationAvailableCh: make(chan struct{}, 1), - postDeploymentCommand: postDeploymentCommand, + store: store, + DeploymentDoneCh: make(chan *protobuf.Deployment, 1), + deployerFunc: deployFunc, + generationAvailableCh: make(chan struct{}, 1), + postDeploymentCommand: postDeploymentCommand, + livelinessCheckCommand: livelinessCheckCommand, resumeCh: make(chan struct{}, 1), } @@ -148,7 +158,11 @@ func (d *Deployer) Submit(generation *protobuf.Generation) { default: } } else { - logrus.Infof("deployer: skipping deployment of the generation %s because it is the same than the last deployment", generation.Uuid) + if previous.Status == store.StatusToString(store.Failed) { + logrus.Infof("deployer: skipping deployment of generation %s because it is the same as the last failed deployment", generation.Uuid) + } else { + logrus.Infof("deployer: skipping deployment of generation %s because it is the same as the last successful deployment", generation.Uuid) + } } d.mu.Unlock() } @@ -193,10 +207,42 @@ func (d *Deployer) Run() { deployment := d.Deployment() deployment.EndedAt = timestamppb.New(time.Now().UTC()) + + // The deployment is finished, we can run the liveliness check if any + if err == nil && d.livelinessCheckCommand != "" { + livelinessCheckCmd := d.livelinessCheckCommand + logrus.Infof("deployer: deploying generation %s, running liveliness check command [%s]", g.Uuid, livelinessCheckCmd) + _, errLiveliness := runLivelinessCheckCommand(livelinessCheckCmd, deployment) + if errLiveliness != nil { + logrus.Errorf("deployer: deploying generation %s, liveliness check command [%s] failed: %v", g.Uuid, livelinessCheckCmd, errLiveliness) + err = errLiveliness + + // Auto-Rollback + lastSuccessful, err := d.store.GetLastSuccessfulDeployment() + if err != nil { + logrus.Errorf("deployer: could not get the last successful deployment: %s", err) + } else { + if err := d.Rollback(lastSuccessful); err != nil { + logrus.Errorf("deployer: rollback to generation %s failed: %s", lastSuccessful.Generation.Uuid, err) + } + } // The main deployment has failed, we update the store and + // we don't run the post-deployment command + d.store.DeploymentFinished(dpl.Uuid, err, cominNeedRestart, profilePath) + d.isDeploying.Store(false) + dpl.Status = store.StatusToString(store.Failed) + d.deployment.Store(dpl) + d.DeploymentDoneCh <- dpl + continue + } else { + logrus.Infof("deployer: deploying generation %s, liveliness check command [%s] succeed", g.Uuid, livelinessCheckCmd) + } + } + if err := d.store.DeploymentFinished(dpl.Uuid, err, cominNeedRestart, profilePath); err != nil { logrus.Errorf("deployer: could not update the deployment %s in the store", dpl.Uuid) continue } + cmd := d.postDeploymentCommand if cmd != "" { _, err = runPostDeploymentCommand(cmd, deployment) @@ -211,3 +257,38 @@ func (d *Deployer) Run() { } }() } + +func (d *Deployer) Rollback(deployment *protobuf.Deployment) error { + logrus.Infof("deployer: rolling back to generation %s", deployment.Generation.Uuid) + operation := "switch" + ctx := context.TODO() + + dpl := d.store.NewDeployment(deployment.Generation, "rollback") + d.previousDeployment.Swap(d.Deployment()) + d.deployment.Store(dpl) + d.isDeploying.Store(true) + defer d.isDeploying.Store(false) + + if err := d.store.DeploymentStarted(dpl.Uuid); err != nil { + return err + } + + cominNeedRestart, profilePath, err := d.deployerFunc( + ctx, + deployment.Generation.OutPath, + operation, + ) + + dpl.EndedAt = timestamppb.New(time.Now().UTC()) + if err != nil { + d.store.DeploymentFinished(dpl.Uuid, err, cominNeedRestart, profilePath) + return err + } + + if err := d.store.DeploymentFinished(dpl.Uuid, nil, cominNeedRestart, profilePath); err != nil { + return err + } + + d.deployment.Store(dpl) + return nil +} diff --git a/internal/deployer/deployer_test.go b/internal/deployer/deployer_test.go index 25b62096..00361e0c 100644 --- a/internal/deployer/deployer_test.go +++ b/internal/deployer/deployer_test.go @@ -21,7 +21,7 @@ func TestDeployerBasic(t *testing.T) { tmp := t.TempDir() s, err := store.New(tmp+"/state.json", tmp+"/gcroots", 1, 1) assert.Nil(t, err) - d := deployer.New(s, deployFunc, nil, "") + d := deployer.New(s, deployFunc, nil, "", "") d.Run() assert.False(t, d.IsDeploying()) @@ -54,7 +54,7 @@ func TestDeployerSubmit(t *testing.T) { tmp := t.TempDir() s, err := store.New(tmp+"/state.json", tmp+"/gcroots", 1, 1) assert.Nil(t, err) - d := deployer.New(s, deployFunc, nil, "") + d := deployer.New(s, deployFunc, nil, "", "") d.Run() assert.False(t, d.IsDeploying()) @@ -74,7 +74,7 @@ func TestDeployerSubmit(t *testing.T) { assert.EventuallyWithT(t, func(c *assert.CollectT) { assert.False(c, d.IsDeploying()) assert.Equal(c, "profile-path", d.Deployment().ProfilePath) - assert.Nil(t, d.GenerationToDeploy) + assert.Nil(c, d.GenerationToDeploy) }, 5*time.Second, 100*time.Millisecond) assert.EventuallyWithT(t, func(c *assert.CollectT) { @@ -94,7 +94,7 @@ func TestDeployerSuspend(t *testing.T) { tmp := t.TempDir() s, err := store.New(tmp+"/state.json", tmp+"/gcroots", 1, 1) assert.Nil(t, err) - d := deployer.New(s, deployFunc, nil, "") + d := deployer.New(s, deployFunc, nil, "", "") d.Run() assert.False(t, d.IsSuspended()) d.Suspend() diff --git a/internal/deployer/liveliness_check.go b/internal/deployer/liveliness_check.go new file mode 100644 index 00000000..87ca6d82 --- /dev/null +++ b/internal/deployer/liveliness_check.go @@ -0,0 +1,41 @@ +package deployer + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "text/template" + + sprig "github.com/Masterminds/sprig/v3" + shellwords "github.com/mattn/go-shellwords" + "github.com/nlewo/comin/internal/protobuf" +) + +func runLivelinessCheckCommand(command string, deployment *protobuf.Deployment) (string, error) { + t, err := template.New("liveliness-check-command"). + Funcs(sprig.TxtFuncMap()). + Parse(command) + if err != nil { + return "", err + } + + var tpl bytes.Buffer + if err := t.Execute(&tpl, deployment); err != nil { + return "", err + } + + parsedArgs, err := shellwords.Parse(tpl.String()) + if err != nil { + return "", err + } + if len(parsedArgs) == 0 { + return "", fmt.Errorf("empty command") + } + + cmd := exec.Command(parsedArgs[0], parsedArgs[1:]...) + cmd.Env = os.Environ() + output, err := cmd.CombinedOutput() + + return string(output), err +} diff --git a/internal/deployer/liveliness_check_test.go b/internal/deployer/liveliness_check_test.go new file mode 100644 index 00000000..0ec8e849 --- /dev/null +++ b/internal/deployer/liveliness_check_test.go @@ -0,0 +1,34 @@ +package deployer + +import ( + "context" + "testing" + + "github.com/nlewo/comin/internal/protobuf" + "github.com/nlewo/comin/internal/store" + "github.com/stretchr/testify/assert" +) + +func TestLivelinessCheck(t *testing.T) { + tmp := t.TempDir() + s, err := store.New(tmp+"/state.json", tmp+"/gcroots", 1, 1) + assert.Nil(t, err) + + deployFunc := func(ctx context.Context, outPath, operation string) (bool, string, error) { + return false, "", nil + } + + // Test with a failing liveliness check + d := New(s, deployFunc, nil, "", "sh -c 'exit 1'") + d.Submit(&protobuf.Generation{Uuid: "a"}) + go d.Run() + deployment := <-d.DeploymentDoneCh + assert.Equal(t, store.StatusToString(store.Failed), deployment.Status) + + // Test with a succeeding liveliness check + d = New(s, deployFunc, nil, "", "sh -c 'exit 0'") + d.Submit(&protobuf.Generation{Uuid: "b"}) + go d.Run() + deployment = <-d.DeploymentDoneCh + assert.Equal(t, store.StatusToString(store.Done), deployment.Status) +} diff --git a/internal/deployer/post_deployment_command.go b/internal/deployer/post_deployment_command.go index 1017f5c5..e499accb 100644 --- a/internal/deployer/post_deployment_command.go +++ b/internal/deployer/post_deployment_command.go @@ -6,6 +6,7 @@ import ( "os/exec" "strings" + shellwords "github.com/mattn/go-shellwords" pb "github.com/nlewo/comin/internal/protobuf" "github.com/sirupsen/logrus" @@ -44,9 +45,15 @@ func envCominFlakeUrl(d *pb.Deployment) string { } func runPostDeploymentCommand(command string, d *pb.Deployment) (string, error) { + args, err := shellwords.Parse(command) + if err != nil { + return "", fmt.Errorf("failed to parse command %q: %w", command, err) + } + if len(args) == 0 { + return "", fmt.Errorf("empty command") + } - cmd := exec.Command(command) - + cmd := exec.Command(args[0], args[1:]...) cmd.Env = append(os.Environ(), "COMIN_GIT_SHA="+envGitSha(d), "COMIN_GIT_REF="+envGitRef(d), @@ -65,6 +72,5 @@ func runPostDeploymentCommand(command string, d *pb.Deployment) (string, error) } logrus.Debugf("cmd:[%s] output:[%s]", command, outputString) - return outputString, nil } diff --git a/internal/executor/executor_test.go b/internal/executor/executor_test.go index 9552a704..ee39a098 100644 --- a/internal/executor/executor_test.go +++ b/internal/executor/executor_test.go @@ -9,7 +9,7 @@ import ( func TestNixExecutorWithDarwinConfiguration(t *testing.T) { // Test creating a NixExecutor with Darwin configuration - executor, err := NewNixExecutor("darwinConfigurations") + executor, err := NewNixExecutor("darwinConfigurations", "/nix/store") assert.NoError(t, err) assert.NotNil(t, executor) assert.Equal(t, "darwinConfigurations", executor.configurationAttr) @@ -17,7 +17,7 @@ func TestNixExecutorWithDarwinConfiguration(t *testing.T) { func TestNixExecutorWithNixOSConfiguration(t *testing.T) { // Test creating a NixExecutor with NixOS configuration - executor, err := NewNixExecutor("nixosConfigurations") + executor, err := NewNixExecutor("nixosConfigurations", "/nix/store") assert.NoError(t, err) assert.NotNil(t, executor) assert.Equal(t, "nixosConfigurations", executor.configurationAttr) @@ -46,7 +46,7 @@ func TestNixExecutorEval(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - executor, err := NewNixExecutor(tt.configurationAttr) + executor, err := NewNixExecutor(tt.configurationAttr, "/nix/store") assert.NoError(t, err) ctx := context.Background() @@ -83,7 +83,7 @@ func TestNixExecutorShowDerivation(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - executor, err := NewNixExecutor(tt.configurationAttr) + executor, err := NewNixExecutor(tt.configurationAttr, "/nix/store") assert.NoError(t, err) ctx := context.Background() @@ -115,7 +115,7 @@ func TestNixExecutorList(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - executor, err := NewNixExecutor(tt.configurationAttr) + executor, err := NewNixExecutor(tt.configurationAttr, "/nix/store") assert.NoError(t, err) // Test that List doesn't panic and handles configuration attribute correctly @@ -148,7 +148,7 @@ func TestNixExecutorDeploy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - executor, err := NewNixExecutor(tt.configurationAttr) + executor, err := NewNixExecutor(tt.configurationAttr, "/nix/store") assert.NoError(t, err) ctx := context.Background() diff --git a/internal/executor/nix.go b/internal/executor/nix.go index f5675de7..90ba40cb 100644 --- a/internal/executor/nix.go +++ b/internal/executor/nix.go @@ -12,10 +12,24 @@ import ( type NixLocal struct { configurationAttr string + storeDir string } -func NewNixExecutor(configurationAttr string) (*NixLocal, error) { - return &NixLocal{configurationAttr: configurationAttr}, nil +func NewNixExecutor(configurationAttr string, storeDir ...string) (*NixLocal, error) { + var s string + var err error + if len(storeDir) > 0 && storeDir[0] != "" { + s = storeDir[0] + } else { + s, err = GetNixStoreDir() + if err != nil { + return nil, err + } + } + return &NixLocal{ + configurationAttr: configurationAttr, + storeDir: s, + }, nil } func (n *NixLocal) ReadMachineId() (string, error) { @@ -45,11 +59,11 @@ func (n *NixLocal) IsStorePathExist(storePath string) bool { } func (n *NixLocal) ShowDerivation(ctx context.Context, flakeUrl, hostname string) (drvPath string, outPath string, err error) { - return showDerivation(ctx, flakeUrl, hostname, n.configurationAttr) + return showDerivation(ctx, flakeUrl, hostname, n.configurationAttr, n.storeDir) } func (n *NixLocal) Eval(ctx context.Context, flakeUrl, hostname string) (drvPath string, outPath string, machineId string, err error) { - drvPath, outPath, err = showDerivation(ctx, flakeUrl, hostname, n.configurationAttr) + drvPath, outPath, err = showDerivation(ctx, flakeUrl, hostname, n.configurationAttr, n.storeDir) if err != nil { return } diff --git a/internal/executor/utils.go b/internal/executor/utils.go index dc2f19e5..a28a0c8c 100644 --- a/internal/executor/utils.go +++ b/internal/executor/utils.go @@ -16,6 +16,18 @@ import ( "github.com/sirupsen/logrus" ) +func GetNixStoreDir() (string, error) { + if dir := os.Getenv("NIX_STORE_DIR"); dir != "" { + return dir, nil + } + cmd := exec.Command("nix", "eval", "--raw", "--expr", "builtins.storeDir") + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to determine Nix store dir: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + // GetExpectedMachineId evals nixosConfigurations or darwinConfigurations based on configurationAttr // returns (machine-id, nil) is comin.machineId is set, ("", nil) otherwise. func getExpectedMachineId(ctx context.Context, path, hostname, configurationAttr string) (machineId string, err error) { @@ -60,8 +72,8 @@ func runNixCommand(ctx context.Context, args []string, stdout, stderr io.Writer) return nil } -func showDerivation(ctx context.Context, flakeUrl, hostname, configurationAttr string) (drvPath string, outPath string, err error) { - installable := fmt.Sprintf("%s#%s.\"%s\".config.system.build.toplevel", flakeUrl, configurationAttr, hostname) +func showDerivation(ctx context.Context, flakeUrl, hostname, configurationAttr, storeDir string) (drvPath string, outPath string, err error) { + installable := fmt.Sprintf("%s#%s.%s.config.system.build.toplevel", flakeUrl, configurationAttr, hostname) args := []string{ "derivation", "show", @@ -84,8 +96,18 @@ func showDerivation(ctx context.Context, flakeUrl, hostname, configurationAttr s for key := range output { keys = append(keys, key) } - drvPath = keys[0] - outPath = output[drvPath].Outputs.Out.Path + rawDrvPath := keys[0] + rawOutPath := output[rawDrvPath].Outputs.Out.Path + + drvPath = rawDrvPath + if !strings.HasPrefix(drvPath, "/") { + drvPath = filepath.Join(storeDir, drvPath) + } + outPath = rawOutPath + if !strings.HasPrefix(outPath, "/") { + outPath = filepath.Join(storeDir, outPath) + } + logrus.Infof("nix: the derivation path is %s", drvPath) logrus.Infof("nix: the output path is %s", outPath) return diff --git a/internal/executor/utils_test.go b/internal/executor/utils_test.go index 07df02c6..353352fd 100644 --- a/internal/executor/utils_test.go +++ b/internal/executor/utils_test.go @@ -73,7 +73,7 @@ func TestShowDerivation(t *testing.T) { ctx := context.Background() // Test that the function doesn't panic and handles the parameters correctly - _, _, err := showDerivation(ctx, tt.flakeUrl, tt.hostname, tt.configurationAttr) + _, _, err := showDerivation(ctx, tt.flakeUrl, tt.hostname, tt.configurationAttr, "") // This will error in test environment because nix command will fail, // but we're testing the code path and parameter handling diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 64e609cb..93eb5b30 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -8,6 +8,8 @@ package manager import ( "fmt" "os" + "os/exec" + "runtime" "github.com/nlewo/comin/internal/builder" "github.com/nlewo/comin/internal/deployer" @@ -181,9 +183,20 @@ func (m *Manager) Run() { m.needToReboot = m.executor.NeedToReboot() m.prometheus.SetHostInfo(m.needToReboot) if dpl.RestartComin.GetValue() { - // TODO: stop contexts - logrus.Infof("manager: comin needs to be restarted") - logrus.Infof("manager: exiting comin to let the service manager restart it") + logrus.Infof("manager: comin needs to be restarted, triggering restarter service") + + // see nix module.nix systemd.services.comin-restarter + if runtime.GOOS == "linux" { + cmd := exec.Command("systemctl", "start", "comin-restarter.service") + if err := cmd.Run(); err != nil { + logrus.Errorf("manager: failed to start comin-restarter.service: %s", err) + } + } + + // On non-systemd platforms like Darwin, we exit to + // let the service manager restart comin. On + // systemd, the comin-restarter.service will + // restart the comin.service. os.Exit(0) } } diff --git a/internal/manager/manager_test.go b/internal/manager/manager_test.go index a05a6620..18181d36 100644 --- a/internal/manager/manager_test.go +++ b/internal/manager/manager_test.go @@ -26,7 +26,7 @@ var mkDeployerMock = func(t *testing.T) *deployer.Deployer { tmp := t.TempDir() s, err := store.New(tmp+"/state.json", tmp+"/gcroots", 1, 1) assert.Nil(t, err) - return deployer.New(s, deployFunc, nil, "") + return deployer.New(s, deployFunc, nil, "", "") } type ExecutorMock struct { @@ -69,8 +69,9 @@ func (n ExecutorMock) Build(ctx context.Context, drvPath string) (err error) { } func NewExecutorMock(machineId string) ExecutorMock { return ExecutorMock{ - evalOk: make(chan bool, 1), - buildOk: make(chan bool, 1), + evalOk: make(chan bool, 1), + buildOk: make(chan bool, 1), + machineId: machineId, } } @@ -86,8 +87,8 @@ func TestBuild(t *testing.T) { var deployFunc = func(context.Context, string, string) (bool, string, error) { return false, "profile-path", nil } - d := deployer.New(s, deployFunc, nil, "") - e, _ := executor.NewNixOS() + d := deployer.New(s, deployFunc, nil, "", "") + e, _ := executor.NewNixExecutor("nixosConfigurations", "/nix/store") m := New(s, prometheus.New(), scheduler.New(), f, b, d, "", e) go m.Run() assert.False(t, m.Fetcher.GetState().IsFetching.GetValue()) @@ -193,8 +194,8 @@ func TestDeploy(t *testing.T) { var deployFunc = func(context.Context, string, string) (bool, string, error) { return false, "profile-path", nil } - d := deployer.New(s, deployFunc, nil, "") - e, _ := executor.NewNixOS() + d := deployer.New(s, deployFunc, nil, "", "") + e, _ := executor.NewNixExecutor("nixosConfigurations", "/nix/store") m := New(s, prometheus.New(), scheduler.New(), f, b, d, "", e) go m.Run() assert.False(t, m.Fetcher.GetState().IsFetching.GetValue()) @@ -217,7 +218,7 @@ func TestIncorrectMachineId(t *testing.T) { eMock := NewExecutorMock("invalid-machine-id") b := builder.New(s, eMock, "repoPath", "", "my-machine", 2*time.Second, 2*time.Second) d := mkDeployerMock(t) - e, _ := executor.NewNixOS() + e, _ := executor.NewNixExecutor("nixosConfigurations", "/nix/store") m := New(s, prometheus.New(), scheduler.New(), f, b, d, "the-test-machine-id", e) go m.Run() @@ -242,7 +243,7 @@ func TestCorrectMachineId(t *testing.T) { eMock.evalOk <- true b := builder.New(s, eMock, "repoPath", "", "my-machine", 2*time.Second, 2*time.Second) d := mkDeployerMock(t) - e, _ := executor.NewNixOS() + e, _ := executor.NewNixExecutor("nixosConfigurations", "/nix/store") m := New(s, prometheus.New(), scheduler.New(), f, b, d, "the-test-machine-id", e) go m.Run() @@ -267,7 +268,7 @@ func TestManagerWithDarwinConfiguration(t *testing.T) { d := mkDeployerMock(t) // Test with Darwin configuration - e, _ := executor.NewNixDarwin() + e, _ := executor.NewNixExecutor("darwinConfigurations", "/nix/store") m := New(s, prometheus.New(), scheduler.New(), f, b, d, "darwin-machine-id", e) // Verify the manager was created with the correct configuration attribute diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index e541313a..779b18ed 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -2,6 +2,7 @@ package scheduler import ( "fmt" + "math/rand" "time" "github.com/go-co-op/gocron/v2" @@ -34,6 +35,11 @@ func (s Scheduler) FetchRemotes(fetcher *fetcher.Fetcher, remotes []types.Remote ), gocron.NewTask( func() { + if remote.Poller.RandomDelay > 0 { + delay := time.Duration(rand.Intn(remote.Poller.RandomDelay)) * time.Second + logrus.Infof("scheduler: sleeping for %s before fetching remote %s", delay, remote.Name) + time.Sleep(delay) + } logrus.Debugf("scheduler: running task for remote %s", remote.Name) fetcher.TriggerFetch([]string{remote.Name}) }, diff --git a/internal/store/store.go b/internal/store/store.go index 407a0299..16c4dabd 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -10,6 +10,7 @@ import ( "google.golang.org/protobuf/encoding/protojson" ) + type State struct { Deployments []*protobuf.Deployment `json:"deployments"` Generations []*protobuf.Generation `json:"generations"` @@ -108,6 +109,43 @@ func (s *Store) LastDeployment() (ok bool, d *protobuf.Deployment) { return } +func (s *Store) GetLastSuccessfulDeployment() (d *protobuf.Deployment, err error) { + for _, d := range s.data.Deployments { + if d.Status == StatusToString(Done) { + return d, nil + } + } + return nil, errors.New("no successful deployment found") +} + +func (s *Store) GetDeploymentByCommitId(commitId string) (d *protobuf.Deployment, err error) { + for _, d := range s.data.Deployments { + if d.Generation.SelectedCommitId == commitId { + return d, nil + } + } + return nil, errors.New("no deployment found for this commit ID") +} + +func (s *Store) GetDeploymentByUUID(uuid string) (d *protobuf.Deployment, err error) { + for _, d := range s.data.Deployments { + if d.Uuid == uuid { + return d, nil + } + } + return nil, errors.New("no deployment found for this UUID") +} + +func (s *Store) GetDeploymentByGenerationUUID(uuid string) (d *protobuf.Deployment, err error) { + for _, d := range s.data.Deployments { + if d.Generation.Uuid == uuid { + return d, nil + } + } + return nil, errors.New("no deployment found for this generation UUID") +} + + func (s *Store) Load() (err error) { var data protobuf.Store content, err := os.ReadFile(s.filename) diff --git a/internal/types/types.go b/internal/types/types.go index 804ea58a..66286de4 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -11,7 +11,8 @@ type Remote struct { } type Poller struct { - Period int `yaml:"period"` + Period int `yaml:"period"` + RandomDelay int `yaml:"random_delay"` } type GitConfig struct { @@ -58,5 +59,6 @@ type Configuration struct { Grpc Grpc `yaml:"grpc"` Exporter HttpServer `yaml:"exporter"` GpgPublicKeyPaths []string `yaml:"gpg_public_key_paths"` - PostDeploymentCommand string `yaml:"post_deployment_command"` + PostDeploymentCommand string `yaml:"post_deployment_command"` + LivelinessCheckCommand string `yaml:"liveliness_check_command"` } diff --git a/nix/comin-config.nix b/nix/comin-config.nix index d8845134..58d76842 100644 --- a/nix/comin-config.nix +++ b/nix/comin-config.nix @@ -16,6 +16,9 @@ in rec { } // ( lib.optionalAttrs (cfg.services.comin.postDeploymentCommand != null) { post_deployment_command = cfg.services.comin.postDeploymentCommand; } + ) // ( + lib.optionalAttrs (cfg.services.comin.livelinessCheckCommand != null) + { liveliness_check_command = cfg.services.comin.livelinessCheckCommand; } ); cominConfigYaml = yaml.generate "comin.yaml" cominConfig; } diff --git a/nix/module-options.nix b/nix/module-options.nix index c1417675..d46653a8 100644 --- a/nix/module-options.nix +++ b/nix/module-options.nix @@ -142,6 +142,13 @@ The poller period in seconds. ''; }; + random_delay = mkOption { + type = types.int; + default = 0; + description = '' + The maximum random delay in seconds before fetching the remote. + ''; + }; }; }; }; @@ -187,6 +194,12 @@ pkgs.writers.writeBash "post" "echo $COMIN_GIT_SHA"; ''; }; + livelinessCheckCommand = mkOption { + description = "A command to be executed after a successful deployment to check if the new generation is healthy. If the command returns a non-zero exit code, the deployment is considered failed and comin will roll back to the previous generation."; + type = nullOr str; + default = null; + example = "curl --fail http://localhost:8080/health"; + }; }; }; } diff --git a/nix/module.nix b/nix/module.nix index 8e051c95..d3e5b6dd 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -22,9 +22,7 @@ in { services.comin.package = lib.mkDefault pkgs.comin or self.packages.${system}.comin or null; systemd.services.comin = { wantedBy = [ "multi-user.target" ]; - path = [ config.nix.package ]; - # The comin service is restarted by comin itself when it - # detects the unit file changed. + path = [ config.nix.package pkgs.systemd ]; restartIfChanged = false; serviceConfig = { ExecStart = @@ -35,5 +33,26 @@ in { Restart = "always"; }; }; + + + # We use an external "restarter" service instead of relying on systemd's automatic + # restart of the comin service because: + # + # - `services.comin.restartIfChanged = false` prevents NixOS from restarting comin + # when its unit file or dependencies change during a switch. + # - The comin service self-updates and calls `switch-to-configuration`, then exits + # to let systemd restart it, but that restart happens using the *previous* + # in-memory unit definition — not the newly generated one. + # - A full `systemctl restart comin.service` is required after `daemon-reload` to + # rebind systemd to the new unit file from the current generation. + + systemd.services.comin-restarter = { + serviceConfig = { + Type = "oneshot"; + ExecStart = '' + ${pkgs.systemd}/bin/systemctl restart comin.service + ''; + }; + }; }; } diff --git a/nix/package.nix b/nix/package.nix index 6ee23cde..414981de 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -24,7 +24,7 @@ in buildGoModule rec { pname = "comin"; - version = "0.8.0"; + version = "0.9.0"; nativeCheckInputs = [ git ]; src = lib.fileset.toSource { root = ../.; @@ -36,7 +36,7 @@ buildGoModule rec { ../main.go ]; }; - vendorHash = "sha256-I4ePkYhuvotmvv8ghLcAm5QWlWHVa/BU2Picbyggy90="; + vendorHash = "sha256-XX2hf6ahJgtpWn6XGW6g9/TJ8y3ALPV6lalbTWxK7/w="; ldflags = [ "-X github.com/nlewo/comin/cmd.version=${version}" ];