WIP: feat(deploy): Add liveliness check, Manual/Automated Rollbacks

.envrc

@@ -0,0 +1,3 @@
+# .envrc
+use flake
+

.gitignore

@@ -0,0 +1,3 @@
+result
+.direnv
+vendor/

cmd/rollback.go

@@ -0,0 +1,110 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"runtime"
+
+	"github.com/nlewo/comin/internal/config"
+	"github.com/nlewo/comin/internal/deployer"
+	executorPkg "github.com/nlewo/comin/internal/executor"
+	"github.com/nlewo/comin/internal/protobuf"
+	storePkg "github.com/nlewo/comin/internal/store"
+	"github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+var deploymentUUID, generationUUID, commitID string
+
+var rollbackCmd = &cobra.Command{
+	Use:   "rollback",
+	Short: "Rollback to a previous deployment",
+	Long: `Rollback to a previous deployment.
+If no flags are provided, it rolls back to the last successful deployment.
+You can specify a deployment to roll back to using one of the following flags:
+--deployment-uuid, --generation-uuid, or --commit-id.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		var stateDir string
+		if configFilepath != "" {
+			cfg, err := config.Read(configFilepath)
+			if err != nil {
+				logrus.Error(err)
+				os.Exit(1)
+			}
+			stateDir = cfg.StateDir
+		} else {
+			stateDir = "/var/lib/comin"
+		}
+
+		storeFilename := path.Join(stateDir, "store.json")
+		gcRootsDir := path.Join(stateDir, "gcroots")
+		store, err := storePkg.New(storeFilename, gcRootsDir, 10, 10)
+		if err != nil {
+			logrus.Error(err)
+			os.Exit(1)
+		}
+		if err := store.Load(); err != nil {
+			logrus.Errorf("Ignoring the state file %s because of the loading error: %s", storeFilename, err)
+		}
+
+		var deploymentToRollback *protobuf.Deployment
+		// Ensure that only one flag is provided
+		if deploymentUUID != "" && generationUUID != "" || deploymentUUID != "" && commitID != "" || generationUUID != "" && commitID != "" {
+			fmt.Println("Error: only one of --deployment-uuid, --generation-uuid, or --commit-id can be provided")
+			os.Exit(1)
+		}
+
+		if deploymentUUID != "" {
+			deploymentToRollback, err = store.GetDeploymentByUUID(deploymentUUID)
+			if err != nil {
+				logrus.Error(err)
+				os.Exit(1)
+			}
+		} else if generationUUID != "" {
+			deploymentToRollback, err = store.GetDeploymentByGenerationUUID(generationUUID)
+			if err != nil {
+				logrus.Error(err)
+				os.Exit(1)
+			}
+		} else if commitID != "" {
+			deploymentToRollback, err = store.GetDeploymentByCommitId(commitID)
+			if err != nil {
+				logrus.Error(err)
+				os.Exit(1)
+			}
+		} else {
+			deploymentToRollback, err = store.GetLastSuccessfulDeployment()
+			if err != nil {
+				logrus.Error(err)
+				os.Exit(1)
+			}
+		}
+
+		executor, err := executorPkg.NewNixOS()
+		if runtime.GOOS == "darwin" {
+			executor, err = executorPkg.NewNixDarwin()
+		}
+		if err != nil {
+			logrus.Errorf("Failed to create the executor: %s", err)
+			return
+		}
+
+		deployer := deployer.New(store, executor.Deploy, nil, "", "")
+		if err := deployer.Rollback(deploymentToRollback); err != nil {
+			logrus.Error(err)
+			os.Exit(1)
+		}
+
+		fmt.Println("Rollback successful")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(rollbackCmd)
+	rollbackCmd.PersistentFlags().StringVarP(&configFilepath, "config", "", "", "the configuration file path")
+	rollbackCmd.Flags().StringVar(&deploymentUUID, "deployment-uuid", "", "The UUID of the deployment to roll back to")
+	rollbackCmd.Flags().StringVar(&generationUUID, "generation-uuid", "", "The UUID of the generation to roll back to")
+	rollbackCmd.Flags().StringVar(&commitID, "commit-id", "", "The commit ID of the deployment to roll back to")
+}

cmd/run.go

@@ -84,7 +84,7 @@ var runCmd = &cobra.Command{
 		// redeployment as well as non fast forward checkouts
 		var mainCommitId string
 		var lastDeployment *protobuf.Deployment
-		if ok, ld := store.LastDeployment(); ok {
+		if ld, err := store.GetLastSuccessfulDeployment(); err == nil {
 			mainCommitId = ld.Generation.MainCommitId
 			lastDeployment = ld
 			metrics.SetDeploymentInfo(ld.Generation.SelectedCommitId, ld.Status)
@@ -101,7 +101,7 @@ var runCmd = &cobra.Command{
 		sched.FetchRemotes(fetcher, cfg.Remotes)
 
 		builder := builder.New(store, executor, gitConfig.Path, gitConfig.Dir, cfg.Hostname, 30*time.Minute, 30*time.Minute)
-		deployer := deployer.New(store, executor.Deploy, lastDeployment, cfg.PostDeploymentCommand)
+		deployer := deployer.New(store, executor.Deploy, lastDeployment, cfg.PostDeploymentCommand, cfg.LivelinessCheckCommand)
 
 		manager := manager.New(store, metrics, sched, fetcher, builder, deployer, machineId, executor)
 

cmd/status.go

@@ -86,14 +86,24 @@ func onelineStatus(status *pb.State) {
 	} else if status.Deployer.Deployment != nil {
 		switch status.Deployer.Deployment.Status {
 		case store.StatusToString(store.Running):
-			fmt.Printf(" deploy %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName,
-				humanize.Time(status.Deployer.Deployment.EndedAt.AsTime()))
+			if status.Deployer.Deployment.Operation == "rollback" {
+				fmt.Printf(" rollback %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName,
+					humanize.Time(status.Deployer.Deployment.EndedAt.AsTime()))
+			} else {
+				fmt.Printf(" deploy %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName,
+					humanize.Time(status.Deployer.Deployment.EndedAt.AsTime()))
+			}
 		case store.StatusToString(store.Failed):
 			fmt.Printf(" %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName,
 				humanize.Time(status.Deployer.Deployment.EndedAt.AsTime()))
 		case store.StatusToString(store.Done):
-			fmt.Printf(" %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName,
-				humanize.Time(status.Deployer.Deployment.EndedAt.AsTime()))
+			if status.Deployer.Deployment.Operation == "rollback" {
+				fmt.Printf(" rollback %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName,
+					humanize.Time(status.Deployer.Deployment.EndedAt.AsTime()))
+			} else {
+				fmt.Printf(" deploy %s/%s (%s)", status.Deployer.Deployment.Generation.SelectedRemoteName, status.Deployer.Deployment.Generation.SelectedBranchName,
+					humanize.Time(status.Deployer.Deployment.EndedAt.AsTime()))
+			}
 		}
 	}
 	if status.NeedToReboot.GetValue() {

docs/advanced-config.md

@@ -17,8 +17,11 @@ services.comin = {
       url = "/your/local/infra/repository";
       # We don't want to deploy the local main branch on each commit
       branches.main.name = "main-tilia";
-	    # We want to fetch this remote every 2 seconds
-      poller.period = 2;
+	    # We want to fetch this remote every 2 seconds with a random delay up to 10 seconds
+      poller = {
+        period = 2;
+        random_delay = 10;
+      };
     }
   ];
   machineId = "22823ba6c96947e78b006c51a56fd89c";

docs/generated-module-options.md

@@ -180,6 +180,29 @@ string
 
 
 
+## services\.comin\.livelinessCheckCommand
+
+
+
+A command to be executed after a successful deployment to check if the new generation is healthy\. If the command returns a non-zero exit code, the deployment is considered failed and comin will roll back to the previous generation\.
+
+
+
+*Type:*
+null or string
+
+
+
+*Default:*
+` null `
+
+
+
+*Example:*
+` "curl --fail http://localhost:8080/health" `
+
+
+
 ## services\.comin\.machineId
 
 
@@ -425,6 +448,24 @@ signed integer
 
 
 
+## services\.comin\.remotes\.\*\.poller\.random_delay
+
+
+
+The maximum random delay in seconds before fetching the remote\.
+
+
+
+*Type:*
+signed integer
+
+
+
+*Default:*
+` 0 `
+
+
+
 ## services\.comin\.remotes\.\*\.timeout
 
 

docs/howtos.md

@@ -21,6 +21,20 @@ has to be on top of the `main` branch.
 To `nixos-rebuild switch` to this configuration, the `main` branch has
 to be rebased on the `testing` branch.
 
+## How to ensure a deployment is healthy
+
+After a deployment, you might want to run some checks to ensure that the new generation is healthy. For example, you might want to check that a web server is responding or that a specific service is running.
+
+Comin provides the `services.comin.livelinessCheckCommand` option to run a command after a successful deployment. If the command returns a non-zero exit code, the deployment is considered failed and comin will roll back to the previous generation.
+
+Here is an example of how to use it:
+
+```nix
+services.comin.livelinessCheckCommand = "curl --fail http://localhost:8080/health";
+```
+
+In this example, comin will run `curl --fail http://localhost:8080/health` after each deployment. If the command fails, comin will roll back to the previous generation.
+
 ## Iterate faster with local repository
 
 By default, comin polls remotes every 60 seconds. You could however
@@ -77,3 +91,13 @@ When comin is running on a Darwin system, it automatically builds and
 deploys a configuration found in the flake output
 `darwinConfigurations.hostname`. So, you only need to set this flake
 output and run comin on the target machine.
+
+## How to rollback to a previous generation
+
+Comin allows you to manually rollback to the last successful deployment. To do so, you can use the `rollback` command:
+
+```bash
+comin rollback --config /etc/comin/configuration.yaml
+```
+
+This command will find the last successful deployment and will redeploy it.

go.mod

@@ -5,21 +5,26 @@ go 1.23.0
 toolchain go1.24.1
 
 require (
+	github.com/Masterminds/sprig/v3 v3.3.0
 	github.com/ProtonMail/go-crypto v1.1.5
-	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
 	github.com/dustin/go-humanize v1.0.1
 	github.com/go-co-op/gocron/v2 v2.11.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/google/uuid v1.6.0
+	github.com/mattn/go-shellwords v1.0.12
 	github.com/prometheus/client_golang v1.19.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
+	google.golang.org/grpc v1.75.0
+	google.golang.org/protobuf v1.36.9
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	dario.cat/mergo v1.0.0 // indirect
+	dario.cat/mergo v1.0.1 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.3.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -30,19 +35,23 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/skeema/knownhosts v1.2.1 // indirect
+	github.com/spf13/cast v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	golang.org/x/crypto v0.39.0 // indirect
@@ -54,8 +63,6 @@ require (
 	golang.org/x/text v0.26.0 // indirect
 	golang.org/x/tools v0.33.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
-	google.golang.org/grpc v1.75.0 // indirect
-	google.golang.org/protobuf v1.36.9 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )