Add repository consolidation

agent/agent_configuration.go

@@ -30,4 +30,5 @@ type AgentConfiguration struct {
 	Profile                    string
 	RedactedVars               []string
 	AcquireJob                 string
+	ShouldConsolidateRepos     bool
 }

agent/job_runner.go

@@ -399,6 +399,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 		`BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT`,
 		`BUILDKITE_GIT_CLEAN_FLAGS`,
 		`BUILDKITE_SHELL`,
+		`BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR`,
 	}
 
 	var ignoredEnv []string
@@ -448,6 +449,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 	env["BUILDKITE_SHELL"] = r.conf.AgentConfiguration.Shell
 	env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(), ",")
 	env["BUILDKITE_REDACTED_VARS"] = strings.Join(r.conf.AgentConfiguration.RedactedVars, ",")
+	env["BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.ShouldConsolidateRepos)
 
 	// Whether to enable profiling in the bootstrap
 	if r.conf.AgentConfiguration.Profile != "" {

bootstrap/bootstrap.go

@@ -2,6 +2,8 @@ package bootstrap
 
 import (
 	"context"
+	"crypto/md5"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -424,8 +426,18 @@ func (b *Bootstrap) setUp() error {
 		if b.BuildPath == "" {
 			return fmt.Errorf("Must set either a BUILDKITE_BUILD_PATH or a BUILDKITE_BUILD_CHECKOUT_PATH")
 		}
-		b.shell.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH",
-			filepath.Join(b.BuildPath, dirForAgentName(b.AgentName), b.OrganizationSlug, b.PipelineSlug))
+
+		// If we should consolidate repos, hash the repository path to use as the build dir
+		if b.Config.ShouldConsolidateRepos && b.Config.Repository != "" {
+
+			hashedRepositoryName := hashRepositoryString(b.Config.Repository)
+
+			b.shell.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH",
+				filepath.Join(b.BuildPath, dirForAgentName(b.AgentName), b.OrganizationSlug, hashedRepositoryName))
+		} else {
+			b.shell.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH",
+				filepath.Join(b.BuildPath, dirForAgentName(b.AgentName), b.OrganizationSlug, b.PipelineSlug))
+		}
 	}
 
 	// The job runner sets BUILDKITE_IGNORED_ENV with any keys that were ignored
@@ -838,6 +850,10 @@ func (b *Bootstrap) removeCheckoutDir() error {
 func (b *Bootstrap) createCheckoutDir() error {
 	checkoutPath, _ := b.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH")
 
+	if b.Config.ShouldConsolidateRepos {
+		b.shell.Commentf("Using hashed repository name as build dir instead of %s to consolidate builds into single dir", b.PipelineSlug)
+	}
+
 	if !fileExists(checkoutPath) {
 		b.shell.Commentf("Creating \"%s\"", checkoutPath)
 		if err := os.MkdirAll(checkoutPath, 0777); err != nil {
@@ -1641,6 +1657,14 @@ func getValuesToRedact(logger shell.Logger, patterns []string, environment map[s
 	return valuesToRedact
 }
 
+func hashRepositoryString(repository string) string {
+	hasher := md5.New()
+	hasher.Write([]byte(repository))
+	hashedRepostoryName := hex.EncodeToString(hasher.Sum(nil))
+
+	return hashedRepostoryName
+}
+
 type pluginCheckout struct {
 	*plugin.Plugin
 	*plugin.Definition

bootstrap/config.go

@@ -123,6 +123,9 @@ type Config struct {
 
 	// List of environment variable globs to redact from job output
 	RedactedVars []string
+
+	// Whether or not to consolidate identical repositories into single build directory
+	ShouldConsolidateRepos bool
 }
 
 // ReadFromEnvironment reads configuration from the Environment, returns a map

clicommand/agent_start.go

@@ -88,6 +88,7 @@ type AgentStartConfig struct {
 	LogFormat                  string   `cli:"log-format"`
 	CancelSignal               string   `cli:"cancel-signal"`
 	RedactedVars               []string `cli:"redacted-vars" normalize:"list"`
+	ShouldConsolidateRepos     bool     `cli:"should-consolidate-repos"`
 
 	// Global flags
 	Debug       bool     `cli:"debug"`
@@ -400,6 +401,11 @@ var AgentStartCommand = cli.Command{
 			EnvVar: "BUILDKITE_REDACTED_VARS",
 			Value:  &cli.StringSlice{"*_PASSWORD", "*_SECRET", "*_TOKEN"},
 		},
+		cli.BoolFlag{
+			Name:   "should-consolidate-repos",
+			Usage:  "Consolidate all builds using identical repositories into a single build directory",
+			EnvVar: "BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR",
+		},
 
 		// API Flags
 		AgentRegisterTokenFlag,
@@ -587,6 +593,7 @@ var AgentStartCommand = cli.Command{
 			Shell:                      cfg.Shell,
 			RedactedVars:               cfg.RedactedVars,
 			AcquireJob:                 cfg.AcquireJob,
+			ShouldConsolidateRepos:     cfg.ShouldConsolidateRepos,
 		}
 
 		if loader.File != nil {
@@ -649,6 +656,10 @@ var AgentStartCommand = cli.Command{
 			l.Info("Agents will disconnect after %d seconds of inactivity", agentConf.DisconnectAfterIdleTimeout)
 		}
 
+		if agentConf.ShouldConsolidateRepos {
+			l.Info("Agent will use hashed name of repository for the build dir of all jobs using identical repositories")
+		}
+
 		cancelSig, err := process.ParseSignal(cfg.CancelSignal)
 		if err != nil {
 			l.Fatal("Failed to parse cancel-signal: %v", err)

clicommand/bootstrap.go

@@ -81,6 +81,7 @@ type BootstrapConfig struct {
 	Phases                       []string `cli:"phases" normalize:"list"`
 	Profile                      string   `cli:"profile"`
 	RedactedVars                 []string `cli:"redacted-vars" normalize:"list"`
+	ShouldConsolidateRepos       bool     `cli:"should-consolidate-repos"`
 }
 
 var BootstrapCommand = cli.Command{
@@ -294,6 +295,11 @@ var BootstrapCommand = cli.Command{
 			Usage:  "Pattern of environment variable names containing sensitive values",
 			EnvVar: "BUILDKITE_REDACTED_VARS",
 		},
+		cli.BoolTFlag{
+			Name:   "should-consolidate-repos",
+			Usage:  "Consolidate all builds using identical repositories into a single build directory",
+			EnvVar: "BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR",
+		},
 		DebugFlag,
 		ExperimentsFlag,
 		ProfileFlag,
@@ -378,6 +384,7 @@ var BootstrapCommand = cli.Command{
 			Shell:                        cfg.Shell,
 			Phases:                       cfg.Phases,
 			RedactedVars:                 cfg.RedactedVars,
+			ShouldConsolidateRepos:       cfg.ShouldConsolidateRepos,
 		})
 
 		ctx, cancel := context.WithCancel(context.Background())

packaging/docker/alpine-linux/buildkite-agent.cfg

@@ -59,3 +59,6 @@ plugins-path="/buildkite/plugins"
 
 # Don't show colors in logging
 # no-color=true
+
+# Consolidate all builds using identical repositories into a single build directory
+# should-consolidate-repos=false

packaging/docker/centos-linux/buildkite-agent.cfg

@@ -59,3 +59,6 @@ plugins-path="/buildkite/plugins"
 
 # Don't show colors in logging
 # no-color=true
+
+# Consolidate all builds using identical repositories into a single build directory
+# should-consolidate-repos=false

packaging/docker/ubuntu-linux/buildkite-agent.cfg

@@ -59,3 +59,6 @@ plugins-path="/buildkite/plugins"
 
 # Don't show colors in logging
 # no-color=true
+
+# Consolidate all builds using identical repositories into a single build directory
+# should-consolidate-repos=false

packaging/github/linux/buildkite-agent.cfg

@@ -59,3 +59,6 @@ plugins-path="$HOME/.buildkite-agent/plugins"
 
 # Don't show colors in logging
 # no-color=true
+
+# Consolidate all builds using identical repositories into a single build directory
+# should-consolidate-repos=false

packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg

@@ -59,3 +59,6 @@ plugins-path="/etc/buildkite-agent/plugins"
 
 # Don't show colors in logging
 # no-color=true
+
+# Consolidate all builds using identical repositories into a single build directory
+# should-consolidate-repos=false