diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index e9969f5dd8..4043fbefef 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -30,4 +30,5 @@ type AgentConfiguration struct { Profile string RedactedVars []string AcquireJob string + ShouldConsolidateRepos bool } diff --git a/agent/job_runner.go b/agent/job_runner.go index bdb892605c..8b8151e525 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -399,6 +399,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) { `BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT`, `BUILDKITE_GIT_CLEAN_FLAGS`, `BUILDKITE_SHELL`, + `BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR`, } var ignoredEnv []string @@ -448,6 +449,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) { env["BUILDKITE_SHELL"] = r.conf.AgentConfiguration.Shell env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(), ",") env["BUILDKITE_REDACTED_VARS"] = strings.Join(r.conf.AgentConfiguration.RedactedVars, ",") + env["BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.ShouldConsolidateRepos) // Whether to enable profiling in the bootstrap if r.conf.AgentConfiguration.Profile != "" { diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index b9d597b530..2f28b07de2 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -2,6 +2,8 @@ package bootstrap import ( "context" + "crypto/md5" + "encoding/hex" "encoding/json" "fmt" "io" @@ -424,8 +426,18 @@ func (b *Bootstrap) setUp() error { if b.BuildPath == "" { return fmt.Errorf("Must set either a BUILDKITE_BUILD_PATH or a BUILDKITE_BUILD_CHECKOUT_PATH") } - b.shell.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", - filepath.Join(b.BuildPath, dirForAgentName(b.AgentName), b.OrganizationSlug, b.PipelineSlug)) + + // If we should consolidate repos, hash the repository path to use as the build dir + if b.Config.ShouldConsolidateRepos && b.Config.Repository != "" { + + hashedRepositoryName := hashRepositoryString(b.Config.Repository) + + b.shell.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", + filepath.Join(b.BuildPath, dirForAgentName(b.AgentName), b.OrganizationSlug, hashedRepositoryName)) + } else { + b.shell.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", + filepath.Join(b.BuildPath, dirForAgentName(b.AgentName), b.OrganizationSlug, b.PipelineSlug)) + } } // The job runner sets BUILDKITE_IGNORED_ENV with any keys that were ignored @@ -838,6 +850,10 @@ func (b *Bootstrap) removeCheckoutDir() error { func (b *Bootstrap) createCheckoutDir() error { checkoutPath, _ := b.shell.Env.Get("BUILDKITE_BUILD_CHECKOUT_PATH") + if b.Config.ShouldConsolidateRepos { + b.shell.Commentf("Using hashed repository name as build dir instead of %s to consolidate builds into single dir", b.PipelineSlug) + } + if !fileExists(checkoutPath) { b.shell.Commentf("Creating \"%s\"", checkoutPath) if err := os.MkdirAll(checkoutPath, 0777); err != nil { @@ -1641,6 +1657,14 @@ func getValuesToRedact(logger shell.Logger, patterns []string, environment map[s return valuesToRedact } +func hashRepositoryString(repository string) string { + hasher := md5.New() + hasher.Write([]byte(repository)) + hashedRepostoryName := hex.EncodeToString(hasher.Sum(nil)) + + return hashedRepostoryName +} + type pluginCheckout struct { *plugin.Plugin *plugin.Definition diff --git a/bootstrap/config.go b/bootstrap/config.go index ddc63daa5c..3f8418fa7d 100644 --- a/bootstrap/config.go +++ b/bootstrap/config.go @@ -123,6 +123,9 @@ type Config struct { // List of environment variable globs to redact from job output RedactedVars []string + + // Whether or not to consolidate identical repositories into single build directory + ShouldConsolidateRepos bool } // ReadFromEnvironment reads configuration from the Environment, returns a map diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 9a7476b023..30dbe72ab9 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -88,6 +88,7 @@ type AgentStartConfig struct { LogFormat string `cli:"log-format"` CancelSignal string `cli:"cancel-signal"` RedactedVars []string `cli:"redacted-vars" normalize:"list"` + ShouldConsolidateRepos bool `cli:"should-consolidate-repos"` // Global flags Debug bool `cli:"debug"` @@ -400,6 +401,11 @@ var AgentStartCommand = cli.Command{ EnvVar: "BUILDKITE_REDACTED_VARS", Value: &cli.StringSlice{"*_PASSWORD", "*_SECRET", "*_TOKEN"}, }, + cli.BoolFlag{ + Name: "should-consolidate-repos", + Usage: "Consolidate all builds using identical repositories into a single build directory", + EnvVar: "BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR", + }, // API Flags AgentRegisterTokenFlag, @@ -587,6 +593,7 @@ var AgentStartCommand = cli.Command{ Shell: cfg.Shell, RedactedVars: cfg.RedactedVars, AcquireJob: cfg.AcquireJob, + ShouldConsolidateRepos: cfg.ShouldConsolidateRepos, } if loader.File != nil { @@ -649,6 +656,10 @@ var AgentStartCommand = cli.Command{ l.Info("Agents will disconnect after %d seconds of inactivity", agentConf.DisconnectAfterIdleTimeout) } + if agentConf.ShouldConsolidateRepos { + l.Info("Agent will use hashed name of repository for the build dir of all jobs using identical repositories") + } + cancelSig, err := process.ParseSignal(cfg.CancelSignal) if err != nil { l.Fatal("Failed to parse cancel-signal: %v", err) diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go index e42e424595..f3d84420b9 100644 --- a/clicommand/bootstrap.go +++ b/clicommand/bootstrap.go @@ -81,6 +81,7 @@ type BootstrapConfig struct { Phases []string `cli:"phases" normalize:"list"` Profile string `cli:"profile"` RedactedVars []string `cli:"redacted-vars" normalize:"list"` + ShouldConsolidateRepos bool `cli:"should-consolidate-repos"` } var BootstrapCommand = cli.Command{ @@ -294,6 +295,11 @@ var BootstrapCommand = cli.Command{ Usage: "Pattern of environment variable names containing sensitive values", EnvVar: "BUILDKITE_REDACTED_VARS", }, + cli.BoolTFlag{ + Name: "should-consolidate-repos", + Usage: "Consolidate all builds using identical repositories into a single build directory", + EnvVar: "BUILDKITE_CONSOLIDATE_REPOS_INTO_BUILD_DIR", + }, DebugFlag, ExperimentsFlag, ProfileFlag, @@ -378,6 +384,7 @@ var BootstrapCommand = cli.Command{ Shell: cfg.Shell, Phases: cfg.Phases, RedactedVars: cfg.RedactedVars, + ShouldConsolidateRepos: cfg.ShouldConsolidateRepos, }) ctx, cancel := context.WithCancel(context.Background()) diff --git a/packaging/docker/alpine-linux/buildkite-agent.cfg b/packaging/docker/alpine-linux/buildkite-agent.cfg index f6ef8d059f..e820f21d70 100644 --- a/packaging/docker/alpine-linux/buildkite-agent.cfg +++ b/packaging/docker/alpine-linux/buildkite-agent.cfg @@ -59,3 +59,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true + +# Consolidate all builds using identical repositories into a single build directory +# should-consolidate-repos=false diff --git a/packaging/docker/centos-linux/buildkite-agent.cfg b/packaging/docker/centos-linux/buildkite-agent.cfg index f6ef8d059f..e820f21d70 100644 --- a/packaging/docker/centos-linux/buildkite-agent.cfg +++ b/packaging/docker/centos-linux/buildkite-agent.cfg @@ -59,3 +59,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true + +# Consolidate all builds using identical repositories into a single build directory +# should-consolidate-repos=false diff --git a/packaging/docker/ubuntu-linux/buildkite-agent.cfg b/packaging/docker/ubuntu-linux/buildkite-agent.cfg index f6ef8d059f..e820f21d70 100644 --- a/packaging/docker/ubuntu-linux/buildkite-agent.cfg +++ b/packaging/docker/ubuntu-linux/buildkite-agent.cfg @@ -59,3 +59,6 @@ plugins-path="/buildkite/plugins" # Don't show colors in logging # no-color=true + +# Consolidate all builds using identical repositories into a single build directory +# should-consolidate-repos=false diff --git a/packaging/github/linux/buildkite-agent.cfg b/packaging/github/linux/buildkite-agent.cfg index af1b10b8d3..f5c9a07769 100644 --- a/packaging/github/linux/buildkite-agent.cfg +++ b/packaging/github/linux/buildkite-agent.cfg @@ -59,3 +59,6 @@ plugins-path="$HOME/.buildkite-agent/plugins" # Don't show colors in logging # no-color=true + +# Consolidate all builds using identical repositories into a single build directory +# should-consolidate-repos=false diff --git a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg index d532798212..f77146af47 100644 --- a/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg +++ b/packaging/linux/root/usr/share/buildkite-agent/buildkite-agent.cfg @@ -59,3 +59,6 @@ plugins-path="/etc/buildkite-agent/plugins" # Don't show colors in logging # no-color=true + +# Consolidate all builds using identical repositories into a single build directory +# should-consolidate-repos=false