diff --git a/lib/dash/cli/app.rb b/lib/dash/cli/app.rb index 2dad98e7..51c6bc59 100644 --- a/lib/dash/cli/app.rb +++ b/lib/dash/cli/app.rb @@ -40,8 +40,8 @@ def boot # Tag once the app booted on all hosts on(DASH.app_hosts) do |host| - execute *DASH.auditor.record("Tagging #{DASH.config.absolute_image} as the latest image"), verbosity: :debug - execute *DASH.app.tag_latest_image + execute *DASH.auditor.record_then("Tagging #{DASH.config.absolute_image} as the latest image", + DASH.app.tag_latest_image) end end end diff --git a/lib/dash/cli/app/boot.rb b/lib/dash/cli/app/boot.rb index cc0d8b37..74e15c37 100644 --- a/lib/dash/cli/app/boot.rb +++ b/lib/dash/cli/app/boot.rb @@ -38,22 +38,36 @@ def run end private + # Both answers come back from one round trip, which means the running version is read + # before any rename happens. When the clashing container IS the running one, the + # version to stop later is the name it was renamed to - the name that was read now + # belongs to the container this boot is about to start. def old_version_renamed_if_clashing - if capture_with_info(*app.container_id_for_version(version), raise_on_non_zero_exit: false).present? + clashing_container_id, old_version = capture_boot_state + + if clashing_container_id.present? renamed_version = "#{version}_replaced_#{SecureRandom.hex(8)}" info "Renaming container #{version} to #{renamed_version} as already deployed on #{host}" - audit("Renaming container #{version} to #{renamed_version}") - execute *app.rename_container(version: version, new_version: renamed_version) + execute *auditor.record_then("Renaming container #{version} to #{renamed_version}", + app.rename_container(version: version, new_version: renamed_version)) + + old_version = renamed_version if old_version == version end - capture_with_info(*app.current_running_version, raise_on_non_zero_exit: false).strip.presence + old_version + end + + def capture_boot_state + output = capture_with_info(*app.boot_state(version), raise_on_non_zero_exit: false).to_s + clashing, _, running = output.partition(/^#{Regexp.escape(Dash::Commands::App::BOOT_STATE_SEPARATOR)}$/) + + [ clashing.strip.presence, running.strip.presence ] end def start_new_version - audit "Booted app version #{version}" hostname = "#{host.to_s[0...51].chomp(".")}-#{SecureRandom.hex(6)}" - execute *app.ensure_env_directory + execute *auditor.record_then("Booted app version #{version}", app.ensure_env_directory) upload! role.secrets_io(host), role.secrets_path, mode: "0600" execute *app.run(hostname: hostname) @@ -161,10 +175,6 @@ def auditor @auditor = DASH.auditor(role: role) end - def audit(message) - execute *auditor.record(message), verbosity: :debug - end - def gatekeeper? barrier && barrier_role? end diff --git a/lib/dash/cli/base.rb b/lib/dash/cli/base.rb index 06d6ef5c..f1c2ba6e 100644 --- a/lib/dash/cli/base.rb +++ b/lib/dash/cli/base.rb @@ -577,10 +577,18 @@ def reset_invocation(cli_class) instance_variable_get("@_invocations")[cli_class].pop end + # Every lock acquire wants the run directory to exist, but the sweep is idempotent + # and a process only needs it once per host - the deploy lock and the server lock + # were paying for it twice. def ensure_run_directory - on(DASH.hosts) do + pending = DASH.hosts.map(&:to_s) - DASH.run_directory_ensured_on + return if pending.empty? + + on(pending) do execute(*DASH.server.ensure_run_directory) end + + DASH.run_directory_ensured_on.concat(pending) end def with_env(env) diff --git a/lib/dash/cli/build.rb b/lib/dash/cli/build.rb index 1d65c5b6..45c66a8e 100644 --- a/lib/dash/cli/build.rb +++ b/lib/dash/cli/build.rb @@ -240,11 +240,13 @@ def mirror_hosts end end + # Audit, clean and pull share one round trip. validate_image keeps its own: folding it + # in would put the pull under validate_image's trailing `|| (echo ... && exit 1)`, and + # a failed pull would then report a missing service label. def pull_on_hosts(hosts) on(hosts) do - execute *DASH.auditor.record("Pulled image with version #{DASH.config.version}"), verbosity: :debug - execute *DASH.builder.clean, raise_on_non_zero_exit: false - execute *DASH.builder.pull + execute *DASH.auditor.record_then("Pulled image with version #{DASH.config.version}", + DASH.builder.clean_then_pull) execute *DASH.builder.validate_image end end diff --git a/lib/dash/cli/prune.rb b/lib/dash/cli/prune.rb index 890e4fec..48c22a42 100644 --- a/lib/dash/cli/prune.rb +++ b/lib/dash/cli/prune.rb @@ -11,9 +11,7 @@ def all def images modify(lock: true, server_lock: true) do on(DASH.hosts) do - execute *DASH.auditor.record("Pruned images"), verbosity: :debug - execute *DASH.prune.dangling_images - execute *DASH.prune.tagged_images + execute *DASH.auditor.record_then("Pruned images", DASH.prune.dangling_images, DASH.prune.tagged_images) end end end @@ -26,11 +24,10 @@ def containers modify(lock: true, server_lock: true) do on(DASH.hosts) do |host| - execute *DASH.auditor.record("Pruned containers"), verbosity: :debug - - DASH.roles_on(host).each do |role| - execute *DASH.prune.app_containers(retain: retain, role: role) - end + # One round trip per host, whatever it runs: a host with no app roles still + # records that the sweep reached it. + execute *DASH.auditor.record_then("Pruned containers", + *DASH.roles_on(host).map { |role| DASH.prune.app_containers(retain: retain, role: role) }) end end end diff --git a/lib/dash/commander.rb b/lib/dash/commander.rb index 43c29c57..b7e51fac 100644 --- a/lib/dash/commander.rb +++ b/lib/dash/commander.rb @@ -7,6 +7,12 @@ class Dash::Commander attr_accessor :verbosity, :holding_lock, :holding_server_lock, :connected, :logging, :lock_wait, :lock_wait_timeout, :lock_wait_interval attr_reader :specific_roles, :specific_hosts, :timings, :report + + # Hosts whose run directory this process has already swept, so the second lock acquire + # of a command does not re-run the migration everywhere. Per host rather than a flag: + # `dash upgrade` narrows the host set between acquires, and a host that was never in + # scope has never been swept. + attr_reader :run_directory_ensured_on delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :app_hosts, :proxy_hosts, :accessory_hosts, to: :specifics def initialize @@ -29,6 +35,7 @@ def reset @config = @config_kwargs = nil @output_logger = nil @commands = {} + @run_directory_ensured_on = [] end def config diff --git a/lib/dash/commands/app.rb b/lib/dash/commands/app.rb index 99be8f6f..c624c5c8 100644 --- a/lib/dash/commands/app.rb +++ b/lib/dash/commands/app.rb @@ -3,6 +3,10 @@ class Dash::Commands::App < Dash::Commands::Base ACTIVE_DOCKER_STATUSES = [ :running, :restarting ] + # Separates the two answers #boot_state returns. A container id is hex and a version is + # a name suffix, so neither can produce this line on its own. + BOOT_STATE_SEPARATOR = "--%--" + attr_reader :role, :host delegate :container_name, to: :role @@ -75,6 +79,20 @@ def current_running_version extract_version_from_name end + # Everything a boot needs to know about a host before it starts anything: whether a + # container for the version being deployed already exists (so it can be renamed out of + # the way) and which version is running now (so it can be stopped once the new one is + # live). Two questions, one round trip, answers split on BOOT_STATE_SEPARATOR. + # + # Chained with `;` rather than `&&`: an empty answer to either is a normal result, not + # a failure, and the second question must be asked whatever the first one said. + def boot_state(version) + chain \ + container_id_for_version(version), + [ :echo, BOOT_STATE_SEPARATOR ], + current_running_version + end + def list_versions(*docker_args, statuses: nil) pipe \ docker(:ps, *container_filter_args(statuses: statuses), *docker_args, "--format", '"{{.Names}}"'), diff --git a/lib/dash/commands/auditor.rb b/lib/dash/commands/auditor.rb index 5a22f927..e226520e 100644 --- a/lib/dash/commands/auditor.rb +++ b/lib/dash/commands/auditor.rb @@ -14,6 +14,16 @@ def record(line, **details) append([ :echo, escape_shell_value(audit_line(line, **details)) ], audit_log_file) end + # The audit line and the action it describes in one round trip, still in that order: + # the log is written first, and `&&` means a failed write aborts the action exactly as + # a failed standalone audit would have. + # + # Only ever fold in commands the caller would `execute`. A `capture` folded in here + # would come back with nothing to distinguish the audit's own output from the answer. + def record_then(line, *commands, **details) + combine record(line, **details), *commands + end + def reveal [ :tail, "-n", 50, audit_log_file ] end diff --git a/lib/dash/commands/builder.rb b/lib/dash/commands/builder.rb index 02c9c709..c48f8c43 100644 --- a/lib/dash/commands/builder.rb +++ b/lib/dash/commands/builder.rb @@ -2,7 +2,7 @@ class Dash::Commands::Builder < Dash::Commands::Base delegate \ - :create, :remove, :dev, :push, :clean, :pull, :info, :inspect_builder, + :create, :remove, :dev, :push, :clean, :pull, :clean_then_pull, :info, :inspect_builder, :validate_image, :first_mirror, :login_to_registry_locally?, :push_env, to: :target diff --git a/lib/dash/commands/builder/base.rb b/lib/dash/commands/builder/base.rb index ad36e7ac..4e48a1ae 100644 --- a/lib/dash/commands/builder/base.rb +++ b/lib/dash/commands/builder/base.rb @@ -14,6 +14,19 @@ def clean docker :image, :rm, "--force", config.absolute_image end + # Dropping the old image is housekeeping - a host that never had it is not an error - + # so it must not short-circuit whatever it shares a round trip with. + # + # The `|| true` is parenthesised because `&&` and `||` bind equally and associate left: + # ungrouped, an `audit && clean || true && pull` chain lets a FAILED audit fall into the + # same `|| true` and pull anyway, exit status 0. The group confines it to the clean. + # + # Composed only, never executed on its own: SSHKit's command map prefixes an unknown + # first word with /usr/bin/env, and the first word here is `(`. + def clean_then_pull + combine [ "(", *any(clean, [ :true ]), ")" ], pull + end + def push(export_action = "registry", tag_as_dirty: false, no_cache: false) docker :buildx, :build, "--output=type=#{export_action}", diff --git a/test/cli/app_test.rb b/test/cli/app_test.rb index 44795759..bf06deb6 100644 --- a/test/cli/app_test.rb +++ b/test/cli/app_test.rb @@ -19,13 +19,7 @@ class CliAppTest < CliTestCase Object.any_instance.stubs(:sleep) run_command("details") # Preheat Kamal const - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet", raise_on_non_zero_exit: false) - .returns("12345678") # running version - - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false) - .returns("123") # old version + stub_boot_state clash: "12345678", running: "123" SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet") @@ -41,6 +35,63 @@ class CliAppTest < CliTestCase Thread.report_on_exception = true end + # The clash check and the running-version read share one round trip, so the running + # version is now read BEFORE the clashing container is renamed. When they are the same + # container, the old version to stop is the name it was renamed to - stopping the name + # that was read would stop the container this boot just started. + test "boot stops the renamed container when the version being deployed was the running one" do + Object.any_instance.stubs(:sleep) + run_command("details") # Preheat Kamal const + + stub_boot_state clash: "12345678", running: "latest" + + SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) + .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet") + .returns("12345678") + + run_command("boot").tap do |output| + renamed = output[/docker rename app-web-latest (app-web-latest_replaced_[0-9a-f]{16})/, 1] + assert renamed, output + + assert_match "docker container ls --all --filter 'name=^#{renamed}$' --quiet | xargs docker stop", output + assert_no_match(/'name=\^app-web-latest\$' --quiet \| xargs docker stop/, output) + end + ensure + Thread.report_on_exception = true + end + + # Counted at the capture layer, not the Printer: both reads are captures, and a stubbed + # capture never reaches execute_command - so counting printed commands would pass + # whether or not the two were folded. + test "boot reads the clash check and the running version in a single round trip" do + Object.any_instance.stubs(:sleep) + + captures = [] + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with { |*args| captures << args.join(" "); true } + .returns("\n#{Dash::Commands::App::BOOT_STATE_SEPARATOR}\n123") + + run_command("boot") + + boot_state = captures.select { |capture| capture.include?(Dash::Commands::App::BOOT_STATE_SEPARATOR) } + assert_equal 1, boot_state.size, captures.inspect + assert_match "docker ps --latest", boot_state.first + + assert_equal 0, captures.count { |capture| capture.include?("docker ps --latest") && !capture.include?(Dash::Commands::App::BOOT_STATE_SEPARATOR) }, + "current_running_version should no longer be a capture of its own" + end + + # An audit line is a write to a file the action it describes is about to change. Folding + # it into the same shell string keeps "audit before action" and halves the round trips. + test "boot records the audit line in the same round trip as the action" do + stub_running + + run_command("boot").tap do |output| + assert_match %r{\[web\] Booted app version latest" >> \.dash/app-audit\.log && mkdir -p \.dash/apps/app/env/roles}, output + assert_match %r{Tagging dhh/app:latest as the latest image" >> \.dash/app-audit\.log && docker tag dhh/app:latest dhh/app:latest}, output + end + end + test "boot uses group strategy when specified" do Dash::Cli::App.any_instance.stubs(:on).with("1.1.1.1").twice Dash::Cli::App.any_instance.stubs(:on).with([ "1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4" ]).times(3) @@ -161,7 +212,8 @@ class CliAppTest < CliTestCase test "a canary opens the barrier for the roles booting after it" do Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -216,13 +268,13 @@ class CliAppTest < CliTestCase test "boot with assets" do Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet", raise_on_non_zero_exit: false) - .returns("12345678") # running version + # The assets step reads the running version on its own, before the boot does. SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false) - .returns("123").twice # old version + .returns("123") # old version + + stub_boot_state clash: "12345678", running: "123" SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet") @@ -241,18 +293,12 @@ class CliAppTest < CliTestCase test "boot with host tags" do Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet", raise_on_non_zero_exit: false) - .returns("12345678") # running version + stub_boot_state clash: "12345678", running: "123" SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet") .returns("12345678") # running version - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false) - .returns("123") # old version - run_command("boot", config: :with_env_tags).tap do |output| assert_match "docker tag dhh/app:latest dhh/app:latest", output assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --network dash --hostname 1.1.1.1-[0-9a-f]{12} --env KAMAL_CONTAINER_NAME="app-web-latest" --env KAMAL_VERSION="latest" --env KAMAL_HOST="1.1.1.1" --env TEST="root" --env EXPERIMENT="disabled" --env SITE="site1"}, output @@ -263,7 +309,8 @@ class CliAppTest < CliTestCase test "boot with web barrier opened" do Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -282,7 +329,8 @@ class CliAppTest < CliTestCase # guarantees the primary role goes first, and it has to survive that flip. Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -301,7 +349,8 @@ class CliAppTest < CliTestCase Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-latest$'", "--quiet", "|", "xargs docker logs --timestamps 2>&1") @@ -336,7 +385,8 @@ class CliAppTest < CliTestCase Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -359,7 +409,8 @@ class CliAppTest < CliTestCase Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -377,7 +428,8 @@ class CliAppTest < CliTestCase Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -407,7 +459,8 @@ class CliAppTest < CliTestCase Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -433,7 +486,8 @@ class CliAppTest < CliTestCase test "boot with only workers" do Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-workers-latest$'", "--quiet", "|", :xargs, :docker, :inspect, "--format", Dash::Commands::Base::DOCKER_HEALTH_STATUS_FORMAT) @@ -777,7 +831,8 @@ class CliAppTest < CliTestCase end test "boot proxy" do - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", config: :with_proxy).tap do |output| assert_match /Renaming container .* to .* as already deployed on 1.1.1.1/, output # Rename @@ -790,7 +845,8 @@ class CliAppTest < CliTestCase end test "boot proxy with role specific config" do - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", config: :with_proxy_roles, host: nil).tap do |output| assert_match "docker exec dash-proxy dash-proxy deploy app-web --target=\"123:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --target-timeout=\"10s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\"", output @@ -800,7 +856,8 @@ class CliAppTest < CliTestCase test "boot runs proxy deploy hooks around the proxy deploy" do Dash::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", config: :with_proxy).tap do |output| assert_hook_ran "pre-proxy-deploy", output @@ -814,7 +871,8 @@ class CliAppTest < CliTestCase Dash::Cli::App.any_instance.expects(:run_hook).with("pre-proxy-deploy", hosts: "1.1.1.1", role: "web") Dash::Cli::App.any_instance.expects(:run_hook).with("post-proxy-deploy", hosts: "1.1.1.1", role: "web") - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", config: :with_proxy) end @@ -822,7 +880,8 @@ class CliAppTest < CliTestCase test "boot skips proxy deploy hooks for roles not running the proxy" do Dash::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) Dash::Cli::Healthcheck::Poller.stubs(:wait_for_healthy) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", config: :with_proxy, host: "1.1.1.3").tap do |output| assert_hook_ran "pre-app-boot", output @@ -833,7 +892,8 @@ class CliAppTest < CliTestCase test "boot skips proxy deploy hooks with --skip-hooks" do Dash::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", "--skip-hooks", config: :with_proxy).tap do |output| assert_match /dash-proxy deploy app-web/, output @@ -844,7 +904,8 @@ class CliAppTest < CliTestCase test "boot aborts when the pre-proxy-deploy hook fails" do fail_hook("pre-proxy-deploy") - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false stderred { run_command("boot", config: :with_proxy, allow_execute_error: true) } @@ -876,7 +937,8 @@ class CliAppTest < CliTestCase test "boot runs app stop hooks for a non-proxied role" do Dash::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) Dash::Cli::Healthcheck::Poller.stubs(:wait_for_healthy) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", config: :with_proxy, host: "1.1.1.3").tap do |output| assert_no_match /hooks\/pre-proxy-deploy/, output @@ -886,7 +948,8 @@ class CliAppTest < CliTestCase test "boot runs app stop hooks for proxied roles too" do Dash::Commands::Hook.any_instance.stubs(:hook_exists?).returns(true) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false run_command("boot", config: :with_proxy).tap do |output| assert_match /dash-proxy deploy app-web.*pre-app-stop/m, output @@ -920,7 +983,8 @@ class CliAppTest < CliTestCase test "boot continues the deploy when the pre-app-stop hook fails" do fail_hook("pre-app-stop") Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false stderred { run_command("boot") } @@ -940,7 +1004,8 @@ class CliAppTest < CliTestCase test "boot leaves the old container running when the exec probe never passes" do Dash::Configuration.any_instance.stubs(:deploy_timeout).returns(0) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: "12345678", running: "123", expect: false @executions = [] SSHKit::Backend::Abstract.any_instance.stubs(:execute) @@ -1068,6 +1133,18 @@ def run_command(*command, config: :with_accessories, host: "1.1.1.1", allow_exec def stub_running Object.any_instance.stubs(:sleep) - SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # old version + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("123") # container id + stub_boot_state clash: nil, running: "123", expect: false + end + + # The one capture Dash::Cli::App::Boot makes before it starts anything: the id of a + # container already holding this version, then the version running now. + def stub_boot_state(clash:, running:, expect: true) + backend = SSHKit::Backend::Abstract.any_instance + matcher = ->(*args) { args.join(" ").include?(Dash::Commands::App::BOOT_STATE_SEPARATOR) } + + (expect ? backend.expects(:capture_with_info) : backend.stubs(:capture_with_info)) + .with(&matcher) + .returns("#{clash}\n#{Dash::Commands::App::BOOT_STATE_SEPARATOR}\n#{running}") end end diff --git a/test/cli/build_test.rb b/test/cli/build_test.rb index a1c927ad..f4c60b21 100644 --- a/test/cli/build_test.rb +++ b/test/cli/build_test.rb @@ -136,6 +136,9 @@ class CliBuildTest < CliTestCase test "a standalone push prints the build rows it measured" do Dash::Commands::Hook.any_instance.stubs(:hook_exists?).returns(false) + # Whether the checkout this runs in is dirty must not decide what is printed above + # the report - it used to, and the indent assertion below only held on a dirty tree. + Dash::Git.stubs(:uncommitted_changes).returns("") stub_build_stream "progress_plain_success" run_command("push", fixture: :without_clone).tap do |output| @@ -150,6 +153,7 @@ class CliBuildTest < CliTestCase # this build's numbers folded into it. test "a standalone push prints the advice under the build rows" do Dash::Commands::Hook.any_instance.stubs(:hook_exists?).returns(false) + Dash::Git.stubs(:uncommitted_changes).returns("") stub_build_stream "progress_plain_success" run_command("push", fixture: :with_report_advice).tap do |output| @@ -348,15 +352,28 @@ class CliBuildTest < CliTestCase assert @executions.none? { |args| args[0..2] == [ :docker, :build ] } end + # The audit line, the stale-image removal and the pull are one round trip per host: the + # audit is still written first, and the removal still cannot fail the pull. test "pull" do run_command("pull").tap do |output| assert_match /docker info --format '{{index .RegistryConfig.Mirrors 0}}'/, output - assert_match /docker image rm --force dhh\/app:999/, output - assert_match /docker pull dhh\/app:999/, output + assert_match %r{Pulled image with version 999" >> \.dash/app-audit\.log && \( docker image rm --force dhh/app:999 \|\| true \) && docker pull dhh/app:999}, output assert_match "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:999 | grep -x app || (echo \"Image dhh/app:999 is missing the 'service' label\" && exit 1)", output end end + test "pull issues two commands per host" do + commands = recorded_commands { run_command("pull") } + + pulls = commands.select { |command| command.include?("docker pull dhh/app:999") } + assert_equal DASH.app_hosts.size, pulls.size + assert pulls.all? { |command| command.include?("app-audit.log") && command.include?("docker image rm --force") }, pulls.inspect + + # An exact total, not a rounded average: integer division would swallow one extra + # command on a single host. + assert_equal 2 * DASH.app_hosts.size, commands.count { |command| command.include?("dhh/app:999") } + end + test "pull with mirror" do SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :info, "--format '{{index .RegistryConfig.Mirrors 0}}'") @@ -595,8 +612,10 @@ def stub_build_stream(fixture, failing: false) build.raises(SSHKit::Command::Failed.new("exit status: 1")) if failing end + # Deliberately not `stdouted`: it strips, which eats the leading indent of a report + # header printed as the first line of the run. Assertions here are about that indent. def run_command(*command, fixture: :with_accessories) - stdouted { stderred { Dash::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } } + capture(:stdout) { stderred { Dash::Cli::Build.start([ *command, "-c", "test/fixtures/deploy_#{fixture}.yml" ]) } } end def stub_dependency_checks diff --git a/test/cli/cli_test_case.rb b/test/cli/cli_test_case.rb index b7c38160..748a25dc 100644 --- a/test/cli/cli_test_case.rb +++ b/test/cli/cli_test_case.rb @@ -27,6 +27,26 @@ class CliTestCase < ActiveSupport::TestCase end private + # Every command the Printer backend was handed during the block, in order. Only what a + # caller `execute`s arrives here - a capture whose `capture_with_info` is stubbed is + # intercepted above this layer and never shows up, so a round-trip count that has to + # see captures must count those instead. + def recorded_commands + commands = [] + SSHKit::Backend::Printer.any_instance.stubs(:execute_command).with { |cmd| commands << cmd.to_command; true } + + begin + yield + ensure + # The stub swallows the command instead of printing it, and mocha would leave it + # standing until the end of the test - so anything run after the block would be + # silently invisible. Recording stops where the block does. + SSHKit::Backend::Printer.any_instance.unstub(:execute_command) + end + + commands + end + # A real `docker buildx build --progress=plain` stream, parsed by the same handler a # build attaches. Cheaper than a Docker daemon and it proves the wiring end to end. def build_report_from_fixture(name = "progress_plain_success") diff --git a/test/cli/main_test.rb b/test/cli/main_test.rb index 1676fb30..c5a6f94f 100644 --- a/test/cli/main_test.rb +++ b/test/cli/main_test.rb @@ -447,15 +447,13 @@ class CliMainTest < CliTestCase test "rollback good version" do Object.any_instance.stubs(:sleep) [ "web", "workers" ].each do |role| + # One capture: no clashing container for 123, version-to-rollback running now. SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-#{role}-123$'", "--quiet", raise_on_non_zero_exit: false) - .returns("").at_least_once + .with { |*args| args.join(" ").include?("'name=^app-#{role}-123$'") && args.join(" ").include?(Dash::Commands::App::BOOT_STATE_SEPARATOR) } + .returns("\n#{Dash::Commands::App::BOOT_STATE_SEPARATOR}\nversion-to-rollback\n").at_least_once SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-#{role}-123$'", "--quiet") .returns("version-to-rollback\n").at_least_once - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=#{role} --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=#{role} --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-#{role}-}; done", raise_on_non_zero_exit: false) - .returns("version-to-rollback\n").at_least_once end SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) @@ -477,14 +475,11 @@ class CliMainTest < CliTestCase Dash::Cli::Main.any_instance.stubs(:container_available?).returns(true) SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-123$'", "--quiet", raise_on_non_zero_exit: false) - .returns("").at_least_once + .with { |*args| args.join(" ").include?(Dash::Commands::App::BOOT_STATE_SEPARATOR) } + .returns("\n#{Dash::Commands::App::BOOT_STATE_SEPARATOR}\n").at_least_once # no clash, nothing running SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) .with(:docker, :container, :ls, "--all", "--filter", "'name=^app-web-123$'", "--quiet") .returns("123").at_least_once - SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) - .with(:sh, "-c", "'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting'", "|", :head, "-1", "|", "while read line; do echo ${line#app-web-}; done", raise_on_non_zero_exit: false) - .returns("").at_least_once run_command("rollback", "123").tap do |output| assert_match "docker run --detach --restart unless-stopped --name app-web-123", output @@ -1267,7 +1262,7 @@ class CliMainTest < CliTestCase test "deploy issues no commands beyond the pinned sequence" do Dash::Cli::Main.any_instance.stubs(:invoke) - assert_equal DEPLOY_COMMAND_SEQUENCE, recorded_commands { run_command("deploy", "--skip_push") } + assert_equal DEPLOY_COMMAND_SEQUENCE, recorded_deploy_commands { run_command("deploy", "--skip_push") } end private @@ -1311,13 +1306,8 @@ def measured_bundle_install # The lock details are a base64 blob of the operator, the time and the version, so # they differ on every run and every machine. The command around them is the point. - def recorded_commands - commands = [] - SSHKit::Backend::Printer.any_instance.stubs(:execute_command).with { |cmd| commands << cmd.to_command; true } - - yield - - commands.map { |command| command.gsub(/echo "[^"]*"/m, %(echo "
")) } + def recorded_deploy_commands(&block) + recorded_commands(&block).map { |command| command.gsub(/echo "[^"]*"/m, %(echo "
")) } end def run_command(*command, config_file: "deploy_simple") diff --git a/test/cli/proxy_test.rb b/test/cli/proxy_test.rb index 9df80d63..0d2a0ce5 100644 --- a/test/cli/proxy_test.rb +++ b/test/cli/proxy_test.rb @@ -502,6 +502,10 @@ class CliProxyTest < CliTestCase Object.any_instance.stubs(:sleep) SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info).returns("12345678") + # A container already holds the version being booted, and 12345678 is running. + SSHKit::Backend::Abstract.any_instance.stubs(:capture_with_info) + .with { |*args| args.join(" ").include?(Dash::Commands::App::BOOT_STATE_SEPARATOR) } + .returns("12345678\n#{Dash::Commands::App::BOOT_STATE_SEPARATOR}\n12345678") stub_no_proxy_drift SSHKit::Backend::Abstract.any_instance.expects(:capture_with_info) @@ -525,7 +529,7 @@ class CliProxyTest < CliTestCase assert_match "docker container start dash-proxy || echo $(cat .dash/proxy/options 2> /dev/null || echo \"--publish 80:80 --publish 443:443 --log-opt max-size=10m\") $(cat .dash/proxy/image 2> /dev/null || echo \"ghcr.io/zoolutions/dash-proxy\"):$(cat .dash/proxy/image_version 2> /dev/null || echo \"#{Dash::Configuration::Proxy::Run::MINIMUM_VERSION}\") $(cat .dash/proxy/run_command 2> /dev/null || echo \"\") | xargs docker run --name dash-proxy --network dash --detach --restart unless-stopped --volume dash-proxy-config:/home/dash-proxy/.config/dash-proxy", output assert_match "/usr/bin/env mkdir -p .dash", output assert_match %r{docker rename app-web-latest app-web-latest_replaced_.*}, output - assert_match "/usr/bin/env mkdir -p .dash/apps/app/env/roles", output + assert_match "Booted app version latest\" >> .dash/app-audit.log && mkdir -p .dash/apps/app/env/roles", output assert_match "Uploading \"\\n\" to .dash/apps/app/env/roles/web.env", output assert_match %r{docker run --detach --restart unless-stopped --name app-web-latest --network dash --hostname 1.1.1.1-.* --env KAMAL_CONTAINER_NAME="app-web-latest" --env KAMAL_VERSION="latest" --env KAMAL_HOST="1.1.1.1" --env-file .dash/apps/app/env/roles/web.env --log-opt max-size="10m" --label service="app" --label role="web" --label destination dhh/app:latest}, output assert_match "docker exec dash-proxy dash-proxy deploy app-web --target=\"12345678:80\" --deploy-timeout=\"6s\" --drain-timeout=\"30s\" --buffer-requests --buffer-responses --log-request-header=\"Cache-Control\" --log-request-header=\"Last-Modified\" --log-request-header=\"User-Agent\"", output diff --git a/test/cli/prune_test.rb b/test/cli/prune_test.rb index f6d80c0d..b4828455 100644 --- a/test/cli/prune_test.rb +++ b/test/cli/prune_test.rb @@ -10,7 +10,7 @@ class CliPruneTest < CliTestCase test "images" do run_command("images").tap do |output| - assert_match "docker image prune --force --filter label=service=app on 1.1.1.", output + assert_match "docker image prune --force --filter label=service=app && docker image ls", output assert_match "docker image ls --filter label=service=app --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep -v -w \"$(docker container ls -a --format '{{.Image}}\\|' --filter label=service=app | tr -d '\\n')dhh/app:latest\\|dhh/app:\" | while read image tag; do docker rmi $tag; done on 1.1.1.", output end end @@ -36,6 +36,37 @@ class CliPruneTest < CliTestCase end end + # A prune sweep is an audit line plus one docker command per thing being pruned. They + # ran as separate round trips; now the audit leads the same shell string. + test "images prunes in one round trip per host" do + commands = recorded_commands { run_command("images") } + + prunes = commands.select { |command| command.include?("docker image prune") } + assert_equal DASH.hosts.size, prunes.size + assert prunes.all? { |command| command.include?("Pruned images") && command.include?("docker image ls") }, prunes.inspect + end + + test "containers prunes a host's roles in one round trip" do + commands = recorded_commands { run_command("containers", config_file: "test/fixtures/deploy_with_roles.yml") } + + prunes = commands.select { |command| command.include?("Pruned containers") } + assert_equal DASH.hosts.size, prunes.size + assert prunes.any? { |command| command.include?("label=role=web --filter status=created") }, prunes.inspect + assert prunes.any? { |command| command.include?("label=role=workers --filter status=created") }, prunes.inspect + end + + # The run directory is swept once per host per process. `prune all` takes the deploy + # lock and then the server lock, and the second acquire used to re-sweep every host. + test "the run directory is ensured once per host across both locks" do + Dash::Cli::Prune.any_instance.stubs(:containers) + Dash::Cli::Prune.any_instance.stubs(:images) + + commands = recorded_commands { run_command("all") } + + sweeps = commands.select { |command| command == "test -d .kamal && test ! -e .dash && mv .kamal .dash || true && mkdir -p .dash" } + assert_equal DASH.hosts.size, sweeps.size + end + private def run_command(*command, config_file: "test/fixtures/deploy_with_accessories.yml") stdouted { Dash::Cli::Prune.start([ *command, "-c", config_file ]) } diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index 1fd146db..ea58c5f8 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -591,6 +591,13 @@ class CommandsAppTest < ActiveSupport::TestCase new_command.current_running_version.join(" ") end + test "boot_state pairs the version clash check with the running version in one command" do + assert_equal \ + "docker container ls --all --filter 'name=^app-web-999$' --quiet ; echo --%-- ; " \ + "sh -c 'docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting --filter ancestor=$(docker image ls --filter reference=dhh/app:latest --format '\\''{{.ID}}'\\'') ; docker ps --latest --format '\\''{{.Names}}'\\'' --filter label=service=app --filter label=destination= --filter label=role=web --filter status=running --filter status=restarting' | head -1 | while read line; do echo ${line#app-web-}; done", + new_command.boot_state("999").join(" ") + end + test "list_versions" do assert_equal \ "docker ps --filter label=service=app --filter label=destination= --filter label=role=web --format \"{{.Names}}\" | while read line; do echo ${line#app-web-}; done", diff --git a/test/commands/auditor_test.rb b/test/commands/auditor_test.rb index 1bf70ce8..c996e381 100644 --- a/test/commands/auditor_test.rb +++ b/test/commands/auditor_test.rb @@ -25,6 +25,21 @@ class CommandsAuditorTest < ActiveSupport::TestCase ], @auditor.record("app removed container") end + test "record_then puts the audit line and the action it describes in one command" do + assert_equal [ + *ENSURE_RUN_DIRECTORY, "&&", + :echo, + "\"[#{@recorded_at}] [#{@performer}] Pruned images\"", + ">>", ".dash/app-audit.log", "&&", + :docker, :image, :prune, "&&", + :docker, :image, :ls + ], @auditor.record_then("Pruned images", [ :docker, :image, :prune ], [ :docker, :image, :ls ]) + end + + test "record_then with no action is just the audit line" do + assert_equal @auditor.record("Pruned containers"), @auditor.record_then("Pruned containers") + end + test "record with destination" do new_command(destination: "staging").tap do |auditor| assert_equal [ diff --git a/test/commands/builder_test.rb b/test/commands/builder_test.rb index 10e869bf..baa91f58 100644 --- a/test/commands/builder_test.rb +++ b/test/commands/builder_test.rb @@ -175,6 +175,14 @@ class CommandsBuilderTest < ActiveSupport::TestCase builder.target.build_options.join(" ") end + # The `|| true` must stay inside the parentheses: `&&` and `||` associate left, so an + # ungrouped form lets a failure EARLIER in the composed chain (the audit write this is + # folded behind) fall into the same `|| true` and pull anyway. + test "clean_then_pull groups the best-effort clean" do + assert_equal "( docker image rm --force dhh/app:123 || true ) && docker pull dhh/app:123", + new_builder_command.clean_then_pull.join(" ") + end + test "validate image" do assert_equal "docker inspect -f '{{ .Config.Labels.service }}' dhh/app:123 | grep -x app || (echo \"Image dhh/app:123 is missing the 'service' label\" && exit 1)", new_builder_command.validate_image.join(" ") end