diff --git a/cmd/ci-operator/main.go b/cmd/ci-operator/main.go index 916b2619b6..6da24aee15 100644 --- a/cmd/ci-operator/main.go +++ b/cmd/ci-operator/main.go @@ -1096,6 +1096,8 @@ func (o *options) Run() (errs []error) { info := o.getResolverInfo(o.jobSpec) o.metricsAgent.RecordConfigurationInsight(o.targets.values, o.promote, info.Org, info.Repo, info.Branch, info.Variant, o.baseNamespace, o.consoleHost, o.nodeName, o.clusterProfiles) + o.adjustLeaseAcquireTimeout() + if runErrs := interrupt.New(handler, o.saveNamespaceArtifacts).Run(func() []error { if leaseClient != nil { if err := o.initializeLeaseClient(); err != nil { @@ -2070,6 +2072,22 @@ func loadLeaseCredentials(leaseServerCredentialsFile string) (string, func() []b return username, passwordGetter, nil } +// adjustLeaseAcquireTimeout increases the lease acquire timeout to match the +// test timeout when a target test has a longer timeout configured. This +// prevents jobs from failing to acquire leases when the pool is busy, even +// though the overall job timeout would allow waiting longer. +func (o *options) adjustLeaseAcquireTimeout() { + for _, test := range o.configSpec.Tests { + if len(o.targets.values) > 0 && !slices.Contains(o.targets.values, test.As) { + continue + } + if test.Timeout != nil && test.Timeout.Duration > o.leaseAcquireTimeout { + logrus.Infof("Adjusting lease acquire timeout from %s to %s to match test %q timeout", o.leaseAcquireTimeout, test.Timeout.Duration, test.As) + o.leaseAcquireTimeout = test.Timeout.Duration + } + } +} + func (o *options) initializeLeaseClient() error { var err error owner := o.namespace + "-" + o.jobSpec.UniqueHash() diff --git a/cmd/ci-operator/main_test.go b/cmd/ci-operator/main_test.go index 5c41913aa3..e4482fa7d8 100644 --- a/cmd/ci-operator/main_test.go +++ b/cmd/ci-operator/main_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "reflect" "testing" + "time" "github.com/google/go-cmp/cmp" @@ -1820,3 +1821,78 @@ func TestGetClusterProfileNamesFromTargets(t *testing.T) { }) } } + +func TestAdjustLeaseAcquireTimeout(t *testing.T) { + dur := func(d time.Duration) *prowapi.Duration { + return &prowapi.Duration{Duration: d} + } + testCases := []struct { + name string + initialTimeout time.Duration + tests []api.TestStepConfiguration + targets []string + expectedTimeout time.Duration + }{ + { + name: "no matching test leaves timeout unchanged", + initialTimeout: 2 * time.Hour, + tests: []api.TestStepConfiguration{ + {As: "other", Timeout: dur(10 * time.Hour)}, + }, + targets: []string{"e2e-parallel"}, + expectedTimeout: 2 * time.Hour, + }, + { + name: "matching test with larger timeout adjusts lease timeout", + initialTimeout: 2 * time.Hour, + tests: []api.TestStepConfiguration{ + {As: "e2e-parallel", Timeout: dur(10 * time.Hour)}, + }, + targets: []string{"e2e-parallel"}, + expectedTimeout: 10 * time.Hour, + }, + { + name: "matching test with smaller timeout does not reduce lease timeout", + initialTimeout: 2 * time.Hour, + tests: []api.TestStepConfiguration{ + {As: "e2e-parallel", Timeout: dur(1 * time.Hour)}, + }, + targets: []string{"e2e-parallel"}, + expectedTimeout: 2 * time.Hour, + }, + { + name: "matching test with no timeout leaves lease timeout unchanged", + initialTimeout: 2 * time.Hour, + tests: []api.TestStepConfiguration{ + {As: "e2e-parallel"}, + }, + targets: []string{"e2e-parallel"}, + expectedTimeout: 2 * time.Hour, + }, + { + name: "no explicit targets with long test timeout adjusts lease timeout", + initialTimeout: 2 * time.Hour, + tests: []api.TestStepConfiguration{ + {As: "e2e-parallel", Timeout: dur(10 * time.Hour)}, + }, + targets: nil, + expectedTimeout: 10 * time.Hour, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + o := &options{ + leaseAcquireTimeout: tc.initialTimeout, + configSpec: &api.ReleaseBuildConfiguration{ + Tests: tc.tests, + }, + } + o.targets.values = tc.targets + o.adjustLeaseAcquireTimeout() + if o.leaseAcquireTimeout != tc.expectedTimeout { + t.Errorf("expected lease acquire timeout %s, got %s", tc.expectedTimeout, o.leaseAcquireTimeout) + } + }) + } +}