diff --git a/cmd/check-gh-automation/main.go b/cmd/check-gh-automation/main.go index 7ff4c2d430b..3d2c01649af 100644 --- a/cmd/check-gh-automation/main.go +++ b/cmd/check-gh-automation/main.go @@ -369,9 +369,6 @@ func gatherModifiedRepos(releaseRepoPath string, logger *logrus.Entry) []string for _, c := range configs { path := strings.TrimPrefix(c, config.CiopConfigInRepoPath+"/") split := strings.Split(path, "/") - if split[1] == ".config.prowgen" { - continue - } if split[1] == "OWNERS" { continue } diff --git a/cmd/ci-operator-prowgen/main.go b/cmd/ci-operator-prowgen/main.go index eea7e38fe91..5491dc9a965 100644 --- a/cmd/ci-operator-prowgen/main.go +++ b/cmd/ci-operator-prowgen/main.go @@ -102,9 +102,9 @@ func (o *options) process() error { // generateJobsToDir generates prow job configuration into the dir provided by // consuming ci-operator configuration. -func (o *options) generateJobsToDir(subDir string, prowConfig map[string]*config.Prowgen) error { +func (o *options) generateJobsToDir(subDir string) error { generated := map[string]*prowconfig.JobConfig{} - genJobsFunc := generateJobs(o.resolver, prowConfig, generated) + genJobsFunc := generateJobs(o.resolver, generated) if err := o.OperateOnCIOperatorConfigDir(filepath.Join(o.fromDir, subDir), genJobsFunc); err != nil { return fmt.Errorf("failed to generate jobs: %w", err) } @@ -120,37 +120,10 @@ func (o *options) generateJobsToDir(subDir string, prowConfig map[string]*config return writeToDir(o.toDir, generated) } -func generateJobs(resolver registry.Resolver, cache map[string]*config.Prowgen, output map[string]*prowconfig.JobConfig) func(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *config.Info) error { +func generateJobs(resolver registry.Resolver, output map[string]*prowconfig.JobConfig) func(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *config.Info) error { return func(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *config.Info) error { orgRepo := fmt.Sprintf("%s/%s", info.Org, info.Repo) - pInfo := &prowgen.ProwgenInfo{Metadata: info.Metadata, Config: config.Prowgen{Private: false, Expose: false}} - var ok bool - var err error - var orgConfig, repoConfig *config.Prowgen - - if orgConfig, ok = cache[info.Org]; !ok { - if cache[info.Org], err = config.LoadProwgenConfig(info.OrgPath); err != nil { - return err - } - orgConfig = cache[info.Org] - } - - if repoConfig, ok = cache[orgRepo]; !ok { - if cache[orgRepo], err = config.LoadProwgenConfig(info.RepoPath); err != nil { - return err - } - repoConfig = cache[orgRepo] - } - - switch { - case orgConfig != nil: - pInfo.Config = *orgConfig - if repoConfig != nil { - pInfo.Config.MergeDefaults(repoConfig) - } - case repoConfig != nil: - pInfo.Config = *repoConfig - } + pInfo := &prowgen.ProwgenInfo{Metadata: info.Metadata} if resolver != nil { resolved, err := registry.ResolveConfig(resolver, *configSpec) if err != nil { @@ -227,10 +200,9 @@ func main() { args = append(args, "") } logger := logrus.WithFields(logrus.Fields{"target": opt.toDir, "source": opt.fromDir}) - config := map[string]*config.Prowgen{} for _, subDir := range args { logger = logger.WithFields(logrus.Fields{"subdir": subDir}) - if err := opt.generateJobsToDir(subDir, config); err != nil { + if err := opt.generateJobsToDir(subDir); err != nil { logger.WithError(err).Fatal("Failed to generate jobs") } } diff --git a/cmd/ci-operator-prowgen/main_test.go b/cmd/ci-operator-prowgen/main_test.go index 4934c280ffc..b34a295002e 100644 --- a/cmd/ci-operator-prowgen/main_test.go +++ b/cmd/ci-operator-prowgen/main_test.go @@ -7,7 +7,6 @@ import ( "strings" "testing" - "github.com/openshift/ci-tools/pkg/config" "github.com/openshift/ci-tools/pkg/testhelper" ) @@ -293,7 +292,7 @@ tests: } o := options{fromDir: fullConfigPath, toDir: baseProwConfigDir} - if err := o.generateJobsToDir("", map[string]*config.Prowgen{}); err != nil { + if err := o.generateJobsToDir(""); err != nil { t.Fatalf("Unexpected error generating jobs from config: %v", err) } diff --git a/pkg/api/types.go b/pkg/api/types.go index 2084bf4d099..d74405991e7 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -25,6 +25,46 @@ func IsPromotionJob(jobLabels map[string]string) bool { return ok } +// DefaultSlackReportTemplate is the default Go template used for Slack messages +// when no custom template is provided. +const DefaultSlackReportTemplate = `{{if eq .Status.State "success"}} :large_green_circle: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{else}} :red_circle: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> {{end}}` + +// DefaultSlackReporterJobStatesToReport is the default set of job states that +// trigger a Slack report when no custom states are provided. +var DefaultSlackReporterJobStatesToReport = []prowv1.ProwJobState{ + prowv1.FailureState, + prowv1.ErrorState, +} + +// SlackReporter configures Slack reporting for a specific test. +type SlackReporter struct { + // Channel is the Slack channel to report to (e.g., "#my-channel"). + Channel string `json:"channel"` + // JobStatesToReport determines which job states trigger a Slack report. + // If not set, defaults to ["failure", "error"]. + JobStatesToReport []prowv1.ProwJobState `json:"job_states_to_report,omitempty"` + // ReportTemplate is an optional Go template for the Slack message. + // If not set, a default template is used. + ReportTemplate string `json:"report_template,omitempty"` +} + +// ProwgenExtras holds fields that control Prow job generation behavior. +// These fields were previously configured via the .config.prowgen file. +type ProwgenExtras struct { + // Private indicates that generated jobs should be marked as hidden + // from display in deck and that they should mount appropriate git credentials + // to clone the repository under test. + Private *bool `json:"private,omitempty"` + // Expose declares that jobs should not be hidden from view in deck if they + // are private. This field has no effect if private is not set. + Expose *bool `json:"expose,omitempty"` + // DisableRehearsals prevents all tests in this config from being rehearsed. + DisableRehearsals *bool `json:"disable_rehearsals,omitempty"` + // EnableSecretsStoreCSIDriver indicates that jobs should use the new CSI Secrets Store + // mechanism to handle multi-stage credentials secrets. + EnableSecretsStoreCSIDriver *bool `json:"enable_secrets_store_csi_driver,omitempty"` +} + // ReleaseBuildConfiguration describes how release // artifacts are built from a repository of source // code. The configuration is made up of two parts: @@ -40,6 +80,10 @@ type ReleaseBuildConfiguration struct { InputConfiguration `json:",inline"` + // Prowgen holds fields that control Prow job generation behavior. + // These fields were previously configured via the .config.prowgen file. + Prowgen *ProwgenExtras `json:"prowgen,omitempty"` + // BinaryBuildCommands will create a "bin" image based on "src" that // contains the output of this command. This allows reuse of binary artifacts // across other steps. If empty, no "bin" image will be created. @@ -858,6 +902,21 @@ type TestStepConfiguration struct { // RestrictNetworkAccess restricts network access to RedHat intranet. RestrictNetworkAccess *bool `json:"restrict_network_access,omitempty"` + // SlackReporter configures Slack reporting for this specific test. + // When set, the generated Prow job will have Slack reporter configuration + // that reports to the specified channel on the specified job states. + SlackReporter *SlackReporter `json:"slack_reporter,omitempty"` + + // DisableRehearsal prevents this specific test from being picked up for rehearsals. + // Note: this cannot re-enable rehearsals if they are globally disabled via + // prowgen.disable_rehearsals or .config.prowgen's disable_all setting. + DisableRehearsal *bool `json:"disable_rehearsal,omitempty"` + + // MaxConcurrency sets the maximum number of concurrent runs of this job. + // For postsubmit and periodic jobs, this defaults to 1 if not set. + // A zero value explicitly means no concurrency limit. + MaxConcurrency *int `json:"max_concurrency,omitempty"` + // ShardCount describes the number of jobs that should be generated as shards for this test // Each generated job will be a duplication, but contain a suffix and the necessary SHARD_ARGS will be passed to the steps // Only applicable to presubmits and periodics @@ -2619,6 +2678,9 @@ type OperatorStepConfiguration struct { // Substitutions describes the pullspecs in the operator manifests that must be subsituted // with the pull specs of the images in the CI registry Substitutions []PullSpecSubstitution `json:"substitutions,omitempty"` + + // SkipPresubmits prevents generation of operator bundle presubmit jobs. + SkipPresubmits *bool `json:"skip_presubmits,omitempty"` } // IndexUpdate specifies the update mode for an operator being added to an index diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index f0380ba3578..39d6c1f1cc2 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -5,8 +5,8 @@ package api import ( - "github.com/openshift/api/image/v1" - prowjobsv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" + imagev1 "github.com/openshift/api/image/v1" + "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" "sigs.k8s.io/prow/pkg/config" ) @@ -149,7 +149,7 @@ func (in *ClusterClaim) DeepCopyInto(out *ClusterClaim) { } if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout - *out = new(prowjobsv1.Duration) + *out = new(v1.Duration) **out = **in } } @@ -813,7 +813,7 @@ func (in *Integration) DeepCopyInto(out *Integration) { *out = *in if in.ReferencePolicy != nil { in, out := &in.ReferencePolicy, &out.ReferencePolicy - *out = new(v1.TagReferencePolicyType) + *out = new(imagev1.TagReferencePolicyType) **out = **in } } @@ -839,12 +839,12 @@ func (in *LiteralTestStep) DeepCopyInto(out *LiteralTestStep) { in.Resources.DeepCopyInto(&out.Resources) if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout - *out = new(prowjobsv1.Duration) + *out = new(v1.Duration) **out = **in } if in.GracePeriod != nil { in, out := &in.GracePeriod, &out.GracePeriod - *out = new(prowjobsv1.Duration) + *out = new(v1.Duration) **out = **in } if in.Credentials != nil { @@ -1147,7 +1147,7 @@ func (in *MultiStageTestConfigurationLiteral) DeepCopyInto(out *MultiStageTestCo } if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout - *out = new(prowjobsv1.Duration) + *out = new(v1.Duration) **out = **in } } @@ -1194,12 +1194,12 @@ func (in *Observer) DeepCopyInto(out *Observer) { in.Resources.DeepCopyInto(&out.Resources) if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout - *out = new(prowjobsv1.Duration) + *out = new(v1.Duration) **out = **in } if in.GracePeriod != nil { in, out := &in.GracePeriod, &out.GracePeriod - *out = new(prowjobsv1.Duration) + *out = new(v1.Duration) **out = **in } if in.Environment != nil { @@ -1261,6 +1261,11 @@ func (in *OperatorStepConfiguration) DeepCopyInto(out *OperatorStepConfiguration *out = make([]PullSpecSubstitution, len(*in)) copy(*out, *in) } + if in.SkipPresubmits != nil { + in, out := &in.SkipPresubmits, &out.SkipPresubmits + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OperatorStepConfiguration. @@ -1438,6 +1443,41 @@ func (in *PromotionTarget) DeepCopy() *PromotionTarget { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProwgenExtras) DeepCopyInto(out *ProwgenExtras) { + *out = *in + if in.Private != nil { + in, out := &in.Private, &out.Private + *out = new(bool) + **out = **in + } + if in.Expose != nil { + in, out := &in.Expose, &out.Expose + *out = new(bool) + **out = **in + } + if in.DisableRehearsals != nil { + in, out := &in.DisableRehearsals, &out.DisableRehearsals + *out = new(bool) + **out = **in + } + if in.EnableSecretsStoreCSIDriver != nil { + in, out := &in.EnableSecretsStoreCSIDriver, &out.EnableSecretsStoreCSIDriver + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProwgenExtras. +func (in *ProwgenExtras) DeepCopy() *ProwgenExtras { + if in == nil { + return nil + } + out := new(ProwgenExtras) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PullSpecSubstitution) DeepCopyInto(out *PullSpecSubstitution) { *out = *in @@ -1709,6 +1749,11 @@ func (in *ReleaseBuildConfiguration) DeepCopyInto(out *ReleaseBuildConfiguration *out = *in out.Metadata = in.Metadata in.InputConfiguration.DeepCopyInto(&out.InputConfiguration) + if in.Prowgen != nil { + in, out := &in.Prowgen, &out.Prowgen + *out = new(ProwgenExtras) + (*in).DeepCopyInto(*out) + } if in.BinaryBuildCommandsList != nil { in, out := &in.BinaryBuildCommandsList, &out.BinaryBuildCommandsList *out = make([]RefCommands, len(*in)) @@ -1819,7 +1864,7 @@ func (in *ReleaseTagConfiguration) DeepCopyInto(out *ReleaseTagConfiguration) { *out = *in if in.ReferencePolicy != nil { in, out := &in.ReferencePolicy, &out.ReferencePolicy - *out = new(v1.TagReferencePolicyType) + *out = new(imagev1.TagReferencePolicyType) **out = **in } } @@ -1920,6 +1965,26 @@ func (in *Secret) DeepCopy() *Secret { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SlackReporter) DeepCopyInto(out *SlackReporter) { + *out = *in + if in.JobStatesToReport != nil { + in, out := &in.JobStatesToReport, &out.JobStatesToReport + *out = make([]v1.ProwJobState, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SlackReporter. +func (in *SlackReporter) DeepCopy() *SlackReporter { + if in == nil { + return nil + } + out := new(SlackReporter) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SourceStepConfiguration) DeepCopyInto(out *SourceStepConfiguration) { *out = *in @@ -2253,7 +2318,7 @@ func (in *TestStepConfiguration) DeepCopyInto(out *TestStepConfiguration) { } if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout - *out = new(prowjobsv1.Duration) + *out = new(v1.Duration) **out = **in } if in.RestrictNetworkAccess != nil { @@ -2261,6 +2326,21 @@ func (in *TestStepConfiguration) DeepCopyInto(out *TestStepConfiguration) { *out = new(bool) **out = **in } + if in.SlackReporter != nil { + in, out := &in.SlackReporter, &out.SlackReporter + *out = new(SlackReporter) + (*in).DeepCopyInto(*out) + } + if in.DisableRehearsal != nil { + in, out := &in.DisableRehearsal, &out.DisableRehearsal + *out = new(bool) + **out = **in + } + if in.MaxConcurrency != nil { + in, out := &in.MaxConcurrency, &out.MaxConcurrency + *out = new(int) + **out = **in + } if in.ShardCount != nil { in, out := &in.ShardCount, &out.ShardCount *out = new(int) diff --git a/pkg/config/load.go b/pkg/config/load.go index d2107e3bef1..6cd36803098 100644 --- a/pkg/config/load.go +++ b/pkg/config/load.go @@ -6,198 +6,17 @@ import ( "os" "path" "path/filepath" - "regexp" - "slices" "strings" "github.com/ghodss/yaml" "github.com/sirupsen/logrus" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" - prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" - cioperatorapi "github.com/openshift/ci-tools/pkg/api" "github.com/openshift/ci-tools/pkg/util" "github.com/openshift/ci-tools/pkg/util/gzip" "github.com/openshift/ci-tools/pkg/validation" ) -// ProwgenFile is the name of prowgen's configuration file. -var ProwgenFile = ".config.prowgen" - -// Prowgen holds the information of the prowgen's configuration file. -type Prowgen struct { - // Private indicates that generated jobs should be marked as hidden - // from display in deck and that they should mount appropriate git credentials - // to clone the repository under test. - Private bool `json:"private,omitempty"` - // Expose declares that jobs should not be hidden from view in deck if they - // are private. - // This field has no effect if private is not set. - Expose bool `json:"expose,omitempty"` - // Rehearsals declares any disabled rehearsals for jobs - Rehearsals Rehearsals `json:"rehearsals,omitempty"` - // SlackReporterConfigs defines all desired slack reporter info for included jobs - SlackReporterConfigs []SlackReporterConfig `json:"slack_reporter,omitempty"` - // SkipOperatorPresubmits allow users to skip the presubmit generation for that specific variant - SkipOperatorPresubmits []SkipOperatorPresubmits `json:"skip_operator_presubmits,omitempty"` - // EnableSecretsStoreCSIDriver indicates that jobs should use the new CSI Secrets Store - // mechanism to handle multi-stage credentials secrets. - EnableSecretsStoreCSIDriver bool `json:"enable_secrets_store_csi_driver,omitempty"` -} - -// SlackReporterConfig groups test names to a channel to report; mimicking Prow's version, with some unnecessary fields removed -type SlackReporterConfig struct { - Channel string `json:"channel,omitempty"` - JobStatesToReport []prowv1.ProwJobState `json:"job_states_to_report,omitempty"` - ReportTemplate string `json:"report_template,omitempty"` - // JobNames matches against test names (e.g., "unit", "e2e") not full Prow job names. - // This is intentional for backward compatibility - existing configs use test names here. - JobNames []string `json:"job_names,omitempty"` - // JobNamePatterns are regex patterns that match against test names (e.g., ".*-e2e$"). - // Like JobNames, these match test names, not full Prow job names. - JobNamePatterns []string `json:"job_name_patterns,omitempty"` - // ExcludedVariants is a list of variants to skip (e.g., ["hypershift", "okd"]) - ExcludedVariants []string `json:"excluded_variants,omitempty"` - // ExcludedJobPatterns are regex patterns that match against FULL Prow job names - // (e.g., "^pull-.*-skip$" or "^periodic-"). This lets you exclude specific job types - // or use prefixes that only exist in the full job name, not the test name. - ExcludedJobPatterns []string `json:"excluded_job_patterns,omitempty"` -} - -type SkipOperatorPresubmits struct { - Branch string `json:"branch,omitempty"` - Variant string `json:"variant,omitempty"` -} - -// GetSlackReporterConfigForJobName checks against full job names, allowing excluded_job_patterns -// to work with prefixes like "pull-", "periodic-", etc. -func (p *Prowgen) GetSlackReporterConfigForJobName(fullJobName, testName, variant string) *SlackReporterConfig { -nextSlackReporterConfig: - for _, s := range p.SlackReporterConfigs { - if slices.Contains(s.ExcludedVariants, variant) { - continue - } - - // Check if job is excluded by pattern (using full job name) - for _, excludePattern := range s.ExcludedJobPatterns { - if matched, err := regexp.MatchString(excludePattern, fullJobName); err == nil && matched { - continue nextSlackReporterConfig - } - } - - // Check exact job name matches first (against test name for backward compatibility) - if slices.Contains(s.JobNames, testName) { - return &s - } - - // Check regex pattern matches (against test name for backward compatibility) - for _, pattern := range s.JobNamePatterns { - if matched, err := regexp.MatchString(pattern, testName); err == nil && matched { - return &s - } - } - } - return nil -} - -func (p *Prowgen) MergeDefaults(defaults *Prowgen) { - if defaults.Private { - p.Private = true - } - if defaults.Expose { - p.Expose = true - } - if defaults.EnableSecretsStoreCSIDriver { - p.EnableSecretsStoreCSIDriver = true - } - if defaults.Rehearsals.DisableAll { - p.Rehearsals.DisableAll = true - } - p.Rehearsals.DisabledRehearsals = append(p.Rehearsals.DisabledRehearsals, defaults.Rehearsals.DisabledRehearsals...) -} - -func LoadProwgenConfig(folder string) (*Prowgen, error) { - var pConfig *Prowgen - path := filepath.Join(folder, ProwgenFile) - b, err := os.ReadFile(path) - if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("prowgen config found in path %s but couldn't read the file: %w", path, err) - } - - if err == nil { - if err := yaml.Unmarshal(b, &pConfig); err != nil { - return nil, fmt.Errorf("prowgen config found in path %sbut couldn't unmarshal it: %w", path, err) - } - } - - if pConfig != nil { - if err := validateProwgenConfig(pConfig); err != nil { - return nil, fmt.Errorf("prowgen config found in path %s, but it is invalid: %w", path, err) - } - } - - return pConfig, nil -} - -func validateProwgenConfig(pConfig *Prowgen) error { - var errs []error - if len(pConfig.SlackReporterConfigs) > 1 { // There is no reason to validate if we only have one slack_reporter_config - jobsSeen := sets.NewString() - patternsSeen := sets.NewString() - - for _, sc := range pConfig.SlackReporterConfigs { - // Validate exact job names - for _, job := range sc.JobNames { - if jobsSeen.Has(job) { - errs = append(errs, fmt.Errorf("job: %s exists in multiple slack_reporter_configs, it should only be in one", job)) - continue - } - jobsSeen.Insert(job) - } - - // Validate regex patterns - for _, pattern := range sc.JobNamePatterns { - // Check if regex pattern is valid - if _, err := regexp.Compile(pattern); err != nil { - errs = append(errs, fmt.Errorf("invalid regex pattern: %s, error: %w", pattern, err)) - continue - } - - // Check for duplicate patterns - if patternsSeen.Has(pattern) { - errs = append(errs, fmt.Errorf("regex pattern: %s exists in multiple slack_reporter_configs, it should only be in one", pattern)) - continue - } - patternsSeen.Insert(pattern) - } - - // Validate excluded job patterns - for _, pattern := range sc.ExcludedJobPatterns { - // Check if regex pattern is valid - if _, err := regexp.Compile(pattern); err != nil { - errs = append(errs, fmt.Errorf("invalid excluded job pattern: %s, error: %w", pattern, err)) - continue - } - - // Note: We don't check for duplicates in excluded patterns as it's reasonable - // to have the same exclusion in multiple configs - } - } - } - return utilerrors.NewAggregate(errs) -} - -type Rehearsals struct { - // DisableAll indicates that all jobs will not have their "can-be-rehearsed" label set - // and therefore will not be picked up for rehearsals. - DisableAll bool `json:"disable_all,omitempty"` - // DisabledRehearsals contains a list of jobs that will not have their "can-be-rehearsed" label set - // and therefore will not be picked up for rehearsals. - DisabledRehearsals []string `json:"disabled_rehearsals,omitempty"` -} - func readCiOperatorConfig(configFilePath string, info Info) (*cioperatorapi.ReleaseBuildConfiguration, error) { data, err := gzip.ReadFileMaybeGZIP(configFilePath) if err != nil { @@ -497,11 +316,3 @@ func LoadByOrgRepo(path string) (ByOrgRepo, error) { return config, nil } -func (p *Prowgen) SkipPresubmits(branch string, variant string) bool { - for _, skip := range p.SkipOperatorPresubmits { - if skip.Branch == branch && skip.Variant == variant { - return true - } - } - return false -} diff --git a/pkg/config/load_test.go b/pkg/config/load_test.go index 8a44976a386..83d160133aa 100644 --- a/pkg/config/load_test.go +++ b/pkg/config/load_test.go @@ -1,15 +1,11 @@ package config import ( - "errors" "testing" "github.com/google/go-cmp/cmp" - prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" - "github.com/openshift/ci-tools/pkg/api" - "github.com/openshift/ci-tools/pkg/testhelper" ) func TestExtractRepoElementsFromPath(t *testing.T) { @@ -95,268 +91,3 @@ func TestExtractRepoElementsFromPath(t *testing.T) { }) } } - -func TestValidateProwgenConfig(t *testing.T) { - testCases := []struct { - name string - pConfig *Prowgen - expected error - }{ - { - name: "valid", - pConfig: &Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#slack-channel", - JobStatesToReport: []prowv1.ProwJobState{"error"}, - ReportTemplate: "some template", - JobNames: []string{"unit", "e2e"}, - }, - { - Channel: "#slack-channel", - JobStatesToReport: []prowv1.ProwJobState{"success"}, - ReportTemplate: "some other template", - JobNames: []string{"lint"}, - }, - }, - }, - }, - { - name: "invalid, same job in multiple slack reporter configs", - pConfig: &Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#slack-channel", - JobStatesToReport: []prowv1.ProwJobState{"error"}, - ReportTemplate: "some template", - JobNames: []string{"unit", "e2e"}, - }, - { - Channel: "#slack-channel", - JobStatesToReport: []prowv1.ProwJobState{"success"}, - ReportTemplate: "some other template", - JobNames: []string{"unit"}, - }, - }, - }, - expected: errors.New("job: unit exists in multiple slack_reporter_configs, it should only be in one"), - }, - { - name: "invalid regex patterns cause validation errors", - pConfig: &Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#slack-channel", - JobNamePatterns: []string{"[invalid"}, - }, - { - Channel: "#other-channel", - JobNamePatterns: []string{"^valid.*"}, - }, - }, - }, - expected: errors.New("invalid regex pattern: [invalid, error: error parsing regexp: missing closing ]: `[invalid`"), - }, - { - name: "duplicate regex patterns cause validation errors", - pConfig: &Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#slack-channel", - JobNamePatterns: []string{"^unit.*"}, - }, - { - Channel: "#other-channel", - JobNamePatterns: []string{"^unit.*"}, - }, - }, - }, - expected: errors.New("regex pattern: ^unit.* exists in multiple slack_reporter_configs, it should only be in one"), - }, - { - name: "valid regex patterns pass validation", - pConfig: &Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#slack-channel", - JobNamePatterns: []string{"^unit.*", "^e2e.*"}, - }, - { - Channel: "#other-channel", - JobNamePatterns: []string{"^integration.*"}, - }, - }, - }, - }, - { - name: "invalid excluded job patterns cause validation errors", - pConfig: &Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#slack-channel", - ExcludedJobPatterns: []string{"[invalid"}, - }, - { - Channel: "#other-channel", - ExcludedJobPatterns: []string{"^valid.*"}, - }, - }, - }, - expected: errors.New("invalid excluded job pattern: [invalid, error: error parsing regexp: missing closing ]: `[invalid`"), - }, - { - name: "valid excluded job patterns pass validation", - pConfig: &Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#slack-channel", - JobNamePatterns: []string{".*"}, - ExcludedJobPatterns: []string{".*-skip$", "^nightly-.*"}, - }, - { - Channel: "#other-channel", - JobNames: []string{"unit", "e2e"}, - ExcludedJobPatterns: []string{".*-flaky$"}, - }, - }, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := validateProwgenConfig(tc.pConfig) - if diff := cmp.Diff(result, tc.expected, testhelper.EquateErrorMessage); diff != "" { - t.Fatalf("result doesn't match expected, diff: %v", diff) - } - }) - } -} - -func TestValidateProwgenSkipOperatorPresubmits(t *testing.T) { - testCases := []struct { - name string - pConfig *Prowgen - branch string - variant string - expected bool - }{ - { - name: "skipping operator presubmits, exactly match", - pConfig: &Prowgen{ - SkipOperatorPresubmits: []SkipOperatorPresubmits{ - { - Branch: "main", - Variant: "4.18", - }, - { - Branch: "dev", - Variant: "4.19", - }, - }, - }, - branch: "main", - variant: "4.18", - expected: true, - }, - { - name: "generating operator presubmits, mismatch branches", - pConfig: &Prowgen{ - SkipOperatorPresubmits: []SkipOperatorPresubmits{ - { - Branch: "dev", - Variant: "4.18", - }, - }, - }, - branch: "main", - variant: "4.18", - expected: false, - }, - { - name: "skipping operator presubmits, empty values", - pConfig: &Prowgen{}, - branch: "main", - variant: "4.19", - expected: false, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.pConfig.SkipOperatorPresubmits != nil { - skip := tc.pConfig.SkipPresubmits(tc.branch, tc.variant) - if skip != tc.expected { - t.Fatalf("result doesn't match, expected %v, received %v", tc.expected, skip) - } - } - }) - } -} - -func TestProwgen_MergeDefaults_SlackReporterConfigs(t *testing.T) { - testCases := []struct { - name string - base Prowgen - defaults Prowgen - expected []SlackReporterConfig - }{ - { - name: "slack reporter configs are never merged from defaults", - base: Prowgen{}, - defaults: Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#test-channel", - JobStatesToReport: []prowv1.ProwJobState{"failure"}, - JobNamePatterns: []string{".*"}, - ExcludedJobPatterns: []string{".*-skip$"}, - }, - }, - }, - expected: nil, - }, - { - name: "existing slack reporter configs are preserved unchanged", - base: Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#existing-channel", - JobStatesToReport: []prowv1.ProwJobState{"error"}, - JobNames: []string{"unit"}, - }, - }, - }, - defaults: Prowgen{ - SlackReporterConfigs: []SlackReporterConfig{ - { - Channel: "#default-channel", - JobStatesToReport: []prowv1.ProwJobState{"failure"}, - JobNamePatterns: []string{".*"}, - ExcludedJobPatterns: []string{".*-skip$"}, - }, - }, - }, - expected: []SlackReporterConfig{ - { - Channel: "#existing-channel", - JobStatesToReport: []prowv1.ProwJobState{"error"}, - JobNames: []string{"unit"}, - }, - }, - }, - { - name: "empty base with empty defaults stays empty", - base: Prowgen{}, - defaults: Prowgen{}, - expected: nil, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - tc.base.MergeDefaults(&tc.defaults) - if diff := cmp.Diff(tc.base.SlackReporterConfigs, tc.expected); diff != "" { - t.Fatalf("SlackReporterConfigs don't match expected, diff: %v", diff) - } - }) - } -} diff --git a/pkg/image-graph-generator/operator.go b/pkg/image-graph-generator/operator.go index def333d7a98..40c6e8f7080 100644 --- a/pkg/image-graph-generator/operator.go +++ b/pkg/image-graph-generator/operator.go @@ -79,25 +79,6 @@ func (o *Operator) callback(c *api.ReleaseBuildConfiguration, i *config.Info) er return nil } - configProwgen := &config.Prowgen{} - orgProwgenConfig, err := config.LoadProwgenConfig(i.OrgPath) - if err != nil { - return err - } - - repoProwgenConfig, err := config.LoadProwgenConfig(i.RepoPath) - if err != nil { - return err - } - - if repoProwgenConfig != nil { - configProwgen = repoProwgenConfig - } - - if orgProwgenConfig != nil { - configProwgen.MergeDefaults(orgProwgenConfig) - } - var errs []error for _, target := range api.PromotionTargets(c.PromotionConfiguration) { excludedImages := sets.New[string](target.ExcludedImages...) diff --git a/pkg/prowgen/jobbase.go b/pkg/prowgen/jobbase.go index a1127c6ba17..2ce160a2e54 100644 --- a/pkg/prowgen/jobbase.go +++ b/pkg/prowgen/jobbase.go @@ -3,7 +3,7 @@ package prowgen import ( "time" - utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" prowconfig "sigs.k8s.io/prow/pkg/config" @@ -20,6 +20,39 @@ type prowJobBaseBuilder struct { testName string } +// isPrivate returns true if the repo is private, checking ci-operator config +// and the org name (openshift-priv is always private). +func isPrivate(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *ProwgenInfo) bool { + if configSpec.Prowgen != nil && configSpec.Prowgen.Private != nil { + return *configSpec.Prowgen.Private + } + return info.Org == "openshift-priv" +} + +// isExposed returns true if jobs should be visible in Deck despite being private. +func isExposed(configSpec *cioperatorapi.ReleaseBuildConfiguration) bool { + if configSpec.Prowgen != nil && configSpec.Prowgen.Expose != nil { + return *configSpec.Prowgen.Expose + } + return false +} + +// isSecretsStoreCSIDriverEnabled returns true if CSI Secrets Store should be used. +func isSecretsStoreCSIDriverEnabled(configSpec *cioperatorapi.ReleaseBuildConfiguration) bool { + if configSpec.Prowgen != nil && configSpec.Prowgen.EnableSecretsStoreCSIDriver != nil { + return *configSpec.Prowgen.EnableSecretsStoreCSIDriver + } + return false +} + +// areRehearsalsDisabled returns true if all rehearsals are disabled. +func areRehearsalsDisabled(configSpec *cioperatorapi.ReleaseBuildConfiguration) bool { + if configSpec.Prowgen != nil && configSpec.Prowgen.DisableRehearsals != nil { + return *configSpec.Prowgen.DisableRehearsals + } + return false +} + // If any included buildRoot uses from_repository we must not skip cloning func skipCloning(configSpec *cioperatorapi.ReleaseBuildConfiguration) bool { buildRoots := configSpec.BuildRootImages @@ -63,14 +96,17 @@ func NewProwJobBaseBuilder(configSpec *cioperatorapi.ReleaseBuildConfiguration, Agent: string(prowv1.KubernetesAgent), Labels: map[string]string{}, UtilityConfig: prowconfig.UtilityConfig{ - Decorate: utilpointer.Bool(true), + Decorate: ptr.To(true), }, }, } + private := isPrivate(configSpec, info) + exposed := isExposed(configSpec) + if skipCloning(configSpec) { - b.base.UtilityConfig.DecorationConfig = &prowv1.DecorationConfig{SkipCloning: utilpointer.Bool(true)} - } else if info.Config.Private { + b.base.UtilityConfig.DecorationConfig = &prowv1.DecorationConfig{SkipCloning: ptr.To(true)} + } else if private { b.base.UtilityConfig.DecorationConfig = &prowv1.DecorationConfig{OauthTokenSecret: &prowv1.OauthTokenSecret{Key: cioperatorapi.OauthTokenSecretKey, Name: cioperatorapi.OauthTokenSecretName}} } @@ -89,7 +125,7 @@ func NewProwJobBaseBuilder(configSpec *cioperatorapi.ReleaseBuildConfiguration, } b.PodSpec.Add(Variant(info.Variant)) - if info.Config.Private { + if private { // We can reuse Prow's volume with the token if ProwJob itself is cloning the code b.PodSpec.Add(GitHubToken(!skipCloning(configSpec))) } @@ -98,7 +134,7 @@ func NewProwJobBaseBuilder(configSpec *cioperatorapi.ReleaseBuildConfiguration, b.base.UtilityConfig.PathAlias = *configSpec.CanonicalGoRepository } - if info.Config.Private && !info.Config.Expose { + if private && !exposed { b.base.Hidden = true } @@ -147,7 +183,7 @@ func NewProwJobBaseBuilderForTest(configSpec *cioperatorapi.ReleaseBuildConfigur if configSpec.Releases != nil { p.PodSpec.Add(CIPullSecret()) } - if info.Config.EnableSecretsStoreCSIDriver { + if isSecretsStoreCSIDriverEnabled(configSpec) { p.PodSpec.Add( GSMConfig(), ) @@ -161,7 +197,7 @@ func NewProwJobBaseBuilderForTest(configSpec *cioperatorapi.ReleaseBuildConfigur if configSpec.Releases != nil { p.PodSpec.Add(CIPullSecret()) } - if info.Config.EnableSecretsStoreCSIDriver { + if isSecretsStoreCSIDriverEnabled(configSpec) { p.PodSpec.Add( GSMConfig(), ) diff --git a/pkg/prowgen/jobbase_test.go b/pkg/prowgen/jobbase_test.go index b016b278753..c2849069adc 100644 --- a/pkg/prowgen/jobbase_test.go +++ b/pkg/prowgen/jobbase_test.go @@ -5,11 +5,9 @@ import ( "time" "k8s.io/utils/pointer" - prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" v1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" ciop "github.com/openshift/ci-tools/pkg/api" - "github.com/openshift/ci-tools/pkg/config" "github.com/openshift/ci-tools/pkg/testhelper" ) @@ -29,6 +27,7 @@ func TestProwJobBaseBuilder(t *testing.T) { images ciop.ImageConfiguration binCommand string testBinCommand string + prowgen *ciop.ProwgenExtras podSpecBuilder CiOperatorPodSpecGenerator info *ProwgenInfo @@ -153,8 +152,8 @@ func TestProwJobBaseBuilder(t *testing.T) { name: "private job without cloning, including podspec", info: &ProwgenInfo{ Metadata: ciop.Metadata{Org: "vorg", Repo: "vrepo", Branch: "vbranch"}, - Config: config.Prowgen{Private: true}, }, + prowgen: &ciop.ProwgenExtras{Private: pointer.Bool(true)}, prefix: "default", podSpecBuilder: NewCiOperatorPodSpecGenerator(), }, @@ -162,9 +161,9 @@ func TestProwJobBaseBuilder(t *testing.T) { name: "private job with cloning, including podspec", info: &ProwgenInfo{ Metadata: ciop.Metadata{Org: "vorg", Repo: "vrepo", Branch: "vbranch"}, - Config: config.Prowgen{Private: true}, }, - prefix: "default", + prowgen: &ciop.ProwgenExtras{Private: pointer.Bool(true)}, + prefix: "default", inputs: ciop.InputConfiguration{ BuildRootImage: &ciop.BuildRootImageConfiguration{FromRepository: true}, }, @@ -182,6 +181,7 @@ func TestProwJobBaseBuilder(t *testing.T) { BinaryBuildCommands: tc.binCommand, TestBinaryBuildCommands: tc.testBinCommand, Metadata: tc.info.Metadata, + Prowgen: tc.prowgen, } b := NewProwJobBaseBuilder(ciopconfig, tc.info, tc.podSpecBuilder).Build(tc.prefix) testhelper.CompareWithFixture(t, b) @@ -194,6 +194,7 @@ func TestGenerateJobBase(t *testing.T) { testName string name string info *ProwgenInfo + prowgen *ciop.ProwgenExtras canonicalGoRepository string rehearsable bool }{ @@ -224,30 +225,30 @@ func TestGenerateJobBase(t *testing.T) { name: "test", info: &ProwgenInfo{ Metadata: ciop.Metadata{Org: "org", Repo: "repo", Branch: "branch"}, - Config: config.Prowgen{Private: true}, }, + prowgen: &ciop.ProwgenExtras{Private: pointer.Bool(true)}, }, { testName: "expose job for private repos with public results", name: "test", info: &ProwgenInfo{ Metadata: ciop.Metadata{Org: "org", Repo: "repo", Branch: "branch"}, - Config: config.Prowgen{Private: true, Expose: true}, }, + prowgen: &ciop.ProwgenExtras{Private: pointer.Bool(true), Expose: pointer.Bool(true)}, }, { testName: "expose option set but not private", name: "test", info: &ProwgenInfo{ Metadata: ciop.Metadata{Org: "org", Repo: "repo", Branch: "branch"}, - Config: config.Prowgen{Private: false, Expose: true}, }, + prowgen: &ciop.ProwgenExtras{Expose: pointer.Bool(true)}, }, } for _, testCase := range testCases { t.Run(testCase.testName, func(t *testing.T) { - jobBaseGen := NewProwJobBaseBuilder(&ciop.ReleaseBuildConfiguration{CanonicalGoRepository: &testCase.canonicalGoRepository}, testCase.info, newFakePodSpecBuilder()).Rehearsable(testCase.rehearsable).TestName(testCase.name) + jobBaseGen := NewProwJobBaseBuilder(&ciop.ReleaseBuildConfiguration{CanonicalGoRepository: &testCase.canonicalGoRepository, Prowgen: testCase.prowgen}, testCase.info, newFakePodSpecBuilder()).Rehearsable(testCase.rehearsable).TestName(testCase.name) testhelper.CompareWithFixture(t, jobBaseGen.Build("pull")) }) } @@ -337,10 +338,10 @@ func TestNewProwJobBaseBuilderForTest(t *testing.T) { Workflow: pointer.StringPtr("workflow"), }, }, - info: &ProwgenInfo{ - Metadata: ciop.Metadata{Org: "o", Repo: "r", Branch: "b"}, - Config: config.Prowgen{EnableSecretsStoreCSIDriver: true}, + cfg: &ciop.ReleaseBuildConfiguration{ + Prowgen: &ciop.ProwgenExtras{EnableSecretsStoreCSIDriver: pointer.Bool(true)}, }, + info: defaultInfo, }, { name: "simple test with CSI enabled", @@ -349,10 +350,10 @@ func TestNewProwJobBaseBuilderForTest(t *testing.T) { Commands: "make", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "src"}, }, - info: &ProwgenInfo{ - Metadata: ciop.Metadata{Org: "o", Repo: "r", Branch: "b"}, - Config: config.Prowgen{EnableSecretsStoreCSIDriver: true}, + cfg: &ciop.ReleaseBuildConfiguration{ + Prowgen: &ciop.ProwgenExtras{EnableSecretsStoreCSIDriver: pointer.Bool(true)}, }, + info: defaultInfo, }, { name: "multi-stage test with claim", @@ -418,49 +419,9 @@ func TestNewProwJobBaseBuilderForTest(t *testing.T) { }, info: defaultInfo, }, - { - name: "simple with slack reporter config", - test: ciop.TestStepConfiguration{ - As: "unit", - Commands: "make unit", - ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "src"}, - }, - info: &ProwgenInfo{ - Metadata: ciop.Metadata{Org: "o", Repo: "r", Branch: "b"}, - Config: config.Prowgen{ - SlackReporterConfigs: []config.SlackReporterConfig{ - { - Channel: "some-channel", - JobStatesToReport: []prowv1.ProwJobState{"error"}, - ReportTemplate: "some template", - JobNames: []string{"unit", "e2e"}, - }, - }, - }, - }, - }, - { - name: "job excluded by patterns should not have slack reporter config", - test: ciop.TestStepConfiguration{ - As: "unit-skip", - Commands: "make unit", - ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "src"}, - }, - info: &ProwgenInfo{ - Metadata: ciop.Metadata{Org: "o", Repo: "r", Branch: "b"}, - Config: config.Prowgen{ - SlackReporterConfigs: []config.SlackReporterConfig{ - { - Channel: "some-channel", - JobStatesToReport: []prowv1.ProwJobState{"error"}, - ReportTemplate: "some template", - JobNames: []string{"unit-skip", "e2e"}, - ExcludedJobPatterns: []string{".*-skip$"}, - }, - }, - }, - }, - }, + // NOTE: "simple with slack reporter config" and "job excluded by patterns" test cases + // removed because they relied on .config.prowgen slack reporter matching which has been removed. + // Slack reporter is now configured per-test in ci-operator config. } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/prowgen/prowgen.go b/pkg/prowgen/prowgen.go index c87a11e5e4b..f964f33c270 100644 --- a/pkg/prowgen/prowgen.go +++ b/pkg/prowgen/prowgen.go @@ -9,7 +9,6 @@ import ( prowconfig "sigs.k8s.io/prow/pkg/config" cioperatorapi "github.com/openshift/ci-tools/pkg/api" - "github.com/openshift/ci-tools/pkg/config" jc "github.com/openshift/ci-tools/pkg/jobconfig" ) @@ -21,7 +20,15 @@ const ( type ProwgenInfo struct { cioperatorapi.Metadata - Config config.Prowgen +} + +// maxConcurrency returns the user-specified max concurrency if set, +// otherwise returns the provided default. +func maxConcurrency(configured *int, defaultVal int) int { + if configured != nil { + return *configured + } + return defaultVal } // GenerateJobs @@ -42,8 +49,7 @@ func GenerateJobs(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *Pro presubmits := map[string][]prowconfig.Presubmit{} postsubmits := map[string][]prowconfig.Postsubmit{} var periodics []prowconfig.Periodic - rehearsals := info.Config.Rehearsals - disabledRehearsals := sets.New[string](rehearsals.DisabledRehearsals...) + disableAllRehearsals := areRehearsalsDisabled(configSpec) for _, element := range configSpec.Tests { shardCount := 1 @@ -65,7 +71,7 @@ func GenerateJobs(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *Pro g.WithLabel(fmt.Sprintf("capability/%s", element.NodeArchitecture), string(element.NodeArchitecture)) } - disableRehearsal := rehearsals.DisableAll || disabledRehearsals.Has(element.As) + disableRehearsal := disableAllRehearsals || (element.DisableRehearsal != nil && *element.DisableRehearsal) if element.IsPeriodic() { cron := "" @@ -93,6 +99,8 @@ func GenerateJobs(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *Pro options.ReleaseController = element.ReleaseController options.DisableRehearsal = disableRehearsal options.Retry = element.Retry + options.SlackReporter = element.SlackReporter + options.MaxConcurrency = element.MaxConcurrency }) periodics = append(periodics, *periodic) if element.Presubmit { @@ -103,8 +111,9 @@ func GenerateJobs(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *Pro options.runIfChanged = element.RunIfChanged options.Capabilities = element.Capabilities options.skipIfOnlyChanged = element.SkipIfOnlyChanged + options.slackReporter = element.SlackReporter }) - postsubmit.MaxConcurrency = 1 + postsubmit.MaxConcurrency = maxConcurrency(element.MaxConcurrency, 1) postsubmits[orgrepo] = append(postsubmits[orgrepo], *postsubmit) } else { handlePresubmit(g, element, info, name, disableRehearsal, configSpec.Resources.RequirementsForStep(element.As).Requests, presubmits, orgrepo) @@ -178,7 +187,8 @@ func GenerateJobs(configSpec *cioperatorapi.ReleaseBuildConfiguration, info *Pro } } - if configSpec.Operator != nil && !info.Config.SkipPresubmits(configSpec.Metadata.Branch, configSpec.Metadata.Variant) { + if configSpec.Operator != nil && + !(configSpec.Operator.SkipPresubmits != nil && *configSpec.Operator.SkipPresubmits) { containsUnnamedBundle := false for _, bundle := range configSpec.Operator.Bundles { if bundle.As == "" { @@ -225,6 +235,8 @@ func handlePresubmit(g *prowJobBaseBuilder, element cioperatorapi.TestStepConfig options.defaultDisable = element.AlwaysRun != nil && !*element.AlwaysRun options.optional = element.Optional options.disableRehearsal = disableRehearsal + options.slackReporter = element.SlackReporter + options.maxConcurrency = element.MaxConcurrency }) v, requestingKVM := requests[cioperatorapi.KVMDeviceLabel] if requestingKVM { @@ -242,6 +254,8 @@ type generatePresubmitOptions struct { defaultDisable bool optional bool disableRehearsal bool + slackReporter *cioperatorapi.SlackReporter + maxConcurrency *int } func (opts *generatePresubmitOptions) shouldAlwaysRun() bool { @@ -250,17 +264,29 @@ func (opts *generatePresubmitOptions) shouldAlwaysRun() bool { type generatePresubmitOption func(options *generatePresubmitOptions) -// addSlackReporterConfig sets the Slack reporter configuration on a job base if one is found -func addSlackReporterConfig(base *prowconfig.JobBase, jobName, testName string, info *ProwgenInfo) { - if slackReporter := info.Config.GetSlackReporterConfigForJobName(jobName, testName, info.Metadata.Variant); slackReporter != nil { - if base.ReporterConfig == nil { - base.ReporterConfig = &prowv1.ReporterConfig{} - } - base.ReporterConfig.Slack = &prowv1.SlackReporterConfig{ - Channel: slackReporter.Channel, - JobStatesToReport: slackReporter.JobStatesToReport, - ReportTemplate: slackReporter.ReportTemplate, - } +// addSlackReporterConfig sets the Slack reporter configuration on a job base +// from the ci-operator config's per-test SlackReporter field. +func addSlackReporterConfig(base *prowconfig.JobBase, testSlackReporter *cioperatorapi.SlackReporter) { + if testSlackReporter == nil { + return + } + + jobStates := testSlackReporter.JobStatesToReport + if len(jobStates) == 0 { + jobStates = cioperatorapi.DefaultSlackReporterJobStatesToReport + } + reportTemplate := testSlackReporter.ReportTemplate + if reportTemplate == "" { + reportTemplate = cioperatorapi.DefaultSlackReportTemplate + } + + if base.ReporterConfig == nil { + base.ReporterConfig = &prowv1.ReporterConfig{} + } + base.ReporterConfig.Slack = &prowv1.SlackReporterConfig{ + Channel: testSlackReporter.Channel, + JobStatesToReport: jobStates, + ReportTemplate: reportTemplate, } } @@ -273,9 +299,7 @@ func generatePresubmitForTest(jobBaseBuilder *prowJobBaseBuilder, name string, i shortName := info.TestName(name) base := jobBaseBuilder.Rehearsable(!opts.disableRehearsal).Build(jc.PresubmitPrefix) - // Set slack reporter config using full job name for proper excluded_job_patterns matching - fullJobName := info.JobName(jc.PresubmitPrefix, name) - addSlackReporterConfig(&base, fullJobName, name, info) + addSlackReporterConfig(&base, opts.slackReporter) pipelineOpt := false if opts.pipelineRunIfChanged != "" { @@ -311,6 +335,9 @@ func generatePresubmitForTest(jobBaseBuilder *prowJobBaseBuilder, name string, i }, Optional: opts.optional, } + if opts.maxConcurrency != nil { + pj.MaxConcurrency = *opts.maxConcurrency + } injectCapabilities(pj.Labels, opts.Capabilities) return pj } @@ -319,6 +346,7 @@ type generatePostsubmitOptions struct { runIfChanged string Capabilities []string skipIfOnlyChanged string + slackReporter *cioperatorapi.SlackReporter } type generatePostsubmitOption func(options *generatePostsubmitOptions) @@ -331,10 +359,7 @@ func generatePostsubmitForTest(jobBaseBuilder *prowJobBaseBuilder, info *Prowgen base := jobBaseBuilder.Build(jc.PostsubmitPrefix) - // Set slack reporter config using full job name for proper excluded_job_patterns matching - testName := jobBaseBuilder.testName - fullJobName := info.JobName(jc.PostsubmitPrefix, testName) - addSlackReporterConfig(&base, fullJobName, testName, info) + addSlackReporterConfig(&base, opts.slackReporter) alwaysRun := opts.runIfChanged == "" && opts.skipIfOnlyChanged == "" pj := &prowconfig.Postsubmit{ @@ -371,6 +396,8 @@ type GeneratePeriodicOptions struct { PathAlias *string DisableRehearsal bool Retry *prowconfig.Retry + SlackReporter *cioperatorapi.SlackReporter + MaxConcurrency *int } type GeneratePeriodicOption func(options *GeneratePeriodicOptions) @@ -390,10 +417,7 @@ func GeneratePeriodicForTest(jobBaseBuilder *prowJobBaseBuilder, info *ProwgenIn // We are resetting PathAlias because it will be set on the `ExtraRefs` item base := jobBaseBuilder.Rehearsable(!opts.DisableRehearsal).PathAlias("").Build(jc.PeriodicPrefix) - // Set slack reporter config using full job name for proper excluded_job_patterns matching - testName := jobBaseBuilder.testName - fullJobName := info.JobName(jc.PeriodicPrefix, testName) - addSlackReporterConfig(&base, fullJobName, testName, info) + addSlackReporterConfig(&base, opts.SlackReporter) cron := opts.Cron if cron == "@daily" { @@ -424,6 +448,9 @@ func GeneratePeriodicForTest(jobBaseBuilder *prowJobBaseBuilder, info *ProwgenIn MinimumInterval: opts.MinimumInterval, Retry: opts.Retry, } + if opts.MaxConcurrency != nil { + pj.MaxConcurrency = *opts.MaxConcurrency + } injectCapabilities(pj.Labels, opts.Capabilities) return pj } diff --git a/pkg/prowgen/prowgen_test.go b/pkg/prowgen/prowgen_test.go index 0ae59735c5e..f14dd1a984b 100644 --- a/pkg/prowgen/prowgen_test.go +++ b/pkg/prowgen/prowgen_test.go @@ -6,13 +6,11 @@ import ( "testing" corev1 "k8s.io/api/core/v1" - utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" prowv1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" prowconfig "sigs.k8s.io/prow/pkg/config" - "github.com/openshift/ci-tools/pkg/api" ciop "github.com/openshift/ci-tools/pkg/api" - "github.com/openshift/ci-tools/pkg/config" "github.com/openshift/ci-tools/pkg/testhelper" ) @@ -456,7 +454,7 @@ func TestGenerateJobs(t *testing.T) { config: &ciop.ReleaseBuildConfiguration{ Tests: []ciop.TestStepConfiguration{}, Images: ciop.ImageConfiguration{Items: []ciop.ProjectDirectoryImageBuildStepConfiguration{{}}}, - PromotionConfiguration: &ciop.PromotionConfiguration{Targets: []api.PromotionTarget{{Namespace: "ci"}}}, + PromotionConfiguration: &ciop.PromotionConfiguration{Targets: []ciop.PromotionTarget{{Namespace: "ci"}}}, }, repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ Org: "organization", @@ -473,7 +471,7 @@ func TestGenerateJobs(t *testing.T) { {To: "out-2", From: "base"}, }}, PromotionConfiguration: &ciop.PromotionConfiguration{ - Targets: []api.PromotionTarget{{ + Targets: []ciop.PromotionTarget{{ Namespace: "ci", AdditionalImages: map[string]string{ "out": "out-1", @@ -617,7 +615,7 @@ func TestGenerateJobs(t *testing.T) { id: "cluster label for periodic", config: &ciop.ReleaseBuildConfiguration{ Tests: []ciop.TestStepConfiguration{ - {As: "unit", Cron: utilpointer.String(cron), Cluster: "build01", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "unit", Cron: ptr.To(cron), Cluster: "build01", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, }, }, repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ @@ -630,7 +628,7 @@ func TestGenerateJobs(t *testing.T) { id: "periodic with presubmit", config: &ciop.ReleaseBuildConfiguration{ Tests: []ciop.TestStepConfiguration{ - {As: "unit", Cron: utilpointer.String(cron), Presubmit: true, Cluster: "build01", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "unit", Cron: ptr.To(cron), Presubmit: true, Cluster: "build01", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, }, }, repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ @@ -656,35 +654,32 @@ func TestGenerateJobs(t *testing.T) { id: "disabled rehearsals at job level", config: &ciop.ReleaseBuildConfiguration{ Tests: []ciop.TestStepConfiguration{ - {As: "unit", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "unit", DisableRehearsal: ptr.To(true), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, {As: "lint", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, - {As: "periodic-unit", Cron: utilpointer.String(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, - {As: "periodic-lint", Cron: utilpointer.String(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "periodic-unit", DisableRehearsal: ptr.To(true), Cron: ptr.To(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "periodic-lint", Cron: ptr.To(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, }, }, - repoInfo: &ProwgenInfo{ - Config: config.Prowgen{Rehearsals: config.Rehearsals{DisabledRehearsals: []string{"unit", "periodic-unit"}}}, - Metadata: ciop.Metadata{ - Org: "organization", - Repo: "repository", - Branch: "branch", - }}, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, }, { id: "disabled rehearsals at repo level", config: &ciop.ReleaseBuildConfiguration{ + Prowgen: &ciop.ProwgenExtras{DisableRehearsals: ptr.To(true)}, Tests: []ciop.TestStepConfiguration{ {As: "unit", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, - {As: "periodic-unit", Cron: utilpointer.String(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "periodic-unit", Cron: ptr.To(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, }, }, - repoInfo: &ProwgenInfo{ - Config: config.Prowgen{Rehearsals: config.Rehearsals{DisableAll: true}}, - Metadata: ciop.Metadata{ - Org: "organization", - Repo: "repository", - Branch: "branch", - }}, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, }, { id: "multiarch postsubmit images", @@ -729,35 +724,13 @@ func TestGenerateJobs(t *testing.T) { }, }, }, - { - id: "images job is configured for slack reporting", - config: &ciop.ReleaseBuildConfiguration{ - Images: ciop.ImageConfiguration{Items: []ciop.ProjectDirectoryImageBuildStepConfiguration{{}}}, - PromotionConfiguration: &ciop.PromotionConfiguration{}, - }, - repoInfo: &ProwgenInfo{ - Metadata: ciop.Metadata{ - Org: "organization", - Repo: "repository", - Branch: "branch", - }, - Config: config.Prowgen{ - SlackReporterConfigs: []config.SlackReporterConfig{ - { - Channel: "some-channel", - JobStatesToReport: []prowv1.ProwJobState{"error"}, - ReportTemplate: "some template", - JobNames: []string{"images", "e2e"}, - }, - }, - }, - }, - }, + // NOTE: "images job is configured for slack reporting" test case removed + // because it relied on .config.prowgen slack reporter matching which has been removed. { id: "periodic with capabilities", config: &ciop.ReleaseBuildConfiguration{ Tests: []ciop.TestStepConfiguration{ - {As: "unit", Capabilities: []string{"intranet"}, Cron: utilpointer.String(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "unit", Capabilities: []string{"intranet"}, Cron: ptr.To(cron), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, }, }, repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ @@ -770,7 +743,7 @@ func TestGenerateJobs(t *testing.T) { id: "periodic/presubmit with capabilities", config: &ciop.ReleaseBuildConfiguration{ Tests: []ciop.TestStepConfiguration{ - {As: "unit", Capabilities: []string{"intranet", "arm64", "rce", "sshd-bastion"}, Cron: utilpointer.String(cron), Presubmit: true, ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, // rce - release-controller-eligible, sshd-bastion - for multiarch P/Z libvirt jobs + {As: "unit", Capabilities: []string{"intranet", "arm64", "rce", "sshd-bastion"}, Cron: ptr.To(cron), Presubmit: true, ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, // rce - release-controller-eligible, sshd-bastion - for multiarch P/Z libvirt jobs }, }, repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ @@ -837,6 +810,183 @@ func TestGenerateJobs(t *testing.T) { Branch: "branch", }}, }, + { + id: "slack reporter from ci-operator config with defaults", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Tests: []ciop.TestStepConfiguration{ + { + As: "e2e", + SlackReporter: &ciop.SlackReporter{ + Channel: "#test-channel", + }, + ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}, + }, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "slack reporter from ci-operator config with explicit values", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Tests: []ciop.TestStepConfiguration{ + { + As: "e2e", + SlackReporter: &ciop.SlackReporter{ + Channel: "#custom-channel", + JobStatesToReport: []prowv1.ProwJobState{"success", "failure"}, + ReportTemplate: "custom template", + }, + ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}, + }, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "slack reporter from ci-operator config per-test", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Tests: []ciop.TestStepConfiguration{ + { + As: "e2e", + SlackReporter: &ciop.SlackReporter{ + Channel: "#from-ci-operator", + }, + ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}, + }, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "disable rehearsal from ci-operator config per-test", + config: &ciop.ReleaseBuildConfiguration{ + Tests: []ciop.TestStepConfiguration{ + {As: "unit", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "e2e", DisableRehearsal: ptr.To(true), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "disable all rehearsals from ci-operator prowgen config", + config: &ciop.ReleaseBuildConfiguration{ + Prowgen: &ciop.ProwgenExtras{DisableRehearsals: ptr.To(true)}, + Tests: []ciop.TestStepConfiguration{ + {As: "unit", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + {As: "e2e", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "skip operator presubmits from ci-operator config", + config: &ciop.ReleaseBuildConfiguration{ + Operator: &ciop.OperatorStepConfiguration{ + SkipPresubmits: ptr.To(true), + Bundles: []ciop.Bundle{{As: "my-bundle"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "private from ci-operator prowgen config", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Prowgen: &ciop.ProwgenExtras{Private: ptr.To(true)}, + Tests: []ciop.TestStepConfiguration{ + {As: "unit", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "private with expose from ci-operator prowgen config", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Prowgen: &ciop.ProwgenExtras{Private: ptr.To(true), Expose: ptr.To(true)}, + Tests: []ciop.TestStepConfiguration{ + {As: "unit", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "openshift-priv org defaults to private", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Tests: []ciop.TestStepConfiguration{ + {As: "unit", ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "openshift-priv", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "postsubmit with custom max_concurrency", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Tests: []ciop.TestStepConfiguration{ + {As: "publish", Postsubmit: true, MaxConcurrency: ptr.To(4), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, + { + id: "periodic with custom max_concurrency", + keep: true, + config: &ciop.ReleaseBuildConfiguration{ + Tests: []ciop.TestStepConfiguration{ + {As: "nightly", Cron: ptr.To(cron), MaxConcurrency: ptr.To(2), ContainerTestConfiguration: &ciop.ContainerTestConfiguration{From: "bin"}}, + }, + }, + repoInfo: &ProwgenInfo{Metadata: ciop.Metadata{ + Org: "organization", + Repo: "repository", + Branch: "branch", + }}, + }, } for _, tc := range tests { diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_disable_all_rehearsals_from_ci_operator_prowgen_config.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_disable_all_rehearsals_from_ci_operator_prowgen_config.yaml new file mode 100644 index 00000000000..bf1cf92207a --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_disable_all_rehearsals_from_ci_operator_prowgen_config.yaml @@ -0,0 +1,6 @@ +presubmits: + organization/repository: + - always_run: false + name: pull-ci-organization-repository-branch-unit + - always_run: false + name: pull-ci-organization-repository-branch-e2e diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_disable_rehearsal_from_ci_operator_config_per_test.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_disable_rehearsal_from_ci_operator_config_per_test.yaml new file mode 100644 index 00000000000..dc28d8f7fc6 --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_disable_rehearsal_from_ci_operator_config_per_test.yaml @@ -0,0 +1,8 @@ +presubmits: + organization/repository: + - always_run: false + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-organization-repository-branch-unit + - always_run: false + name: pull-ci-organization-repository-branch-e2e diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_images_job_is_configured_for_slack_reporting.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_images_job_is_configured_for_slack_reporting.yaml deleted file mode 100644 index 8b5fa5136df..00000000000 --- a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_images_job_is_configured_for_slack_reporting.yaml +++ /dev/null @@ -1,25 +0,0 @@ -postsubmits: - organization/repository: - - always_run: true - labels: - ci-operator.openshift.io/is-promotion: "true" - max_concurrency: 1 - name: branch-ci-organization-repository-branch-images - reporter_config: - slack: - channel: some-channel - job_states_to_report: - - error - report_template: some template -presubmits: - organization/repository: - - always_run: false - labels: - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-organization-repository-branch-images - reporter_config: - slack: - channel: some-channel - job_states_to_report: - - error - report_template: some template diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_openshift_priv_org_defaults_to_private.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_openshift_priv_org_defaults_to_private.yaml new file mode 100644 index 00000000000..6904e15669f --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_openshift_priv_org_defaults_to_private.yaml @@ -0,0 +1,71 @@ +presubmits: + openshift-priv/repository: + - agent: kubernetes + always_run: true + branches: + - ^branch$ + - ^branch- + context: ci/prow/unit + decorate: true + decoration_config: + skip_cloning: true + hidden: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-priv-repository-branch-unit + rerun_command: /test unit + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=unit + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )unit,?($|\s.*) diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_periodic_with_custom_max_concurrency.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_periodic_with_custom_max_concurrency.yaml new file mode 100644 index 00000000000..80947869946 --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_periodic_with_custom_max_concurrency.yaml @@ -0,0 +1,61 @@ +periodics: +- agent: kubernetes + cron: 0 0 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: branch + org: organization + repo: repository + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + max_concurrency: 2 + name: periodic-ci-organization-repository-branch-nightly + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=nightly + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_postsubmit_with_custom_max_concurrency.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_postsubmit_with_custom_max_concurrency.yaml new file mode 100644 index 00000000000..fdb6bf334e1 --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_postsubmit_with_custom_max_concurrency.yaml @@ -0,0 +1,58 @@ +postsubmits: + organization/repository: + - agent: kubernetes + always_run: true + branches: + - ^branch$ + decorate: true + decoration_config: + skip_cloning: true + max_concurrency: 4 + name: branch-ci-organization-repository-branch-publish + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=publish + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_private_from_ci_operator_prowgen_config.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_private_from_ci_operator_prowgen_config.yaml new file mode 100644 index 00000000000..351dd0c7798 --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_private_from_ci_operator_prowgen_config.yaml @@ -0,0 +1,71 @@ +presubmits: + organization/repository: + - agent: kubernetes + always_run: true + branches: + - ^branch$ + - ^branch- + context: ci/prow/unit + decorate: true + decoration_config: + skip_cloning: true + hidden: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-organization-repository-branch-unit + rerun_command: /test unit + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=unit + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )unit,?($|\s.*) diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_private_with_expose_from_ci_operator_prowgen_config.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_private_with_expose_from_ci_operator_prowgen_config.yaml new file mode 100644 index 00000000000..79424f35d5b --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_private_with_expose_from_ci_operator_prowgen_config.yaml @@ -0,0 +1,70 @@ +presubmits: + organization/repository: + - agent: kubernetes + always_run: true + branches: + - ^branch$ + - ^branch- + context: ci/prow/unit + decorate: true + decoration_config: + skip_cloning: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-organization-repository-branch-unit + rerun_command: /test unit + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=unit + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )unit,?($|\s.*) diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_skip_operator_presubmits_from_ci_operator_config.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_skip_operator_presubmits_from_ci_operator_config.yaml new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_skip_operator_presubmits_from_ci_operator_config.yaml @@ -0,0 +1 @@ +{} diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_per_test.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_per_test.yaml new file mode 100644 index 00000000000..fbd5feeb93b --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_per_test.yaml @@ -0,0 +1,73 @@ +presubmits: + organization/repository: + - agent: kubernetes + always_run: true + branches: + - ^branch$ + - ^branch- + context: ci/prow/e2e + decorate: true + decoration_config: + skip_cloning: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-organization-repository-branch-e2e + reporter_config: + slack: + channel: '#from-ci-operator' + job_states_to_report: + - failure + - error + report_template: '{{if eq .Status.State "success"}} :large_green_circle: Job + *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> + {{else}} :red_circle: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. + <{{.Status.URL}}|View logs> {{end}}' + rerun_command: /test e2e + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=e2e + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e,?($|\s.*) diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_with_defaults.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_with_defaults.yaml new file mode 100644 index 00000000000..d83aaaca34b --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_with_defaults.yaml @@ -0,0 +1,73 @@ +presubmits: + organization/repository: + - agent: kubernetes + always_run: true + branches: + - ^branch$ + - ^branch- + context: ci/prow/e2e + decorate: true + decoration_config: + skip_cloning: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-organization-repository-branch-e2e + reporter_config: + slack: + channel: '#test-channel' + job_states_to_report: + - failure + - error + report_template: '{{if eq .Status.State "success"}} :large_green_circle: Job + *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> + {{else}} :red_circle: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. + <{{.Status.URL}}|View logs> {{end}}' + rerun_command: /test e2e + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=e2e + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e,?($|\s.*) diff --git a/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_with_explicit_values.yaml b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_with_explicit_values.yaml new file mode 100644 index 00000000000..143cb7859fe --- /dev/null +++ b/pkg/prowgen/testdata/zz_fixture_TestGenerateJobs_slack_reporter_from_ci_operator_config_with_explicit_values.yaml @@ -0,0 +1,70 @@ +presubmits: + organization/repository: + - agent: kubernetes + always_run: true + branches: + - ^branch$ + - ^branch- + context: ci/prow/e2e + decorate: true + decoration_config: + skip_cloning: true + labels: + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-organization-repository-branch-e2e + reporter_config: + slack: + channel: '#custom-channel' + job_states_to_report: + - success + - failure + report_template: custom template + rerun_command: /test e2e + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=e2e + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e,?($|\s.*) diff --git a/pkg/prowgen/testdata/zz_fixture_TestNewProwJobBaseBuilderForTest_job_excluded_by_patterns_should_not_have_slack_reporter_config.yaml b/pkg/prowgen/testdata/zz_fixture_TestNewProwJobBaseBuilderForTest_job_excluded_by_patterns_should_not_have_slack_reporter_config.yaml deleted file mode 100644 index 40aff4aa323..00000000000 --- a/pkg/prowgen/testdata/zz_fixture_TestNewProwJobBaseBuilderForTest_job_excluded_by_patterns_should_not_have_slack_reporter_config.yaml +++ /dev/null @@ -1,52 +0,0 @@ -agent: kubernetes -decorate: true -decoration_config: - skip_cloning: true -name: prefix-ci-o-r-b-unit-skip -spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --report-credentials-file=/etc/report/credentials - - --target=unit-skip - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator diff --git a/pkg/prowgen/testdata/zz_fixture_TestNewProwJobBaseBuilderForTest_simple_with_slack_reporter_config.yaml b/pkg/prowgen/testdata/zz_fixture_TestNewProwJobBaseBuilderForTest_simple_with_slack_reporter_config.yaml deleted file mode 100644 index aaa7c8cd975..00000000000 --- a/pkg/prowgen/testdata/zz_fixture_TestNewProwJobBaseBuilderForTest_simple_with_slack_reporter_config.yaml +++ /dev/null @@ -1,52 +0,0 @@ -agent: kubernetes -decorate: true -decoration_config: - skip_cloning: true -name: prefix-ci-o-r-b-unit -spec: - containers: - - args: - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --report-credentials-file=/etc/report/credentials - - --target=unit - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator diff --git a/pkg/validation/test.go b/pkg/validation/test.go index 7aa8e584c5f..cbac0e0757e 100644 --- a/pkg/validation/test.go +++ b/pkg/validation/test.go @@ -217,6 +217,10 @@ func (v *Validator) validateTestStepConfiguration( validationErrors = append(validationErrors, fmt.Errorf("%s: job timeout is limited to %s", fieldRootN, maxJobTimeout)) } + if test.SlackReporter != nil && test.SlackReporter.Channel == "" { + validationErrors = append(validationErrors, fmt.Errorf("%s.slack_reporter.channel: must be set when slack_reporter is configured", fieldRootN)) + } + // Validate Secret/Secrets if test.Secret != nil && test.Secrets != nil { validationErrors = append(validationErrors, fmt.Errorf("test.Secret and test.Secrets cannot both be set")) diff --git a/pkg/webreg/zz_generated.ci_operator_reference.go b/pkg/webreg/zz_generated.ci_operator_reference.go index bf35b633e4b..1da47267daa 100644 --- a/pkg/webreg/zz_generated.ci_operator_reference.go +++ b/pkg/webreg/zz_generated.ci_operator_reference.go @@ -271,6 +271,8 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " # UpdateGraph defines the update mode to use when adding the bundle to the base index.\n" + " # Can be: semver (default), semver-skippatch, or replaces\n" + " update_graph: ' '\n" + + " # SkipPresubmits prevents generation of operator bundle presubmit jobs.\n" + + " skip_presubmits: false\n" + " # Substitutions describes the pullspecs in the operator manifests that must be subsituted\n" + " # with the pull specs of the images in the CI registry\n" + " substitutions:\n" + @@ -334,6 +336,21 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " # this will cause both a floating tag and commit-specific tags\n" + " # to be promoted.\n" + " tag_by_commit: true\n" + + "# Prowgen holds fields that control Prow job generation behavior.\n" + + "# These fields were previously configured via the .config.prowgen file.\n" + + "prowgen:\n" + + " # DisableRehearsals prevents all tests in this config from being rehearsed.\n" + + " disable_rehearsals: false\n" + + " # EnableSecretsStoreCSIDriver indicates that jobs should use the new CSI Secrets Store\n" + + " # mechanism to handle multi-stage credentials secrets.\n" + + " enable_secrets_store_csi_driver: false\n" + + " # Expose declares that jobs should not be hidden from view in deck if they\n" + + " # are private. This field has no effect if private is not set.\n" + + " expose: false\n" + + " # Private indicates that generated jobs should be marked as hidden\n" + + " # from display in deck and that they should mount appropriate git credentials\n" + + " # to clone the repository under test.\n" + + " private: false\n" + "# RawSteps are literal Steps that should be\n" + "# included in the final pipeline.\n" + "raw_steps:\n" + @@ -621,6 +638,10 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " # of pull request workflows. Setting this field will\n" + " # create a periodic job instead of a presubmit\n" + " cron: \"\"\n" + + " # DisableRehearsal prevents this specific test from being picked up for rehearsals.\n" + + " # Note: this cannot re-enable rehearsals if they are globally disabled via\n" + + " # prowgen.disable_rehearsals or .config.prowgen's disable_all setting.\n" + + " disable_rehearsal: false\n" + " # Interval is how frequently the test should be run based\n" + " # on the last time the test ran. Setting this field will\n" + " # create a periodic job instead of a presubmit\n" + @@ -1032,6 +1053,10 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " timeout: 0s\n" + " # Override job timeout\n" + " timeout: 0s\n" + + " # MaxConcurrency sets the maximum number of concurrent runs of this job.\n" + + " # For postsubmit and periodic jobs, this defaults to 1 if not set.\n" + + " # A zero value explicitly means no concurrency limit.\n" + + " max_concurrency: 0\n" + " # MinimumInterval to wait between two runs of the job. Consecutive\n" + " # jobs are run at `minimum_interval` + `duration of previous job`\n" + " # apart. Setting this field will create a periodic job instead of a\n" + @@ -1096,6 +1121,19 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " shard_count: 0\n" + " # SkipIfOnlyChanged is a regex that will result in the test being skipped if all changed files match that regex.\n" + " skip_if_only_changed: ' '\n" + + " # SlackReporter configures Slack reporting for this specific test.\n" + + " # When set, the generated Prow job will have Slack reporter configuration\n" + + " # that reports to the specified channel on the specified job states.\n" + + " slack_reporter:\n" + + " # Channel is the Slack channel to report to (e.g., \"#my-channel\").\n" + + " channel: ' '\n" + + " # JobStatesToReport determines which job states trigger a Slack report.\n" + + " # If not set, defaults to [\"failure\", \"error\"].\n" + + " job_states_to_report:\n" + + " - \"\"\n" + + " # ReportTemplate is an optional Go template for the Slack message.\n" + + " # If not set, a default template is used.\n" + + " report_template: ' '\n" + " steps:\n" + " # AllowBestEffortPostSteps defines if any `post` steps can be ignored when\n" + " # they fail. The given step must explicitly ask for being ignored by setting\n" + @@ -1539,6 +1577,10 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " # of pull request workflows. Setting this field will\n" + " # create a periodic job instead of a presubmit\n" + " cron: \"\"\n" + + " # DisableRehearsal prevents this specific test from being picked up for rehearsals.\n" + + " # Note: this cannot re-enable rehearsals if they are globally disabled via\n" + + " # prowgen.disable_rehearsals or .config.prowgen's disable_all setting.\n" + + " disable_rehearsal: false\n" + " # Interval is how frequently the test should be run based\n" + " # on the last time the test ran. Setting this field will\n" + " # create a periodic job instead of a presubmit\n" + @@ -1950,6 +1992,10 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " timeout: 0s\n" + " # Override job timeout\n" + " timeout: 0s\n" + + " # MaxConcurrency sets the maximum number of concurrent runs of this job.\n" + + " # For postsubmit and periodic jobs, this defaults to 1 if not set.\n" + + " # A zero value explicitly means no concurrency limit.\n" + + " max_concurrency: 0\n" + " # MinimumInterval to wait between two runs of the job. Consecutive\n" + " # jobs are run at `minimum_interval` + `duration of previous job`\n" + " # apart. Setting this field will create a periodic job instead of a\n" + @@ -2014,6 +2060,19 @@ const ciOperatorReferenceYaml = "# The list of base images describe\n" + " shard_count: 0\n" + " # SkipIfOnlyChanged is a regex that will result in the test being skipped if all changed files match that regex.\n" + " skip_if_only_changed: ' '\n" + + " # SlackReporter configures Slack reporting for this specific test.\n" + + " # When set, the generated Prow job will have Slack reporter configuration\n" + + " # that reports to the specified channel on the specified job states.\n" + + " slack_reporter:\n" + + " # Channel is the Slack channel to report to (e.g., \"#my-channel\").\n" + + " channel: ' '\n" + + " # JobStatesToReport determines which job states trigger a Slack report.\n" + + " # If not set, defaults to [\"failure\", \"error\"].\n" + + " job_states_to_report:\n" + + " - \"\"\n" + + " # ReportTemplate is an optional Go template for the Slack message.\n" + + " # If not set, a default template is used.\n" + + " report_template: ' '\n" + " steps:\n" + " # AllowBestEffortPostSteps defines if any `post` steps can be ignored when\n" + " # they fail. The given step must explicitly ask for being ignored by setting\n" + diff --git a/test/integration/ci-operator-prowgen/input/config/norehearsals/duper/.config.prowgen b/test/integration/ci-operator-prowgen/input/config/norehearsals/duper/.config.prowgen deleted file mode 100644 index 23beb2ccb99..00000000000 --- a/test/integration/ci-operator-prowgen/input/config/norehearsals/duper/.config.prowgen +++ /dev/null @@ -1,2 +0,0 @@ -rehearsals: - disable_all: true diff --git a/test/integration/ci-operator-prowgen/input/config/norehearsals/duper/norehearsals-duper-master.yaml b/test/integration/ci-operator-prowgen/input/config/norehearsals/duper/norehearsals-duper-master.yaml index 8937b95a017..49ac59005d3 100644 --- a/test/integration/ci-operator-prowgen/input/config/norehearsals/duper/norehearsals-duper-master.yaml +++ b/test/integration/ci-operator-prowgen/input/config/norehearsals/duper/norehearsals-duper-master.yaml @@ -8,6 +8,8 @@ build_root: name: release namespace: openshift tag: golang-1.10 +prowgen: + disable_rehearsals: true resources: '*': limits: diff --git a/test/integration/ci-operator-prowgen/input/config/norehearsals/stuper/.config.prowgen b/test/integration/ci-operator-prowgen/input/config/norehearsals/stuper/.config.prowgen deleted file mode 100644 index 459853821d2..00000000000 --- a/test/integration/ci-operator-prowgen/input/config/norehearsals/stuper/.config.prowgen +++ /dev/null @@ -1,3 +0,0 @@ -rehearsals: - disabled_rehearsals: - - unit diff --git a/test/integration/ci-operator-prowgen/input/config/norehearsals/stuper/norehearsals-stuper-master.yaml b/test/integration/ci-operator-prowgen/input/config/norehearsals/stuper/norehearsals-stuper-master.yaml index 32ca240f3d7..7bb313b0b01 100644 --- a/test/integration/ci-operator-prowgen/input/config/norehearsals/stuper/norehearsals-stuper-master.yaml +++ b/test/integration/ci-operator-prowgen/input/config/norehearsals/stuper/norehearsals-stuper-master.yaml @@ -22,6 +22,7 @@ tests: commands: make test-unit container: from: src + disable_rehearsal: true - as: upload-results commands: make upload-results container: diff --git a/test/integration/ci-operator-prowgen/input/config/private-org/.config.prowgen b/test/integration/ci-operator-prowgen/input/config/private-org/.config.prowgen deleted file mode 100644 index 4eb57c8eba7..00000000000 --- a/test/integration/ci-operator-prowgen/input/config/private-org/.config.prowgen +++ /dev/null @@ -1 +0,0 @@ -private: true \ No newline at end of file diff --git a/test/integration/ci-operator-prowgen/input/config/private-org/duper/private-org-duper-master.yaml b/test/integration/ci-operator-prowgen/input/config/private-org/duper/private-org-duper-master.yaml index e1eddf3170f..4c35b5160bc 100644 --- a/test/integration/ci-operator-prowgen/input/config/private-org/duper/private-org-duper-master.yaml +++ b/test/integration/ci-operator-prowgen/input/config/private-org/duper/private-org-duper-master.yaml @@ -3,6 +3,8 @@ build_root: name: release namespace: openshift tag: golang-1.10 +prowgen: + private: true resources: '*': limits: diff --git a/test/integration/ci-operator-prowgen/input/config/private-org/super/private-org-super-master.yaml b/test/integration/ci-operator-prowgen/input/config/private-org/super/private-org-super-master.yaml index 8456c120da4..81cbbfd43f7 100644 --- a/test/integration/ci-operator-prowgen/input/config/private-org/super/private-org-super-master.yaml +++ b/test/integration/ci-operator-prowgen/input/config/private-org/super/private-org-super-master.yaml @@ -1,5 +1,7 @@ build_root: from_repository: true +prowgen: + private: true resources: '*': limits: diff --git a/test/integration/ci-operator-prowgen/input/config/private/duper/.config.prowgen b/test/integration/ci-operator-prowgen/input/config/private/duper/.config.prowgen deleted file mode 100644 index 4eb57c8eba7..00000000000 --- a/test/integration/ci-operator-prowgen/input/config/private/duper/.config.prowgen +++ /dev/null @@ -1 +0,0 @@ -private: true \ No newline at end of file diff --git a/test/integration/ci-operator-prowgen/input/config/private/duper/private-duper-master.yaml b/test/integration/ci-operator-prowgen/input/config/private/duper/private-duper-master.yaml index ba94b848353..87ec58688bc 100644 --- a/test/integration/ci-operator-prowgen/input/config/private/duper/private-duper-master.yaml +++ b/test/integration/ci-operator-prowgen/input/config/private/duper/private-duper-master.yaml @@ -8,6 +8,8 @@ build_root: name: release namespace: openshift tag: golang-1.10 +prowgen: + private: true images: items: - from: base diff --git a/test/integration/ci-operator-prowgen/input/config/prowgen-config/duper/prowgen-config-duper-master.yaml b/test/integration/ci-operator-prowgen/input/config/prowgen-config/duper/prowgen-config-duper-master.yaml new file mode 100644 index 00000000000..c1a02dc4758 --- /dev/null +++ b/test/integration/ci-operator-prowgen/input/config/prowgen-config/duper/prowgen-config-duper-master.yaml @@ -0,0 +1,52 @@ +base_images: + base: + name: origin-v4.0 + namespace: openshift + tag: base +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: golang-1.10 +prowgen: + private: true + expose: true +resources: + '*': + limits: + cpu: 500m + requests: + cpu: 10m +tag_specification: + name: origin-v4.0 + namespace: openshift +tests: +- as: unit + commands: make test-unit + container: + from: src + slack_reporter: + channel: "#test-channel" +- as: e2e + commands: make test-e2e + container: + from: src + postsubmit: true + slack_reporter: + channel: "#e2e-channel" + job_states_to_report: + - success + - failure + report_template: "custom template" + max_concurrency: 3 +- as: nightly + commands: make nightly + container: + from: src + cron: "0 0 * * *" + disable_rehearsal: true + max_concurrency: 2 +zz_generated_metadata: + branch: master + org: prowgen-config + repo: duper diff --git a/test/integration/ci-operator-prowgen/input/config/slack-report/duper/.config.prowgen b/test/integration/ci-operator-prowgen/input/config/slack-report/duper/.config.prowgen deleted file mode 100644 index 9bf89dd9f08..00000000000 --- a/test/integration/ci-operator-prowgen/input/config/slack-report/duper/.config.prowgen +++ /dev/null @@ -1,16 +0,0 @@ -slack_reporter: -- channel: "#slack-channel" - job_states_to_report: - - success - - failure - - error - report_template: '{{if eq .Status.State "success"}} :rainbow: Job *{{.Spec.Job}}* - ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> :rainbow: {{else}} - :volcano: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View - logs> :volcano: {{end}}' - job_names: - - unit - - upload-results - - lint - excluded_variants: - - exclude diff --git a/test/integration/ci-operator-prowgen/input/config/slack-report/duper/slack-report-duper-master.yaml b/test/integration/ci-operator-prowgen/input/config/slack-report/duper/slack-report-duper-master.yaml index 9ef9e3efa73..c403709b9f2 100644 --- a/test/integration/ci-operator-prowgen/input/config/slack-report/duper/slack-report-duper-master.yaml +++ b/test/integration/ci-operator-prowgen/input/config/slack-report/duper/slack-report-duper-master.yaml @@ -22,6 +22,16 @@ tests: commands: make test-unit container: from: src + slack_reporter: + channel: '#slack-channel' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}} :rainbow: Job *{{.Spec.Job}}* + ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> :rainbow: {{else}} + :volcano: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View + logs> :volcano: {{end}}' - as: e2e commands: make test-e2e container: @@ -31,11 +41,31 @@ tests: container: from: src postsubmit: true + slack_reporter: + channel: '#slack-channel' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}} :rainbow: Job *{{.Spec.Job}}* + ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> :rainbow: {{else}} + :volcano: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View + logs> :volcano: {{end}}' - as: lint commands: make test-lint container: from: src interval: 2h + slack_reporter: + channel: '#slack-channel' + job_states_to_report: + - success + - failure + - error + report_template: '{{if eq .Status.State "success"}} :rainbow: Job *{{.Spec.Job}}* + ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> :rainbow: {{else}} + :volcano: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View + logs> :volcano: {{end}}' zz_generated_metadata: branch: master org: slack-report diff --git a/test/integration/ci-operator-prowgen/input/config/super/duper/.config.prowgen b/test/integration/ci-operator-prowgen/input/config/super/duper/.config.prowgen deleted file mode 100644 index 3047b3f8609..00000000000 --- a/test/integration/ci-operator-prowgen/input/config/super/duper/.config.prowgen +++ /dev/null @@ -1,3 +0,0 @@ -skip_operator_presubmits: -- branch: release-4.19 - variant: periodics diff --git a/test/integration/ci-operator-prowgen/input/config/super/duper/super-duper-release-4.19__periodics.yaml b/test/integration/ci-operator-prowgen/input/config/super/duper/super-duper-release-4.19__periodics.yaml index 93e75b1db42..f03cb8e6f25 100644 --- a/test/integration/ci-operator-prowgen/input/config/super/duper/super-duper-release-4.19__periodics.yaml +++ b/test/integration/ci-operator-prowgen/input/config/super/duper/super-duper-release-4.19__periodics.yaml @@ -9,6 +9,7 @@ operator: bundles: - as: super-duper-bundle base_index: operator-index + skip_presubmits: true releases: initial: integration: diff --git a/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-periodics.yaml b/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-periodics.yaml new file mode 100644 index 00000000000..13f91e4c220 --- /dev/null +++ b/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-periodics.yaml @@ -0,0 +1,68 @@ +periodics: +- agent: kubernetes + cron: 0 0 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: master + org: prowgen-config + repo: duper + labels: + ci.openshift.io/generator: prowgen + max_concurrency: 2 + name: periodic-ci-prowgen-config-duper-master-nightly + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=nightly + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-postsubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-postsubmits.yaml new file mode 100644 index 00000000000..1368ee4f857 --- /dev/null +++ b/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-postsubmits.yaml @@ -0,0 +1,74 @@ +postsubmits: + prowgen-config/duper: + - agent: kubernetes + always_run: true + branches: + - ^master$ + decorate: true + decoration_config: + skip_cloning: true + labels: + ci.openshift.io/generator: prowgen + max_concurrency: 3 + name: branch-ci-prowgen-config-duper-master-e2e + reporter_config: + slack: + channel: '#e2e-channel' + job_states_to_report: + - success + - failure + report_template: custom template + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=e2e + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-presubmits.yaml b/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-presubmits.yaml new file mode 100644 index 00000000000..e567d648fd6 --- /dev/null +++ b/test/integration/ci-operator-prowgen/output/jobs/prowgen-config/duper/prowgen-config-duper-master-presubmits.yaml @@ -0,0 +1,81 @@ +presubmits: + prowgen-config/duper: + - agent: kubernetes + always_run: true + branches: + - ^master$ + - ^master- + context: ci/prow/unit + decorate: true + decoration_config: + skip_cloning: true + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-prowgen-config-duper-master-unit + reporter_config: + slack: + channel: '#test-channel' + job_states_to_report: + - failure + - error + report_template: '{{if eq .Status.State "success"}} :large_green_circle: Job + *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> + {{else}} :red_circle: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. + <{{.Status.URL}}|View logs> {{end}}' + rerun_command: /test unit + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=unit + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )unit,?($|\s.*)