diff --git a/cmd/backport-verifier/main.go b/cmd/backport-verifier/main.go deleted file mode 100644 index 3d6e71feb47..00000000000 --- a/cmd/backport-verifier/main.go +++ /dev/null @@ -1,186 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/ghodss/yaml" - "github.com/sirupsen/logrus" - - utilerrors "k8s.io/apimachinery/pkg/util/errors" - prowconfig "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/config/secret" - prowflagutil "sigs.k8s.io/prow/pkg/flagutil" - "sigs.k8s.io/prow/pkg/githubeventserver" - "sigs.k8s.io/prow/pkg/interrupts" - "sigs.k8s.io/prow/pkg/logrusutil" - "sigs.k8s.io/prow/pkg/pjutil" - - "github.com/openshift/ci-tools/pkg/util/gzip" -) - -// Config maps upstreams to downstreams for verification -type Config struct { - // Repositories is a mapping of downstream org/repo to upstream org/repo - Repositories map[string]string `json:"repositories,omitempty"` -} - -func (c *Config) validate() error { - var errs []error - for downstreamRepo, upstreamRepo := range c.Repositories { - if len(strings.Split(downstreamRepo, "/")) != 2 { - return fmt.Errorf("%s should be in org/repo format", downstreamRepo) - } - - if len(strings.Split(upstreamRepo, "/")) != 2 { - return fmt.Errorf("%s should be in org/repo format", upstreamRepo) - } - } - - return utilerrors.NewAggregate(errs) -} - -type options struct { - mut *sync.RWMutex - - configPath string - webhookSecretFile string - - config *Config - - githubEventServerOptions githubeventserver.Options - github prowflagutil.GitHubOptions - - dryRun bool -} - -func gatherOptions() options { - o := options{ - mut: &sync.RWMutex{}, - } - fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - - fs.StringVar(&o.configPath, "config-path", "", "Path to backport verifier configuration.") - fs.StringVar(&o.webhookSecretFile, "hmac-secret-file", "", "Path to the file containing the GitHub HMAC secret.") - - o.github.AddFlags(fs) - o.githubEventServerOptions.Bind(fs) - - if err := fs.Parse(os.Args[1:]); err != nil { - logrus.WithError(err).Fatalf("cannot parse args: '%s'", os.Args[1:]) - } - return o -} - -func (o *options) Validate() error { - if err := o.github.Validate(o.dryRun); err != nil { - return err - } - - bytes, err := gzip.ReadFileMaybeGZIP(o.configPath) - if err != nil { - return fmt.Errorf("couldn't read configuration file: %v", o.configPath) - } - - var config Config - if err := yaml.Unmarshal(bytes, &config); err != nil { - return fmt.Errorf("couldn't unmarshal configuration: %w", err) - } - o.config = &config - - if err := o.config.validate(); err != nil { - return err - } - - if err := o.githubEventServerOptions.DefaultAndValidate(); err != nil { - return err - } - - return nil -} - -func (o *options) getConfigWatchAndUpdate() (func(ctx context.Context), error) { - errFunc := func(err error, msg string) { - logrus.WithError(err).Error(msg) - } - - eventFunc := func() error { - bytes, err := gzip.ReadFileMaybeGZIP(o.configPath) - if err != nil { - return fmt.Errorf("couldn't read configuration file %s: %w", o.configPath, err) - } - - var c Config - if err := yaml.Unmarshal(bytes, &c); err != nil { - return fmt.Errorf("couldn't unmarshal configuration: %w", err) - } - - if err := c.validate(); err != nil { - return err - } - - o.mut.Lock() - defer o.mut.Unlock() - o.config = &c - logrus.Info("Configuration updated") - - return nil - } - watcher, err := prowconfig.GetCMMountWatcher(eventFunc, errFunc, filepath.Dir(o.configPath)) - if err != nil { - return nil, fmt.Errorf("couldn't get the file watcher: %w", err) - } - - return watcher, nil -} - -func main() { - logrusutil.ComponentInit() - logger := logrus.WithField("plugin", "backport-verifier") - - o := gatherOptions() - if err := o.Validate(); err != nil { - logger.Fatalf("Invalid options: %v", err) - } - - configWatchAndUpdate, err := o.getConfigWatchAndUpdate() - if err != nil { - logger.WithError(err).Fatal("couldn't get config file watch and update function") - } - interrupts.Run(configWatchAndUpdate) - - if err := secret.Add(o.github.TokenPath, o.webhookSecretFile); err != nil { - logger.WithError(err).Fatal("Error starting secrets agent.") - } - - githubClient, err := o.github.GitHubClient(o.dryRun) - if err != nil { - logger.WithError(err).Fatal("Error getting GitHub client.") - } - - serv := &server{ - config: func() *Config { - o.mut.Lock() - defer o.mut.Unlock() - return o.config - }, - ghc: githubClient, - } - - eventServer := githubeventserver.New(o.githubEventServerOptions, secret.GetTokenGenerator(o.webhookSecretFile), logger) - eventServer.RegisterHandleIssueCommentEvent(serv.handleIssueComment) - eventServer.RegisterHandlePullRequestEvent(serv.handlePullRequestEvent) - eventServer.RegisterHelpProvider(helpProvider, logger) - - health := pjutil.NewHealth() - health.ServeReady() - - interrupts.ListenAndServe(eventServer, time.Second*30) - interrupts.WaitForGracefulShutdown() -} diff --git a/cmd/backport-verifier/server.go b/cmd/backport-verifier/server.go deleted file mode 100644 index ebe647b9908..00000000000 --- a/cmd/backport-verifier/server.go +++ /dev/null @@ -1,194 +0,0 @@ -package main - -import ( - "fmt" - "regexp" - "sort" - "strconv" - "strings" - - "github.com/sirupsen/logrus" - - prowconfig "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/pluginhelp" -) - -type githubClient interface { - ListPullRequestCommits(org, repo string, number int) ([]github.RepositoryCommit, error) - GetPullRequest(org, repo string, number int) (*github.PullRequest, error) - - CreateComment(owner, repo string, number int, comment string) error - - AddLabel(org, repo string, number int, label string) error - RemoveLabel(org, repo string, number int, label string) error -} - -const ( - validatedBackportsLabel = "backports/validated-commits" - unvalidatedBackportsLabel = "backports/unvalidated-commits" -) - -var ( - commandRe = regexp.MustCompile(`(?mi)^/validate-backports\s*$`) - upstreamPullRe = regexp.MustCompile(`^UPSTREAM: ([0-9]+): `) -) - -func helpProvider(_ []prowconfig.OrgRepo) (*pluginhelp.PluginHelp, error) { - pluginHelp := &pluginhelp.PluginHelp{ - Description: `The backport validation plugin is used to validate that backports come from merged PRs in a configured upstream repository.`, - } - pluginHelp.AddCommand(pluginhelp.Command{ - Usage: "/validate-backports", - Description: "Validate that backports come from merged PRs in the upstream repository", - WhoCanUse: "Anyone", - Examples: []string{"/validate-backports"}, - }) - return pluginHelp, nil -} - -type server struct { - config func() *Config - - ghc githubClient -} - -func (s *server) handleIssueComment(l *logrus.Entry, ic github.IssueCommentEvent) { - if !commandRe.MatchString(ic.Comment.Body) { - return - } - l.Info("Backport validation of PR has been requested.") - s.handle(l, ic.Repo.Owner.Login, ic.Repo.Name, ic.Comment.User.Login, ic.Issue.Number, true) -} - -func (s *server) handlePullRequestEvent(l *logrus.Entry, event github.PullRequestEvent) { - if event.Action != github.PullRequestActionOpened && event.Action != github.PullRequestActionSynchronize { - return - } - l.Info("Changes to pull request require backport validation") - s.handle(l, event.Repo.Owner.Login, event.Repo.Name, event.PullRequest.User.Login, event.PullRequest.Number, false) -} - -func (s *server) handle(l *logrus.Entry, org, repo, user string, num int, requested bool) { - logger := l.WithFields(logrus.Fields{ - github.OrgLogField: org, - github.RepoLogField: repo, - github.PrLogField: num, - }) - - upstream, configured := s.config().Repositories[fmt.Sprintf("%s/%s", org, repo)] - if !configured { - if requested { - if err := s.ghc.CreateComment(org, repo, num, fmt.Sprintf("@%s: no upstream repository is configured for validating backports for this repository.", user)); err != nil { - logger.WithError(err).Warn("couldn't create comment") - } - ensureLabels(s.ghc, l, unvalidatedBackportsLabel, org, repo, num) - } - return - } - - parts := strings.Split(upstream, "/") - upstreamOrg, upstreamRepo := parts[0], parts[1] - - commits, err := s.ghc.ListPullRequestCommits(org, repo, num) - if err != nil { - if commentErr := s.ghc.CreateComment(org, repo, num, fmt.Sprintf(`@%s: could not list commits in this pull request. Please try again with /validate-backports. - -
- -%s - -
`, user, err)); commentErr != nil { - logger.WithError(commentErr).Warn("couldn't list commits") - } - return - } - - invalidCommits := map[string]string{} - validCommits := map[string]string{} - upstreamPullsByCommit := map[string]int{} - errorsByCommit := map[string]string{} - messagesByCommit := map[string]string{} - for _, commit := range commits { - messagesByCommit[commit.SHA] = strings.Split(commit.Commit.Message, "\n")[0] - parts := upstreamPullRe.FindStringSubmatch(commit.Commit.Message) - if len(parts) != 2 { - invalidCommits[commit.SHA] = "does not specify an upstream backport in the commit message" - continue - } - pr, err := strconv.Atoi(parts[1]) - if err != nil { - // based on the regex this should not happen ... - logger.WithError(err).Warn("Failed to parse PR as integer") - errorsByCommit[commit.SHA] = fmt.Sprintf("failed to parse PR identifier: %s", err.Error()) - continue - } - upstreamPullsByCommit[commit.SHA] = pr - } - - for commit, pull := range upstreamPullsByCommit { - prMention := fmt.Sprintf("the upstream PR [%s/%s#%d](https://redirect.github.com/%s/%s/pull/%d)", upstreamOrg, upstreamRepo, pull, upstreamOrg, upstreamRepo, pull) - pr, err := s.ghc.GetPullRequest(upstreamOrg, upstreamRepo, pull) - if err != nil { - if !github.IsNotFound(err) { - logger.WithError(err).Warn("Failed to get upstream PR") - errorsByCommit[commit] = fmt.Sprintf("failed to fetch upstream PR: %s", err.Error()) - continue - } - invalidCommits[commit] = prMention + " does not exist" - continue - } - if !pr.Merged { - invalidCommits[commit] = prMention + " has not yet merged" - } else { - validCommits[commit] = prMention + " has merged" - } - } - - desired := unvalidatedBackportsLabel - verb := "could not" - if len(errorsByCommit) == 0 && len(invalidCommits) == 0 { - desired = validatedBackportsLabel - verb = "could" - } - ensureLabels(s.ghc, l, desired, org, repo, num) - - message := fmt.Sprintf("@%s: the contents of this pull request %s be automatically validated.", user, verb) - for _, item := range []struct { - qualifier string - data map[string]string - }{ - {"are valid", validCommits}, - {"could not be validated and must be approved by a top-level approver", invalidCommits}, - {"could not be processed", errorsByCommit}, - } { - if len(item.data) > 0 { - var formatted []string - for commit, why := range item.data { - formatted = append(formatted, fmt.Sprintf(" - [%s|%s](https://github.com/%s/%s/commit/%s): %s", commit[0:7], messagesByCommit[commit], org, repo, commit, why)) - } - sort.Strings(formatted) - message = fmt.Sprintf("%s\n\nThe following commits %s:\n%s", message, item.qualifier, strings.Join(formatted, "\n")) - } - } - footer := "\n\nComment /validate-backports to re-evaluate validity of the upstream PRs, for example when they are merged upstream." - message = message + footer - if commentErr := s.ghc.CreateComment(org, repo, num, message); commentErr != nil { - logger.WithError(commentErr).Warn("couldn't respond to user") - } -} - -func ensureLabels(client githubClient, l *logrus.Entry, desired string, org, repo string, num int) { - var unwanted string - if desired == validatedBackportsLabel { - unwanted = unvalidatedBackportsLabel - } else { - unwanted = validatedBackportsLabel - } - if err := client.AddLabel(org, repo, num, desired); err != nil { - l.WithError(err).Warn("could not add label", err) - } - if err := client.RemoveLabel(org, repo, num, unwanted); err != nil { - l.WithError(err).Warn("could not remove label", err) - } -} diff --git a/cmd/backport-verifier/server_test.go b/cmd/backport-verifier/server_test.go deleted file mode 100644 index 5cb09eb5b7a..00000000000 --- a/cmd/backport-verifier/server_test.go +++ /dev/null @@ -1,200 +0,0 @@ -package main - -import ( - "errors" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/github" -) - -type orgrepopr struct { - org, repo string - pr int -} - -type fakeClient struct { - commits map[orgrepopr][]github.RepositoryCommit - commitErrors map[orgrepopr]error - - prs map[orgrepopr]*github.PullRequest - prErrors map[orgrepopr]error - - comments map[orgrepopr][]string - - labels map[orgrepopr][]string -} - -func (c *fakeClient) ListPullRequestCommits(org, repo string, number int) ([]github.RepositoryCommit, error) { - orp := orgrepopr{org: org, repo: repo, pr: number} - if err, exist := c.commitErrors[orp]; exist && err != nil { - return nil, err - } - - if data, exist := c.commits[orp]; exist { - return data, nil - } else { - return nil, errors.New("no commits configured for this PR") - } -} - -func (c *fakeClient) GetPullRequest(org, repo string, number int) (*github.PullRequest, error) { - orp := orgrepopr{org: org, repo: repo, pr: number} - if err, exist := c.prErrors[orp]; exist && err != nil { - return nil, err - } - if data, exist := c.prs[orp]; exist { - return data, nil - } else { - return nil, errors.New("no data configured for this PR") - } -} - -func (c *fakeClient) CreateComment(owner, repo string, number int, comment string) error { - orp := orgrepopr{org: owner, repo: repo, pr: number} - c.comments[orp] = append(c.comments[orp], comment) - return nil -} - -func (c *fakeClient) AddLabel(org, repo string, number int, label string) error { - orp := orgrepopr{org: org, repo: repo, pr: number} - c.labels[orp] = append(c.labels[orp], label) - return nil -} - -func (c *fakeClient) RemoveLabel(org, repo string, number int, label string) error { - orp := orgrepopr{org: org, repo: repo, pr: number} - var updated []string - for _, old := range c.labels[orp] { - if old != label { - updated = append(updated, old) - } - } - c.labels[orp] = updated - return nil -} - -func TestHandle(t *testing.T) { - var testCases = []struct { - name string - config Config - requested bool - commits []github.RepositoryCommit - commitError error - prs map[orgrepopr]*github.PullRequest - prErrors map[orgrepopr]error - labels []string - expectedLabels []string - expectedComments []string - }{ - { - name: "no config", - requested: true, - config: Config{Repositories: map[string]string{}}, - expectedLabels: []string{unvalidatedBackportsLabel}, - expectedComments: []string{"@author: no upstream repository is configured for validating backports for this repository."}, - }, - { - name: "no but not requested explicitly", - requested: false, - config: Config{Repositories: map[string]string{}}, - expectedComments: []string{}, - }, - { - name: "valid upstreams", - config: Config{Repositories: map[string]string{"org/repo": "upstream/repo"}}, - commits: []github.RepositoryCommit{ - {SHA: "123456789", Commit: github.GitCommit{Message: "UPSTREAM: 1: whoa"}}, - {SHA: "456789abc", Commit: github.GitCommit{Message: "UPSTREAM: 2: whoa"}}, - {SHA: "789abcdef", Commit: github.GitCommit{Message: "UPSTREAM: 3: whoa"}}, - }, - prs: map[orgrepopr]*github.PullRequest{ - {org: "upstream", repo: "repo", pr: 1}: {Merged: true}, - {org: "upstream", repo: "repo", pr: 2}: {Merged: true}, - {org: "upstream", repo: "repo", pr: 3}: {Merged: true}, - }, - labels: []string{unvalidatedBackportsLabel}, - expectedLabels: []string{validatedBackportsLabel}, - expectedComments: []string{`@author: the contents of this pull request could be automatically validated. - -The following commits are valid: - - [1234567|UPSTREAM: 1: whoa](https://github.com/org/repo/commit/123456789): the upstream PR [upstream/repo#1](https://redirect.github.com/upstream/repo/pull/1) has merged - - [456789a|UPSTREAM: 2: whoa](https://github.com/org/repo/commit/456789abc): the upstream PR [upstream/repo#2](https://redirect.github.com/upstream/repo/pull/2) has merged - - [789abcd|UPSTREAM: 3: whoa](https://github.com/org/repo/commit/789abcdef): the upstream PR [upstream/repo#3](https://redirect.github.com/upstream/repo/pull/3) has merged - -Comment /validate-backports to re-evaluate validity of the upstream PRs, for example when they are merged upstream.`}, - }, - { - name: "invalid upstreams", - config: Config{Repositories: map[string]string{"org/repo": "upstream/repo"}}, - commits: []github.RepositoryCommit{ - {SHA: "123456789", Commit: github.GitCommit{Message: "UPSTREAM: 1: whoa"}}, - {SHA: "456789abc", Commit: github.GitCommit{Message: "UPSTREAM: 2: whoa"}}, - {SHA: "789abcdef", Commit: github.GitCommit{Message: "UPSTREAM: 3: whoa"}}, - {SHA: "abcdefghi", Commit: github.GitCommit{Message: "UPSTREAM: : whoa"}}, - {SHA: "defghijkl", Commit: github.GitCommit{Message: "UPSTREAM: 4: whoa\nmore\ndata\nto\nbe\nskipped"}}, - }, - prs: map[orgrepopr]*github.PullRequest{ - {org: "upstream", repo: "repo", pr: 1}: {Merged: true}, - {org: "upstream", repo: "repo", pr: 2}: {Merged: false}, - }, - prErrors: map[orgrepopr]error{ - {org: "upstream", repo: "repo", pr: 3}: errors.New("injected error"), - {org: "upstream", repo: "repo", pr: 4}: github.NewNotFound(), - }, - expectedLabels: []string{unvalidatedBackportsLabel}, - expectedComments: []string{`@author: the contents of this pull request could not be automatically validated. - -The following commits are valid: - - [1234567|UPSTREAM: 1: whoa](https://github.com/org/repo/commit/123456789): the upstream PR [upstream/repo#1](https://redirect.github.com/upstream/repo/pull/1) has merged - -The following commits could not be validated and must be approved by a top-level approver: - - [456789a|UPSTREAM: 2: whoa](https://github.com/org/repo/commit/456789abc): the upstream PR [upstream/repo#2](https://redirect.github.com/upstream/repo/pull/2) has not yet merged - - [abcdefg|UPSTREAM: : whoa](https://github.com/org/repo/commit/abcdefghi): does not specify an upstream backport in the commit message - - [defghij|UPSTREAM: 4: whoa](https://github.com/org/repo/commit/defghijkl): the upstream PR [upstream/repo#4](https://redirect.github.com/upstream/repo/pull/4) does not exist - -The following commits could not be processed: - - [789abcd|UPSTREAM: 3: whoa](https://github.com/org/repo/commit/789abcdef): failed to fetch upstream PR: injected error - -Comment /validate-backports to re-evaluate validity of the upstream PRs, for example when they are merged upstream.`}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - testCase := tc - t.Parallel() - orp := orgrepopr{ - org: "org", - repo: "repo", - pr: 1, - } - client := &fakeClient{ - commits: map[orgrepopr][]github.RepositoryCommit{orp: testCase.commits}, - commitErrors: map[orgrepopr]error{orp: testCase.commitError}, - prs: testCase.prs, - prErrors: testCase.prErrors, - comments: map[orgrepopr][]string{orp: {}}, - labels: map[orgrepopr][]string{orp: testCase.labels}, - } - s := &server{ - config: func() *Config { - return &testCase.config - }, - ghc: client, - } - - s.handle(logrus.WithField("testcase", testCase.name), "org", "repo", "author", 1, testCase.requested) - - if diff := cmp.Diff(testCase.expectedComments, client.comments[orp]); diff != "" { - t.Errorf("%s: got incorrect comments: %v", testCase.name, diff) - } - - if diff := cmp.Diff(testCase.expectedLabels, client.labels[orp]); diff != "" { - t.Errorf("%s: got incorrect labels: %v", testCase.name, diff) - } - }) - } -} diff --git a/cmd/ci-scheduling-webhook/main.go b/cmd/ci-scheduling-webhook/main.go deleted file mode 100644 index 793b20b258b..00000000000 --- a/cmd/ci-scheduling-webhook/main.go +++ /dev/null @@ -1,207 +0,0 @@ -package main - -import ( - "context" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "flag" - "fmt" - "log" - "math/big" - "net/http" - "os" - "time" - - "github.com/spf13/cobra" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" -) - -const ( - CiWorkloadLabelName = "ci-workload" - CiWorkloadPreferNoScheduleTaintName = "ci-workload-avoid" - CiWorkloadPreferNoExecuteTaintName = "ci-workload-evict" - CiWorkloadNamespaceLabelName = "ci-workload-namespace" -) - -var ( - tlsCertFile string - tlsKeyFile string - port int - impersonateUser string - minBuildMillicores int64 - codecs = serializer.NewCodecFactory(runtime.NewScheme()) - logger = log.New(os.Stdout, "http: ", log.LstdFlags) - - prioritization Prioritization -) - -func generateTestCertificate() (*tls.Certificate, error) { - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return nil, fmt.Errorf("test private key cannot be created: %w", err) - } - - // Generate a pem block with the private key - keyPem := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(key), - }) - - tml := x509.Certificate{ - // you can add any attr that you need - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(5, 0, 0), - // you have to generate a different serial number each execution - SerialNumber: big.NewInt(123123), - Subject: pkix.Name{ - CommonName: "New Name", - Organization: []string{"New Org."}, - }, - BasicConstraintsValid: true, - } - cert, err := x509.CreateCertificate(rand.Reader, &tml, &tml, &key.PublicKey, key) - if err != nil { - return nil, fmt.Errorf("test certificate key cannot be created: %w", err) - } - - // Generate a pem block with the certificate - certPem := pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: cert, - }) - - tlsCert, err := tls.X509KeyPair(certPem, keyPem) - if err != nil { - return nil, fmt.Errorf("test certificate could not be loaded: %w", err) - } - - return &tlsCert, nil -} - -func Run(_ *cobra.Command, _ []string) { - - if (tlsCertFile == "" || tlsKeyFile == "") && (tlsCertFile != "" || tlsKeyFile != "") { - fmt.Println("--tls-cert and --tls-key required must both be specified or both omitted") - os.Exit(1) - } - - var cert *tls.Certificate - var err error - if tlsCertFile == "" { - klog.Warning("No cert/key pair specified -- generating test certificate") - cert, err = generateTestCertificate() - if err != nil { - fmt.Printf("Error creating test cert / key: %v", err) - os.Exit(1) - } - } else { - certP, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile) - if err != nil { - fmt.Printf("Error loading tls files from file system: %v", err) - os.Exit(1) - } - cert = &certP - } - - kubeConfigPath, kubeConfigPresent := os.LookupEnv("KUBECONFIG") - ctx := context.TODO() - - kubeConfig := "" - if kubeConfigPresent { - kubeConfig = kubeConfigPath - } - - config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) - if err != nil { - klog.Errorf("Error initializing client config: %v", err) - os.Exit(1) - } - - if impersonateUser != "" { - config.Impersonate = rest.ImpersonationConfig{ - UserName: impersonateUser, - } - } - - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - klog.Errorf("Error initializing kubernetes client set: %v", err) - os.Exit(1) - } - - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - klog.Errorf("Error initializing dynamic client: %v", err) - os.Exit(1) - } - - prioritization = Prioritization{ - context: ctx, - k8sClientSet: clientSet, - dynamicClient: dynamicClient, - } - err = prioritization.initializePrioritization() - if err != nil { - klog.Errorf("Error initializing node prioritization processes: %v", err) - os.Exit(1) - } - runWebhookServer(cert) -} - -var rootCmd = &cobra.Command{ - Use: "ci-scheduling-webhook", - Short: "Improves cost-efficiency when scheduling OpenShift's CI workloads", - Long: `Improves cost-efficiency when scheduling OpenShift's CI workloads. - -Example: -$ ci-scheduling-webhook --tls-cert --tls-key --port `, - Run: Run, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - cobra.CheckErr(rootCmd.Execute()) -} - -func init() { - klog.InitFlags(nil) - rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine) - - rootCmd.Flags().StringVar(&tlsCertFile, "tls-cert", "", "Certificate for TLS") - rootCmd.Flags().StringVar(&tlsKeyFile, "tls-key", "", "Private key file for TLS") - rootCmd.Flags().IntVar(&port, "port", 443, "Port to listen on for HTTPS traffic") - rootCmd.Flags().StringVar(&impersonateUser, "as", "", "Impersonate a user, like system:admin") - rootCmd.Flags().Int64Var(&minBuildMillicores, "min-build-millicores", 4000, "Minimum CPU millicores to enforce for docker-build containers") -} - -func runWebhookServer(cert *tls.Certificate) { - fmt.Println("Starting webhook server") - http.HandleFunc("/mutate", mutatePod) - server := http.Server{ - Addr: fmt.Sprintf(":%d", port), - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{*cert}, - }, - ErrorLog: logger, - } - - if err := server.ListenAndServeTLS("", ""); err != nil { - panic(err) - } -} - -func main() { - Execute() -} diff --git a/cmd/ci-scheduling-webhook/monitoring/failed-pods.sh b/cmd/ci-scheduling-webhook/monitoring/failed-pods.sh deleted file mode 100755 index d406214d27e..00000000000 --- a/cmd/ci-scheduling-webhook/monitoring/failed-pods.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo "Failed pods:" -oc get pods --all-namespaces -o=wide --field-selector=status.phase==Failed diff --git a/cmd/ci-scheduling-webhook/monitoring/problem-pods.sh b/cmd/ci-scheduling-webhook/monitoring/problem-pods.sh deleted file mode 100755 index fc1f833c594..00000000000 --- a/cmd/ci-scheduling-webhook/monitoring/problem-pods.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo "Unschedulable pods:" -oc get pods --all-namespaces -o=wide --field-selector=status.phase==Pending diff --git a/cmd/ci-scheduling-webhook/mutation.go b/cmd/ci-scheduling-webhook/mutation.go deleted file mode 100644 index 3e5350f49d6..00000000000 --- a/cmd/ci-scheduling-webhook/mutation.go +++ /dev/null @@ -1,682 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" - - admissionv1 "k8s.io/api/admission/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" - "k8s.io/utils/ptr" -) - -const ( - CiBuildNameLabelName = "openshift.io/build.name" - CiNamepsace = "ci" - CiCreatedByProwLabelName = "created-by-prow" - KubernetesHostnameLabelName = "kubernetes.io/hostname" -) - -var ( - // Non "openshift-*" namespace that need safe-to-evict - safeToEvictNamespace = map[string]bool{ - "rh-corp-logging": true, - "ocp": true, - "cert-manager": true, - } - memoryThreshold = resource.MustParse("32Gi") - cpuThreshold = resource.MustParse("13") -) - -func admissionReviewFromRequest(r *http.Request, deserializer runtime.Decoder) (*admissionv1.AdmissionReview, error) { - // Validate that the incoming content type is correct. - if r.Header.Get("Content-Type") != "application/json" { - return nil, fmt.Errorf("expected application/json content-type") - } - - // Get the body data, which will be the AdmissionReview - // content for the request. - var body []byte - if r.Body != nil { - requestData, err := io.ReadAll(r.Body) - if err != nil { - return nil, err - } - body = requestData - } - - // Decode the request body into - admissionReviewRequest := &admissionv1.AdmissionReview{} - if _, _, err := deserializer.Decode(body, nil, admissionReviewRequest); err != nil { - return nil, err - } - - return admissionReviewRequest, nil -} - -func mutatePod(w http.ResponseWriter, r *http.Request) { - start := time.Now() - lastProfileTime := &start - - deserializer := codecs.UniversalDeserializer() - - writeHttpError := func(statusCode int, err error) { - msg := fmt.Sprintf("error during mutation operation: %v", err) - klog.Error(msg) - w.WriteHeader(statusCode) - _, err = w.Write([]byte(msg)) - if err != nil { - klog.Errorf("Unable to return http error response to caller: %v", err) - } - } - - // Parse the AdmissionReview from the http request. - admissionReviewRequest, err := admissionReviewFromRequest(r, deserializer) - if err != nil { - writeHttpError(400, fmt.Errorf("error getting admission review from request: %w", err)) - return - } - - nodeResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes"} - if admissionReviewRequest.Request == nil || admissionReviewRequest.Request.Resource == nodeResource { - mutateNode(admissionReviewRequest, w) - return - } - - // Do server-side validation that we are only dealing with a pod resource. This - // should also be part of the MutatingWebhookConfiguration in the cluster, but - // we should verify here before continuing. - podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} - if admissionReviewRequest.Request == nil || admissionReviewRequest.Request.Resource != podResource { - writeHttpError(400, fmt.Errorf("did not receive pod, got %v", admissionReviewRequest.Request)) - return - } - - // Decode the pod from the AdmissionReview. - rawRequest := admissionReviewRequest.Request.Object.Raw - pod := corev1.Pod{} - if _, _, err := deserializer.Decode(rawRequest, nil, &pod); err != nil { - writeHttpError(500, fmt.Errorf("error decoding raw pod: %w", err)) - return - } - - profile := func(action string) { - duration := time.Since(start) - sinceLastProfile := time.Since(*lastProfileTime) - now := time.Now() - lastProfileTime = &now - klog.Infof("mutate-pod [%v] [%v] within (ms): %v [diff %v]", admissionReviewRequest.Request.UID, action, duration.Milliseconds(), sinceLastProfile.Milliseconds()) - } - - profile("decoded request") - - podClass := PodClassNone // will be set to CiWorkloadLabelValueBuilds or CiWorkloadLabelValueTests depending on analysis - - patchEntries := make([]map[string]interface{}, 0) - addPatchEntry := func(op string, path string, value interface{}) { - patch := map[string]interface{}{ - "op": op, - "path": path, - "value": value, - } - patchEntries = append(patchEntries, patch) - } - - podName := admissionReviewRequest.Request.Name - namespace := admissionReviewRequest.Request.Namespace - - // OSD has so. many. operator related resources which aren't using replicasets. - // These are normally unevictable and thus prevent the autoscaler from considering - // a node for deletion. We mark them evictable. - // OSD operator catalogs are presently unevictable, so do those wherever we find them. - _, needsEvitable := safeToEvictNamespace[namespace] - if strings.HasPrefix(namespace, "openshift-") || strings.Contains(podName, "-catalog-") || needsEvitable { - annotations := pod.Annotations - if annotations == nil { - annotations = make(map[string]string, 0) - } - annotations["cluster-autoscaler.kubernetes.io/safe-to-evict"] = "true" - addPatchEntry("add", "/metadata/annotations", annotations) - } - - if namespace == CiNamepsace { - if _, ok := pod.Labels[CiCreatedByProwLabelName]; ok { - // if we are in 'ci' and created by prow, this the direct prowjob pod. Treat as test. - podClass = PodClassProwJobs - } - } - - labels := pod.Labels - if labels == nil { - labels = make(map[string]string, 0) - } - - if strings.HasPrefix(namespace, "ci-op-") || strings.HasPrefix(namespace, "ci-ln-") { - skipPod := false // certain pods with special resources we can schedule onto our workload sets - - checkForSpecialContainers := func(containers []corev1.Container) { - for i := range containers { - c := &containers[i] - for key := range c.Resources.Requests { - if key != corev1.ResourceCPU && key != corev1.ResourceMemory && key != corev1.ResourceEphemeralStorage { - // There is a special resource required - avoid trying to schedule it with build/test machinesets - skipPod = true - } - } - } - } - - checkForSpecialContainers(pod.Spec.InitContainers) - checkForSpecialContainers(pod.Spec.Containers) - - if !skipPod { - if _, ok := labels[CiBuildNameLabelName]; ok { - podClass = PodClassBuilds - } else { - podClass = PodClassTests - } - } - } - - klog.Infof("Pod %s in namespace %s is classified as %s", podName, namespace, podClass) - - // Add NET_ADMIN and NET_RAW capabilities to test containers that require them - if podClass == PodClassTests { - // Check each container for the environment variable - for i, container := range pod.Spec.Containers { - if container.Name == "test" { - // Check if this container has TEST_REQUIRES_BUILDFARM_NET_ADMIN=true - requiresNetAdmin := false - for _, env := range container.Env { - if env.Name == "TEST_REQUIRES_BUILDFARM_NET_ADMIN" && env.Value == "true" { - requiresNetAdmin = true - break - } - } - - if requiresNetAdmin { - // Build the correct patch based on existing securityContext - if container.SecurityContext == nil { - // No securityContext exists, create one with capabilities - container.SecurityContext = &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN", "NET_RAW", "SETUID", "SETGID"}, - }, - } - } else if container.SecurityContext.Capabilities == nil { - // securityContext exists but no capabilities, add capabilities - container.SecurityContext.Capabilities = &corev1.Capabilities{ - Add: []corev1.Capability{"NET_ADMIN", "NET_RAW", "SETUID", "SETGID"}, - } - } else { - // Both securityContext and capabilities exist, merge the "add" array - container.SecurityContext.Capabilities.Add = append(container.SecurityContext.Capabilities.Add, "NET_ADMIN", "NET_RAW", "SETUID", "SETGID") - } - container.SecurityContext.RunAsUser = ptr.To(int64(0)) - container.SecurityContext.RunAsNonRoot = ptr.To(false) - - addPatchEntry("replace", fmt.Sprintf("/spec/containers/%d/securityContext", i), container.SecurityContext) - klog.Infof("Added NET_ADMIN, NET_RAW, SETUID, and SETGID capabilities, ensured runAsUser=0 and allowPrivilegeEscalation=true for test container in pod %s in namespace %s due to TEST_REQUIRES_BUILDFARM_NET_ADMIN=true", podName, namespace) - - // Set CPU resources to 12 cores for NET_ADMIN pods - const netAdminCPUMillicores = 12000 // 12 cores - - var cpuRequest resource.Quantity - var hasCPURequest bool - var cpuLimit resource.Quantity - var hasCPULimit bool - - // Safely check for existing CPU requests/limits - if container.Resources.Requests != nil { - cpuRequest, hasCPURequest = container.Resources.Requests[corev1.ResourceCPU] - } - if container.Resources.Limits != nil { - cpuLimit, hasCPULimit = container.Resources.Limits[corev1.ResourceCPU] - } - - // Handle CPU requests - if !hasCPURequest || cpuRequest.MilliValue() < netAdminCPUMillicores { - // Preserve all existing resource requests - newRequests := make(map[string]interface{}) - for resourceName, quantity := range container.Resources.Requests { - newRequests[string(resourceName)] = quantity.String() - } - // Set/override CPU to 12 cores - newRequests[string(corev1.ResourceCPU)] = fmt.Sprintf("%dm", netAdminCPUMillicores) - - addPatchEntry("add", fmt.Sprintf("/spec/containers/%d/resources/requests", i), newRequests) - if hasCPURequest { - klog.Infof("Increasing NET_ADMIN pod CPU request from %vm to %vm for container %s", cpuRequest.MilliValue(), netAdminCPUMillicores, container.Name) - } else { - klog.Infof("Setting NET_ADMIN pod CPU request to %vm for container %s", netAdminCPUMillicores, container.Name) - } - } - - // Handle CPU limits - only if they exist and are below 12 cores - if hasCPULimit && cpuLimit.MilliValue() < netAdminCPUMillicores { - // Preserve all existing resource limits - newLimits := make(map[string]interface{}) - for resourceName, quantity := range container.Resources.Limits { - newLimits[string(resourceName)] = quantity.String() - } - // Set/override CPU to 12 cores - newLimits[string(corev1.ResourceCPU)] = fmt.Sprintf("%dm", netAdminCPUMillicores) - - addPatchEntry("add", fmt.Sprintf("/spec/containers/%d/resources/limits", i), newLimits) - klog.Infof("Increasing NET_ADMIN pod CPU limit from %vm to %vm for container %s", cpuLimit.MilliValue(), netAdminCPUMillicores, container.Name) - } - } - break - } - } - } - - if podClass == PodClassTests { - // Segmenting long run tests onto their own node set helps normal tests nodes scale down - // more effectively. - if strings.HasPrefix(podName, "release-images-") || - strings.HasPrefix(podName, "release-analysis-aggregator-") || - strings.HasPrefix(podName, "e2e-aws-upgrade") || - strings.HasPrefix(podName, "rpm-repo") || - strings.HasPrefix(podName, "osde2e-stage") || - strings.HasPrefix(podName, "e2e-aws-cnv") || - strings.Contains(podName, "ovn-upgrade-ipi") || - strings.Contains(podName, "ovn-upgrade-ovn") || - strings.Contains(podName, "ovn-upgrade-openshift-e2e-test") { - podClass = PodClassLongTests - } - } - - if podClass != PodClassNone { - profile("classified request") - - // Setup labels we might want to use in the future to set pod affinity - labels[CiWorkloadLabelName] = string(podClass) - labels[CiWorkloadNamespaceLabelName] = namespace - - // Ensure build pods request at least the configured minimum cores for docker-build containers - if podClass == PodClassBuilds && minBuildMillicores > 0 { - for i := range pod.Spec.Containers { - c := &pod.Spec.Containers[i] - - // Only apply to docker-build containers - if c.Name != "docker-build" { - continue - } - - cpuRequest, hasCPURequest := c.Resources.Requests[corev1.ResourceCPU] - cpuLimit, hasCPULimit := c.Resources.Limits[corev1.ResourceCPU] - - // Handle CPU requests - if !hasCPURequest || cpuRequest.MilliValue() < minBuildMillicores { - // Preserve all existing resource requests - newRequests := make(map[string]interface{}) - for resourceName, quantity := range c.Resources.Requests { - newRequests[string(resourceName)] = quantity.String() - } - // Set/override CPU to minimum - newRequests[string(corev1.ResourceCPU)] = fmt.Sprintf("%dm", minBuildMillicores) - - addPatchEntry("add", fmt.Sprintf("/spec/containers/%d/resources/requests", i), newRequests) - if hasCPURequest { - klog.Infof("Increasing build pod CPU request from %vm to %vm for container %s", cpuRequest.MilliValue(), minBuildMillicores, c.Name) - } else { - klog.Infof("Setting build pod CPU request to %vm for container %s", minBuildMillicores, c.Name) - } - } - - // Handle CPU limits - only if they exist and are below minimum - if hasCPULimit && cpuLimit.MilliValue() < minBuildMillicores { - // Preserve all existing resource limits - newLimits := make(map[string]interface{}) - for resourceName, quantity := range c.Resources.Limits { - newLimits[string(resourceName)] = quantity.String() - } - // Set/override CPU to minimum - newLimits[string(corev1.ResourceCPU)] = fmt.Sprintf("%dm", minBuildMillicores) - - addPatchEntry("add", fmt.Sprintf("/spec/containers/%d/resources/limits", i), newLimits) - klog.Infof("Increasing build pod CPU limit from %vm to %vm for container %s", cpuLimit.MilliValue(), minBuildMillicores, c.Name) - } - } - } - - // Setup toleration appropriate for podClass so that it can only land on desired machineset. - // This is achieved by virtue of using a RuntimeClass object which specifies the necessary - // tolerations for each workload. - addPatchEntry("add", "/spec/runtimeClassName", "ci-scheduler-runtime-"+podClass) - - // Set a nodeSelector to ensure this finds our desired machineset nodes - nodeSelector := pod.Spec.NodeSelector - if nodeSelector == nil { - nodeSelector = make(map[string]string) - } - nodeSelector[CiWorkloadLabelName] = string(podClass) - addPatchEntry("add", "/spec/nodeSelector", nodeSelector) - - precludedHostnames := prioritization.findHostnamesToPreclude(podClass) - - // Preserve existing affinity rules (e.g. podAntiAffinity from pod-scaler) - // by merging our nodeAffinity into the pod's existing affinity rather than - // replacing it entirely. - affinity := pod.Spec.Affinity - if affinity == nil { - affinity = &corev1.Affinity{} - } - if affinity.NodeAffinity == nil { - affinity.NodeAffinity = &corev1.NodeAffinity{} - } - affinityChanged := false - if err == nil { - if len(precludedHostnames) > 0 { - // Use MatchExpressions here because MatchFields because MatchExpressions - // only allows one value in the Values list. - requiredNoSchedulingSelector := corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: KubernetesHostnameLabelName, - Operator: "NotIn", - Values: precludedHostnames, - }, - }, - }, - }, - } - affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &requiredNoSchedulingSelector - affinityChanged = true - } - } else { - klog.Errorf("No node precludes will be set in pod due to error: %v", err) - } - - highPerfPod := false - if podClass == PodClassBuilds { - // Use high performance nodes for large pods - for _, container := range pod.Spec.Containers { - if container.Resources.Requests.Memory().Cmp(memoryThreshold) >= 0 || container.Resources.Requests.Cpu().Cmp(cpuThreshold) >= 0 { - klog.Infof("Pod %s in namespace %s requests high performance node", podName, namespace) - highPerfPod = true - } - } - // If this is a build pod, prefer to be scheduled to spot instances for cost efficiency. - // If there are no spot instances, this will be ignored. - affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.PreferredSchedulingTerm{ - { - Weight: 100, - Preference: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - // Prefer spot.io instances that are actual spot instances - Key: "spotinst.io/node-lifecycle", - Operator: "In", - Values: []string{"spot"}, - }, - }, - }, - }, - } - affinityChanged = true - } - - if highPerfPod { - patchHighPerfPod(&pod, podName, namespace, addPatchEntry) - } - - if affinityChanged { - unstructuredAffinity, err := runtime.DefaultUnstructuredConverter.ToUnstructured(affinity) - if err != nil { - writeHttpError(500, fmt.Errorf("error decoding affinity to unstructured data: %w", err)) - return - } - - addPatchEntry("add", "/spec/affinity", unstructuredAffinity) - } - - // There is currently an issue with cluster scale up where pods are stacked up, unschedulable. - // A machine is provisioned. As soon as the machine is provisioned, pods are scheduled to the - // node and they begin to run before DNS daemonset pods can successfully configure the pod. - // These leads to issues like being unable to resolve github.com in clonerefs. - initContainers := pod.Spec.InitContainers - if initContainers == nil { - initContainers = make([]corev1.Container, 0) - } - - initContainerName := "ci-scheduling-dns-wait" - - // This webhook supports reinvocation. Don't add an initContainer every time we are invoked. - found := false - for _, container := range initContainers { - if container.Name == initContainerName { - found = true - break - } - } - - if !found { - - // We've found DNS issues with pods coming up and not being able - // to resolve hosts. This initContainer is a workaround which - // will poll for a successful DNS lookup to a file that should - // always be available. - delayInitContainer := []corev1.Container{ - { - Name: initContainerName, - Image: "registry.access.redhat.com/ubi8", - Command: []string{ - "/bin/sh", - "-c", - `declare -i T; until [[ "$ret" == "0" ]] || [[ "$T" -gt "120" ]]; do curl http://static.redhat.com/test/rhel-networkmanager.txt > /dev/null; ret=$?; sleep 1; let "T+=1"; done`, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("200Mi"), - }, - }, - }, - } - - initContainersMap := map[string][]corev1.Container{ - "initContainers": append(delayInitContainer, initContainers...), // prepend sleep container - } - unstructedInitContainersMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&initContainersMap) - if err != nil { - writeHttpError(500, fmt.Errorf("error decoding initContainers to unstructured data: %w", err)) - return - } - - addPatchEntry("replace", "/spec/initContainers", unstructedInitContainersMap["initContainers"]) - } - - addPatchEntry("add", "/metadata/labels", labels) - } - - // Create a response that will add a label to the pod if it does - // not already have a label with the key of "hello". In this case - // it does not matter what the value is, as long as the key exists. - admissionResponse := &admissionv1.AdmissionResponse{} - patch := make([]byte, 0) - patchType := admissionv1.PatchTypeJSONPatch - - if len(patchEntries) > 0 { - marshalled, err := json.Marshal(patchEntries) - if err != nil { - klog.Errorf("Error marshalling JSON patch (%v) from: %v", patchEntries, err) - writeHttpError(500, fmt.Errorf("error marshalling jsonpatch: %w", err)) - return - } - patch = marshalled - } - - admissionResponse.Allowed = true - if len(patch) > 0 { - klog.InfoS("Incoming pod to be modified", "podClass", podClass, "pod", fmt.Sprintf("-n %v pod/%v", namespace, podName)) - admissionResponse.PatchType = &patchType - admissionResponse.Patch = patch - } - - // Construct the response, which is just another AdmissionReview. - var admissionReviewResponse admissionv1.AdmissionReview - admissionReviewResponse.Response = admissionResponse - admissionReviewResponse.SetGroupVersionKind(admissionReviewRequest.GroupVersionKind()) - admissionReviewResponse.Response.UID = admissionReviewRequest.Request.UID - - resp, err := json.Marshal(admissionReviewResponse) - if err != nil { - writeHttpError(500, fmt.Errorf("error marshalling admission review response: %w", err)) - return - } - profile("ready to write response") - - w.Header().Set("Content-Type", "application/json") - _, err = w.Write(resp) - if err != nil { - klog.Errorf("Unable to respond to caller with admission review: %v", err) - } -} - -func patchHighPerfPod(pod *corev1.Pod, podName, namespace string, addPatchEntry func(string, string, interface{})) { - klog.Infof("Pod %s in namespace %s is a high performance pod", podName, namespace) - tolerations := pod.Spec.Tolerations - tolerations = append(tolerations, corev1.Toleration{ - Key: "ci-instance-type", - Operator: corev1.TolerationOpEqual, - Value: "high-perf", - Effect: corev1.TaintEffectNoSchedule, - }) - addPatchEntry("add", "/spec/tolerations", tolerations) - - if pod.Spec.NodeSelector == nil { - pod.Spec.NodeSelector = make(map[string]string) - } - pod.Spec.NodeSelector["ci-instance-type"] = "high-perf" - addPatchEntry("replace", "/spec/nodeSelector", pod.Spec.NodeSelector) -} - -func mutateNode(admissionReviewRequest *admissionv1.AdmissionReview, w http.ResponseWriter) { - start := time.Now() - lastProfileTime := &start - - deserializer := codecs.UniversalDeserializer() - - writeHttpError := func(statusCode int, err error) { - msg := fmt.Sprintf("error during mutation operation: %v", err) - klog.Error(msg) - w.WriteHeader(statusCode) - _, err = w.Write([]byte(msg)) - if err != nil { - klog.Errorf("Unable to return http error response to caller: %v", err) - } - } - - // Do server-side validation that we are only dealing with a pod resource. This - // should also be part of the MutatingWebhookConfiguration in the cluster, but - // we should verify here before continuing. - nodeResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes"} - if admissionReviewRequest.Request == nil || admissionReviewRequest.Request.Resource != nodeResource { - writeHttpError(400, fmt.Errorf("did not receive node, got %v", admissionReviewRequest.Request)) - return - } - - // Decode the Node from the AdmissionReview. - rawRequest := admissionReviewRequest.Request.Object.Raw - node := corev1.Node{} - if _, _, err := deserializer.Decode(rawRequest, nil, &node); err != nil { - writeHttpError(500, fmt.Errorf("error decoding raw node: %w", err)) - return - } - - profile := func(action string) { - duration := time.Since(start) - sinceLastProfile := time.Since(*lastProfileTime) - now := time.Now() - lastProfileTime = &now - klog.Infof("mutate-node [%v] [%v] within (ms): %v [diff %v]", admissionReviewRequest.Request.UID, action, duration.Milliseconds(), sinceLastProfile.Milliseconds()) - } - - profile("decoded request") - - podClass := PodClassNone // will be set to CiWorkloadLabelValueBuilds or CiWorkloadLabelValueTests depending on analysis - - patchEntries := make([]map[string]interface{}, 0) - addPatchEntry := func(op string, path string, value interface{}) { - patch := map[string]interface{}{ - "op": op, - "path": path, - "value": value, - } - patchEntries = append(patchEntries, patch) - } - - nodeName := admissionReviewRequest.Request.Name - - labels := node.Labels - if labels != nil { - if pc, ok := labels[CiWorkloadLabelName]; ok { - podClass = PodClass(pc) - } - } - - if podClass != PodClassNone { - profile("classified request") - - if _, ok := node.Annotations[NodeDisableScaleDownAnnotationKey]; !ok { - // If this webhook owns this class of node, then we own its scale down in order to prevent - // contention with the autoscaler. Ideally, we would apply this annotation declaratively - // in the machineset, but it doesn't appear to support annotations. Instead, - // we apply it on first sight of it not being present. - // https://github.com/kubernetes/autoscaler/blob/a13c59c2430e5fe0e07d8233a536326394e0c925/cluster-autoscaler/FAQ.md#how-can-i-prevent-cluster-autoscaler-from-scaling-down-a-particular-node - escapedKey := strings.ReplaceAll(NodeDisableScaleDownAnnotationKey, "/", "~1") - addPatchEntry("add", "/metadata/annotations/"+escapedKey, "true") - } - } - - admissionResponse := &admissionv1.AdmissionResponse{} - patch := make([]byte, 0) - patchType := admissionv1.PatchTypeJSONPatch - - if len(patchEntries) > 0 { - marshalled, err := json.Marshal(patchEntries) - if err != nil { - klog.Errorf("Error marshalling JSON patch (%v) from: %v", patchEntries, err) - writeHttpError(500, fmt.Errorf("error marshalling jsonpatch: %w", err)) - return - } - patch = marshalled - } - - admissionResponse.Allowed = true - if len(patch) > 0 { - klog.Info("Incoming node to be modified", "podClass", podClass, "node", nodeName) - admissionResponse.PatchType = &patchType - admissionResponse.Patch = patch - } - - // Construct the response, which is just another AdmissionReview. - var admissionReviewResponse admissionv1.AdmissionReview - admissionReviewResponse.Response = admissionResponse - admissionReviewResponse.SetGroupVersionKind(admissionReviewRequest.GroupVersionKind()) - admissionReviewResponse.Response.UID = admissionReviewRequest.Request.UID - - resp, err := json.Marshal(admissionReviewResponse) - if err != nil { - writeHttpError(500, fmt.Errorf("error marshalling admission review response: %w", err)) - return - } - profile("ready to write response") - - w.Header().Set("Content-Type", "application/json") - _, err = w.Write(resp) - if err != nil { - klog.Errorf("Unable to respond to caller with admission review: %v", err) - } -} diff --git a/cmd/ci-scheduling-webhook/prioritization.go b/cmd/ci-scheduling-webhook/prioritization.go deleted file mode 100644 index ae52d22be5e..00000000000 --- a/cmd/ci-scheduling-webhook/prioritization.go +++ /dev/null @@ -1,1320 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "math" - "math/rand" - "sort" - "strconv" - "strings" - "sync" - "time" - - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/util/taints" -) - -type PodClass string - -const ( - PodClassBuilds PodClass = "builds" - PodClassTests PodClass = "tests" - PodClassLongTests PodClass = "longtests" - PodClassProwJobs PodClass = "prowjobs" - PodClassNone PodClass = "" - - // MachineDeleteAnnotationKey When a machine is annotated with this and the machineset is scaled down, - // it will target machines with this annotation to satisfy the change. - OldMachineDeleteAnnotationKey = "machine.openshift.io/cluster-api-delete-machine" - MachineDeleteAnnotationKey = "machine.openshift.io/delete-machine" - - // NodeDisableScaleDownAnnotationKey makes the autoscaler ignore a node for scale down consideration. - NodeDisableScaleDownAnnotationKey = "cluster-autoscaler.kubernetes.io/scale-down-disabled" - - // NodeMachineAnnotationKey Value is the machine name associated with this node - NodeMachineAnnotationKey = "machine.openshift.io/machine" - - // CiSchedulingKeepNodeAnnotationKey is an annotation with "true" / "false" value which - // can be used by humans to prevent specific nodes from being scaled down (or being avoided). - CiSchedulingKeepNodeAnnotationKey = "ci-scheduling.ci.openshift.io/keep-node" - // Also available as a label applicable by machineset - CiSchedulingKeepNodeLabelKey = "ci-scheduling.ci.openshift.io/keep-node" - - // NodeMachineConfigurationStateAnnotationKey is an annotation machine the machine config - // controller describing whether the machine is being updated or node. - NodeMachineConfigurationStateAnnotationKey = "machineconfiguration.openshift.io/state" - - CiMachineSetClassLabelKey = "ci-machineset-class" -) - -var ( - nodesInformer cache.SharedIndexInformer - podsInformer cache.SharedIndexInformer - machineSetResource = schema.GroupVersionResource{Group: "machine.openshift.io", Version: "v1beta1", Resource: "machinesets"} - machineResource = schema.GroupVersionResource{Group: "machine.openshift.io", Version: "v1beta1", Resource: "machines"} - - // If a node name exists in this map, scale down operations are being attempted for it. - scalingDownNodesByClass = map[PodClass]*sync.Map{ - PodClassBuilds: {}, - PodClassTests: {}, - PodClassLongTests: {}, - PodClassProwJobs: {}, - } - scalingDownAddLock sync.Mutex - - // Locks used to make sure access to machineset and other races are prevented for scale down operations. - nodeClassScaleDownLock = map[PodClass]*sync.Mutex{ - PodClassBuilds: {}, - PodClassTests: {}, - PodClassLongTests: {}, - PodClassProwJobs: {}, - } - - nodeAvoidanceLock sync.Mutex -) - -type Prioritization struct { - context context.Context - k8sClientSet *kubernetes.Clientset - dynamicClient dynamic.Interface -} - -const IndexPodsByNode = "IndexPodsByNode" -const IndexNodesByCiWorkload = "IndexNodesByCiWorkload" - -func (p *Prioritization) nodeUpdated(old, new interface{}) { - oldNode := old.(*corev1.Node) - newNode := new.(*corev1.Node) - - addP, removeP := taints.TaintSetDiff(newNode.Spec.Taints, oldNode.Spec.Taints) - add := make([]string, len(addP)) - remove := make([]string, len(removeP)) - for i := range addP { - add[i] = fmt.Sprintf("%v=%v", addP[i].Key, addP[i].Effect) - } - for i := range removeP { - remove[i] = fmt.Sprintf("%v=%v", removeP[i].Key, removeP[i].Effect) - } - current := make([]string, len(newNode.Spec.Taints)) - for i := range newNode.Spec.Taints { - taint := newNode.Spec.Taints[i] - current[i] = fmt.Sprintf("%v=%v", taint.Key, taint.Effect) - } - - if len(add) > 0 || len(remove) > 0 { - klog.Infof( - "Node taints updated. %v adding(%#v); removing(%#v): %#v", - newNode.Name, add, remove, current, - ) - } -} - -func (p *Prioritization) initializePrioritization() error { - - informerFactory := informers.NewSharedInformerFactory(p.k8sClientSet, 0) - nodesInformer = informerFactory.Core().V1().Nodes().Informer() - - _, err := nodesInformer.AddEventHandler( - cache.ResourceEventHandlerFuncs{ - // Called on resource update and every resyncPeriod on existing resources. - UpdateFunc: p.nodeUpdated, - }, - ) - - if err != nil { - return fmt.Errorf("unable to create new node informer: %w", err) - } - - err = nodesInformer.AddIndexers(map[string]cache.IndexFunc{ - IndexNodesByCiWorkload: func(obj interface{}) ([]string, error) { - node := obj.(*corev1.Node) - workloads := []string{""} - if workload, ok := node.Labels[CiWorkloadLabelName]; ok { - workloads = []string{workload} - } - return workloads, nil - }, - }) - - if err != nil { - return fmt.Errorf("unable to create new node informer index: %w", err) - } - - podsInformer = informerFactory.Core().V1().Pods().Informer() - - // Index pods by the nodes they are assigned to - err = podsInformer.AddIndexers(map[string]cache.IndexFunc{ - IndexPodsByNode: func(obj interface{}) ([]string, error) { - nodeNames := []string{obj.(*corev1.Pod).Spec.NodeName} - return nodeNames, nil - }, - }) - - if err != nil { - return fmt.Errorf("unable to create new pod informer index: %w", err) - } - - err = podsInformer.AddIndexers(map[string]cache.IndexFunc{ - IndexNodesByCiWorkload: func(obj interface{}) ([]string, error) { - pod := obj.(*corev1.Pod) - ciWorkloadClasses := make([]string, 0) // this should be - if pod.Labels != nil { - if workloadClass, ok := pod.Labels[CiWorkloadLabelName]; ok { - ciWorkloadClasses = append(ciWorkloadClasses, workloadClass) - } - } else { - ciWorkloadClasses = append(ciWorkloadClasses, fmt.Sprintf("%v", PodClassNone)) - } - return ciWorkloadClasses, nil - }, - }) - - if err != nil { - return fmt.Errorf("unable to create new pod informer index: %w", err) - } - - stopCh := make(chan struct{}) - informerFactory.Start(stopCh) // runs in background - informerFactory.WaitForCacheSync(stopCh) - - for podClass := range nodeClassScaleDownLock { - // Setup a timer which will help scale down nodes supporting this pod class - go p.pollNodeClassForScaleDown(podClass) - } - - // go p.encourageSpotInstances() - - return nil -} - -func (p *Prioritization) encourageSpotInstances() { //nolint: unused - - onDemandBuildsMachineSetSelector := CiMachineSetClassLabelKey + "=builds" - interruptibleBuildsMachineSetSelector := CiMachineSetClassLabelKey + "=interruptible-builds" - - machineSetClient := p.dynamicClient.Resource(machineSetResource).Namespace("openshift-machine-api") // connect or reconnect client on error - lastOnDemandReplicas := int64(-1) - for range time.Tick(10 * time.Second) { - - onDemandMachineSetList, err := machineSetClient.List(p.context, metav1.ListOptions{LabelSelector: onDemandBuildsMachineSetSelector}) - if err != nil { - klog.Errorf("Error finding on demand machinesets to assess spot instance usage: %v", err) - continue - } - - var currentOnDemandReplicas int64 - for _, onDemandMachineSet := range onDemandMachineSetList.Items { - msName := onDemandMachineSet.GetName() - replicas, found, err := unstructured.NestedInt64(onDemandMachineSet.Object, "spec", "replicas") - if err != nil { - klog.Errorf("Error finding replicas in on demand machineset %v: %v", msName, err) - continue - } - if !found { - klog.Errorf("Did not finding replica count in on demand machineset %v", msName) - continue - } - currentOnDemandReplicas += replicas - } - - minimumInterruptibleReplicas := currentOnDemandReplicas - interruptiblesToEncourage := int64(0) - - if lastOnDemandReplicas > -1 { - // If there are more ondemand than when we checked last time, ensure that - // spot instances are added to match. - if currentOnDemandReplicas > lastOnDemandReplicas { - interruptiblesToEncourage += currentOnDemandReplicas - lastOnDemandReplicas - } - } - lastOnDemandReplicas = currentOnDemandReplicas - - interruptibleMachineSetList, err := machineSetClient.List(p.context, metav1.ListOptions{LabelSelector: interruptibleBuildsMachineSetSelector}) - if err != nil { - klog.Errorf("Error finding interruptible machinesets to assess spot instance usage: %v", err) - continue - } - - adjustableInterruptibleMachineSets := make([]unstructured.Unstructured, 0) - for _, interruptibleMachineSet := range interruptibleMachineSetList.Items { - msName := interruptibleMachineSet.GetName() - replicas, found, err := unstructured.NestedInt64(interruptibleMachineSet.Object, "spec", "replicas") - if err != nil { - klog.Errorf("Error finding replicas in interruptible machineset %v: %v", msName, err) - continue - } - if !found { - klog.Errorf("Did not finding replica count in interruptible machineset %v", msName) - continue - } - availableReplicas, found, err := unstructured.NestedInt64(interruptibleMachineSet.Object, "status", "availableReplicas") - if err != nil { - klog.Errorf("Error finding available replicas in interruptible machineset %v: %v", msName, err) - continue - } - if !found { - klog.Errorf("Did not finding available replica count in interruptible machineset %v", msName) - continue - } - - if replicas > availableReplicas { - // We are already requesting spot instances that have not been successfully provisioned. - klog.Infof("Ignoring interruptible machineset because replica count is greater than available") - continue - } - - adjustableInterruptibleMachineSets = append(adjustableInterruptibleMachineSets, interruptibleMachineSet) - } - - interruptibleMachineSetCount := len(adjustableInterruptibleMachineSets) - if interruptibleMachineSetCount == 0 { - klog.Infof("No interruptible machinesets which can be adjusted. Will check again in 5 minutes.") - time.Sleep(5 * time.Minute) - continue - } - - var totalInterruptibleReplicas int64 - interruptibleReplicaCounts := make([]int64, interruptibleMachineSetCount) - for i, interruptibleMachineSet := range adjustableInterruptibleMachineSets { - msName := interruptibleMachineSet.GetName() - replicas, found, err := unstructured.NestedInt64(interruptibleMachineSet.Object, "spec", "replicas") - if err != nil { - klog.Errorf("Error finding replicas in interruptible machineset %v: %v", msName, err) - totalInterruptibleReplicas = -1 - break - } - if !found { - klog.Errorf("Did not finding replica count in interruptible machineset %v", msName) - totalInterruptibleReplicas = -1 - break - } - totalInterruptibleReplicas += replicas - interruptibleReplicaCounts[i] = replicas - } - - if totalInterruptibleReplicas < 0 { - continue - } - - targetInterruptibleReplicas := totalInterruptibleReplicas + interruptiblesToEncourage - if minimumInterruptibleReplicas > targetInterruptibleReplicas { - targetInterruptibleReplicas = minimumInterruptibleReplicas - } - - interruptiblesToAllocate := targetInterruptibleReplicas - totalInterruptibleReplicas - - if interruptiblesToAllocate == 0 { - klog.Infof("No additional interruptible instances are desired.") - continue - } - - // Randomly allocate the number of new desired interruptibles across interruptible machinesets. - // We don't want the machinesets balanced necessarily. We want to use interruptible instances - // wherever we can successfully get them. Random allocation will succeed in locations - // with available instances and so gravitate in that direction after multiple iterations. - for interruptiblesToAllocate > 0 { - interruptibleReplicaCounts[rand.Intn(interruptibleMachineSetCount)]++ - interruptiblesToAllocate-- - } - - nodeClassScaleDownLock[PodClassBuilds].Lock() - for i, interruptibleMachineSet := range adjustableInterruptibleMachineSets { - msName := interruptibleMachineSet.GetName() - - if interruptibleReplicaCounts[i] > 50 { - // Sanity check - klog.Errorf("Refusing to increase interruptible machineset %v scale beyond 50: %v", msName, err) - continue - } - - scaleUpPatch := []interface{}{ - map[string]interface{}{ - "op": "replace", - "path": "/spec/replicas", - "value": interruptibleReplicaCounts[i], - }, - } - scaleUpPayload, err := json.Marshal(scaleUpPatch) - if err != nil { - klog.Errorf("unable to marshal interruptible machineset %v scale up patch: %#v", msName, err) - continue - } - - _, err = machineSetClient.Patch(p.context, msName, types.JSONPatchType, scaleUpPayload, metav1.PatchOptions{}) - if err != nil { - klog.Errorf("unable to patch interruptible machineset %v with scale down patch: %#v", msName, err) - continue - } - - } - nodeClassScaleDownLock[PodClassBuilds].Unlock() - } -} - -func (p *Prioritization) pollNodeClassForScaleDown(podClass PodClass) { - p.evaluateNodeClassScaleDown(podClass) // just for faster debug - for range time.Tick(time.Minute) { - p.evaluateNodeClassScaleDown(podClass) - } -} - -func (p *Prioritization) isNodeSchedulable(node *corev1.Node) bool { - if node.Spec.Unschedulable { - return false - } - for _, condition := range node.Status.Conditions { - if condition.Type == corev1.NodeReady { - if condition.Status == corev1.ConditionTrue { - return true - } - } - } - return false -} - -// getWorkloadNodes returns all nodes presently available which support a given -// podClass (workload type). -func (p *Prioritization) getWorkloadNodes(podClass PodClass, schedulableNodesOnly bool, minNodeAge time.Duration) ([]*corev1.Node, error) { - items, err := nodesInformer.GetIndexer().ByIndex(IndexNodesByCiWorkload, string(podClass)) - if err != nil { - return nil, err - } - nodes := make([]*corev1.Node, 0) - now := time.Now() - for i := range items { - nodeByIndex := items[i].(*corev1.Node) - nodeObj, exists, err := nodesInformer.GetIndexer().GetByKey(nodeByIndex.Name) - - if err != nil { - klog.Errorf("Error trying to find node object %v: %v", nodeByIndex.Name, err) - continue - } - - if !exists { - klog.Warningf("Node no longer exists: %v", nodeByIndex.Name) - // If the node is cordoned or otherwise unavailable, don't - // include it. We should only return viable nodes for new workloads. - continue - } - - node := nodeObj.(*corev1.Node) - if schedulableNodesOnly && !p.isNodeSchedulable(node) { - // If the node is cordoned or otherwise unavailable, don't - // include it. We should only return viable nodes for new workloads. - continue - } - - // If machine config is doing anything with the node, ignore it. - // machineconfig will taint the node and try to drain it. If we uncordon - // during that time, more pods will be scheduled and redrained leading to - // ci workload pods being deleted unexpectedly. - if mcState, ok := node.Annotations[NodeMachineConfigurationStateAnnotationKey]; ok { - if strings.ToLower(mcState) != "done" { - klog.Warningf("Node %v does not have valid configuration yet %v is in state %v; waiting for done state", nodeByIndex.Name, NodeMachineConfigurationStateAnnotationKey, mcState) - continue - } - } else { - klog.Errorf("Unable to find %v annotation for node: %v", NodeMachineConfigurationStateAnnotationKey, nodeByIndex.Name) - continue - } - - if node.Annotations != nil { - if val, ok := node.Annotations[CiSchedulingKeepNodeAnnotationKey]; ok { - keepNode, _ := strconv.ParseBool(val) - if keepNode { - // If the node should be kept, hide it from all calculations about workloads. - // This prevents it from being scaled down or avoided. - continue - } - } - } - - if node.Labels != nil { - if val, ok := node.Labels[CiSchedulingKeepNodeLabelKey]; ok { - keepNode, _ := strconv.ParseBool(val) - if keepNode { - // If the node should be kept, hide it from all calculations about workloads. - // This prevents it from being scaled down or avoided. - continue - } - } - } - - if now.Sub(node.CreationTimestamp.Time) < minNodeAge { - // node does not meet caller's criteria - continue - } - - nodes = append(nodes, node) - } - return nodes, nil -} - -func (p *Prioritization) isPodActive(pod *corev1.Pod, within time.Duration) bool { - active := pod.Status.Phase == corev1.PodPending || pod.Status.Phase == corev1.PodRunning - if !active { - if len(pod.Finalizers) > 0 { - return true - } - // For the sake of timing conditions, like this, allow pods to be considered - // active within to caller's window: - // https://github.com/openshift/ci-tools/blob/361bb525d35f7fc5ec8eed87d5014b61a99300fc/pkg/steps/template.go#L577 - // count the pod as active for several minutes after actual termination. - for _, cs := range pod.Status.ContainerStatuses { - if cs.State.Terminated == nil { - return true - } - if time.Since(cs.State.Terminated.FinishedAt.Time) < within { - return true - } - } - } - return active -} - -func (p *Prioritization) getPodsUsingNode(nodeName string, classedPodsOnly bool, activeWithin time.Duration) ([]*corev1.Pod, error) { //nolint: unparam - items, err := podsInformer.GetIndexer().ByIndex(IndexPodsByNode, nodeName) - if err != nil { - return nil, err - } - - pods := make([]*corev1.Pod, 0) - for i := range items { - pod := items[i].(*corev1.Pod) - - if classedPodsOnly { - if _, ok := pod.Labels[CiWorkloadLabelName]; !ok { - continue - } - } - - if p.isPodActive(pod, activeWithin) { - // Count only pods which are consuming resources - pods = append(pods, pod) - } - } - - return pods, nil -} - -func (p *Prioritization) getNodeHostname(node *corev1.Node) string { - val, ok := node.Labels[KubernetesHostnameLabelName] - if ok { - return val - } else { - return "" - } -} - -func (p *Prioritization) evaluateNodeScaleDown(podClass PodClass, node *corev1.Node) { - - // Prevent multiple evaluations on the same node at the same time - scalingDownAddLock.Lock() - scalingDownNodes := scalingDownNodesByClass[podClass] - if _, ok := scalingDownNodes.Load(node.Name); ok { - // work is ongoing for this node in another thread. Nothing to do. - scalingDownAddLock.Unlock() - return - } - scalingDownNodes.Store(node.Name, node) - defer scalingDownNodes.Delete(node.Name) - scalingDownAddLock.Unlock() - - klog.Infof("Evaluating second stage of scale down for podClass %v node: %v", podClass, node.Name) - - podCount := 0 - - // Final live query to see if there are really no pods vs shared informer indexer used for quick checks. - queriedPods, err := p.k8sClientSet.CoreV1().Pods(metav1.NamespaceAll).List(p.context, metav1.ListOptions{ - FieldSelector: fmt.Sprintf("spec.nodeName=%v", node.Name), - }) - - if err != nil { - klog.Errorf("Unable to determine real-time pods for node %v: %#v", node.Name, err) - return // Try again later - } - - for _, queriedPod := range queriedPods.Items { - _, ok := queriedPod.Labels[CiWorkloadLabelName] // only count CI workload pods - if ok && p.isPodActive(&queriedPod, 0) { - podCount++ - } - } - - if podCount != 0 { - klog.Errorf("found non zero real-time pod count %v for %v", podCount, node.Name) - return // Try again later - } - - klog.Warningf("Triggering final stage of scale down for podClass %v node: %v", podClass, node.Name) - machineSetNamespace, machineSetName, machineName, err := p.scaleDown(podClass, node) - if err != nil { - // Keep the node cordoned and try again later. - klog.Errorf("Unable to scale down node %v: %v", node.Name, err) - return - } - - // Hold in this method waiting for this node to disappear. This method holds a lock - // which will prevent the code from trying to scale down this particular node again, - // but it will allow attempts on other nodes to proceed. - // There are three ways out of this loop: - // - 1h timeout (machineapi controller is wedged?) - // - Node disappears - // - Machineset says that it is reconciled AND machine is in the "running" phase - - for i := 0; i < 60; i++ { - time.Sleep(1 * time.Minute) - - _, exists, err := nodesInformer.GetIndexer().GetByKey(node.Name) - if err != nil { - klog.Errorf("Error checking scaled down node %v existence: %v", node.Name, err) - } else { - if !exists { - // Success! This node should no longer show up in the avoidance nodes, - klog.Infof("Successfully scaled down node: %v", node.Name) - return - } else { - klog.Infof("Check [%v] - node %v still exists after scale down attempt. This is fine if it is in the process of shutting down.", i, node.Name) - } - } - - machineSetClient := p.dynamicClient.Resource(machineSetResource).Namespace(machineSetNamespace) - ms, err := machineSetClient.Get(p.context, machineSetName, metav1.GetOptions{}) - if err != nil { - klog.Errorf("Error finding machineset %v after scale down attempt for %v existence: %v", machineSetName, node.Name, err) - continue - } - - // Check if machineset status.replicas matches spec.replicas. Should be eventually consistent. - replicas, found, err := unstructured.NestedInt64(ms.UnstructuredContent(), "spec", "replicas") - if err != nil || !found { - klog.Errorf("unable to get current replicas in machineset %v: %#v", machineSetName, err) - continue - } - - statusReplicas, found, err := unstructured.NestedInt64(ms.UnstructuredContent(), "status", "replicas") - if err != nil || !found { - klog.Errorf("unable to get current status.replicas in machineset %v: %#v", machineSetName, err) - continue - } - - if replicas != statusReplicas { - klog.Warningf("existing replicas (%v) != status.replicas (%v) in machineset %v ; still waiting for scale down of %v", replicas, statusReplicas, machineSetName, node.Name) - continue - } - - // Check if machineset status.readyReplicas matches spec.replicas. Should be eventually consistent or not present - // if replicas == 0. - readyReplicas, found, err := unstructured.NestedInt64(ms.UnstructuredContent(), "status", "readyReplicas") - if err != nil { - klog.Errorf("unable to get current status.readyReplicas in machineset %v: %#v", machineSetName, err) - continue - } - - machinePhase, _, _, err := p.getMachinePhase(machineSetNamespace, machineName) - if err != nil { - klog.Errorf("unable to get machine phase for machine %v / node %v: %v", machineName, node.Name, err) - continue - } - - if found && replicas != readyReplicas { - klog.Warningf("existing replicas (%v) != status.readyReplicas (%v) in machineset %v ; still waiting for scale down of %v", replicas, readyReplicas, machineSetName, node.Name) - continue - } else { - if machinePhase == "running" { - // If these values match and our machine is "running", the machineset thinks it is reconciled, so we are clear to try again if the nodes - // is still present. - klog.Errorf("machineset %v appears reconciled but node %v was not removed -- will try again later", machineSetName, node.Name) - return - } - } - - } - - // Its possible some other node was annotated with the deletion annotation and was chosen by - // the machine controller to scale down instead of our node. Allow other scale down attempts. - klog.Errorf("Expected node %v to have disappeared after scale down attempt -- will try again later", node.Name) -} - -// evaluateNodeClassScaleDown is called by a single thread, periodically, to see what -// nodes should be updated in order to scale down or to encourage scale down conditions. -func (p *Prioritization) evaluateNodeClassScaleDown(podClass PodClass) { - - // First, check to see if any nodes have been targeted for scale down in this class. - // Nodes which have been targeted have getNodeAvoidanceState of TaintEffectNoSchedule - // and they are actually cordoned on the cluster. - // Make sure the nodes are at least 15 minutes old, or you might catch one that is cordoned - // during initialization. - allWorkloadNodes, err := p.getWorkloadNodes(podClass, false, 15*time.Minute) - if err != nil { - klog.Errorf("Error finding workload nodes for scale down assessment of podClass %v: %v", podClass, err) - return - } - - for _, node := range allWorkloadNodes { - - if _, ok := node.Labels["spot-io"]; ok { - // This is a spot.io node. It is responsible for scale down, so ignore it. - continue - } - - if p.getNodeAvoidanceState(node) == corev1.TaintEffectNoSchedule { - pods, err := p.getPodsUsingNode(node.Name, true, 0) - if err != nil { - klog.Errorf("Unable to check pod count during class scale down eval for node %v: %#v", node.Name, err) - return - } - - if len(pods) == 0 { - // We set NoSchedule in a previous loop and there are still no pods on the - // node (e.g. a race between our patch and a pod being scheduled might - // have violated that expectation). Time to try scale it down if the operation - // is not already underway. - scalingDownNodes := scalingDownNodesByClass[podClass] - if _, ok := scalingDownNodes.Load(node.Name); !ok { // avoid spawning a thread if it appears work is in progress for this node already - go p.evaluateNodeScaleDown(podClass, node) - } - } else { - klog.Warningf("Pods are still running on node targeted for scale down: %v", node.Name) - } - } - } - - nodeNamesUnderActiveScaleDown := make([]string, 0) - scalingDownNodes := scalingDownNodesByClass[podClass] - scalingDownNodes.Range(func(key, value interface{}) bool { - nodeNamesUnderActiveScaleDown = append(nodeNamesUnderActiveScaleDown, fmt.Sprintf("%v", key)) - return true - }) - - if len(nodeNamesUnderActiveScaleDown) > 0 { - klog.Infof("Active attempts to scale down the following %v nodes are underway: %v", podClass, nodeNamesUnderActiveScaleDown) - } - - // Now we want to look at nodes that are schedulable / active. Taint / cordon these nodes to help - // a portion of them become idle and targets for scale down. - - // find all nodes that are relevant to this workload class and at least x minutes old - workloadNodes, err := p.getWorkloadNodesInAvoidanceOrder(podClass) - if err != nil { - klog.Errorf("Error finding avoidance workload nodes for scale down assessment of podClass %v: %v", podClass, err) - return - } - - if len(workloadNodes) == 0 { - // There is nothing to consider scaling down at present - return - } - - avoidanceNodes := make([]*corev1.Node, 0) - maxAvoidanceTargets := int(math.Ceil(float64(len(workloadNodes)) / 4)) // find appox 25% of nodes - avoidanceInfo := make([]string, 0) - - for _, node := range workloadNodes { - - if len(avoidanceNodes) >= maxAvoidanceTargets { - // Allow any remaining node to be scheduled if it is beyond our - // maximum target count. - err := p.setNodeAvoidanceState(node, podClass, TaintEffectNone) - if err != nil { - klog.Errorf("Unable to turn off avoidance for node %v: %#v", node.Name, err) - } - } else { - // Otherwise, we want to encourage pods away from this node. - pods, err := p.getPodsUsingNode(node.Name, true, 0) - if err != nil { - klog.Errorf("Unable to check pod count during class scale down eval for node %v: %#v", node.Name, err) - continue - } - - avoidanceNodes = append(avoidanceNodes, node) - activeAvoidanceEffect := p.getNodeAvoidanceState(node) - - if len(pods) == 0 { - // Only set NoSchedule if the instance is NOT spot.io. spot.io will handle all scale down - // for its nodes, so we don't taint them to encourage our own scale down logic. - if _, ok := node.Labels["spot-io"]; !ok { - // This is a ready / schedulable / non-spotio node with no pods. Set it up for scale down on the - // next call of this method. - err := p.setNodeAvoidanceState(node, podClass, corev1.TaintEffectNoSchedule) - if err != nil { - klog.Errorf("Unable to turn on NoSchedule avoidance for node %v: %#v", node.Name, err) - } else { - activeAvoidanceEffect = corev1.TaintEffectNoSchedule - } - } - } else { - // The node is the in top 25% of nodes close to being able to scale down. Encourage pods - // not to land on it unless necessary. We do this even for spot.io nodes to make it easier - // for the service to find empty scale down candidates. - err := p.setNodeAvoidanceState(node, podClass, corev1.TaintEffectPreferNoSchedule) - if err != nil { - klog.Errorf("Unable to turn on PreferNoSchedule avoidance for node %v: %#v", node.Name, err) - } else { - activeAvoidanceEffect = corev1.TaintEffectPreferNoSchedule - } - } - - avoidanceInfo = append(avoidanceInfo, fmt.Sprintf("%v;pods=%v;avoidance=%v", node.Name, len(pods), activeAvoidanceEffect)) - } - } - - klog.Infof("Avoidance info for podClass %v ; avoiding: %v", podClass, avoidanceInfo) -} - -func (p *Prioritization) getWorkloadNodesInAvoidanceOrder(podClass PodClass) ([]*corev1.Node, error) { - // find all nodes that are relevant to this workload class and have been around at least x minutes. - workloadNodes, err := p.getWorkloadNodes(podClass, true, 15*time.Minute) - - if err != nil { - return nil, fmt.Errorf("unable to find workload nodes for %v: %w", podClass, err) - } - - if len(workloadNodes) <= 1 { - // Nothing to put in order. There is either 1 or zero nodes to avoid. - return workloadNodes, nil - } - - cachedPodCount := make(map[string]int) // maps node name to running pod count - getCachedPodCount := func(node *corev1.Node) int { - nodeName := node.Name - if val, ok := cachedPodCount[nodeName]; ok { - return val - } - - // For the purposes of node avoidance, we only want to look at pods that are - // actively running (activeWithin 0s). - pods, err := p.getPodsUsingNode(nodeName, true, 0) - if err != nil { - klog.Errorf("Unable to get pod count for node: %v: %v", nodeName, err) - return 255 - } - - classedPodCount := len(pods) - - if spot_io_lifecycle, ok := node.Labels["spotinst.io/node-lifecycle"]; ok && spot_io_lifecycle == "spot" { - // Ensure spot instances always sort AFTER on demand instances; i.e. favor - // eliminating the more expensive on demand instance. - classedPodCount += 100 - } - - cachedPodCount[nodeName] = classedPodCount - return classedPodCount - } - - // Sort first by podCount then by oldest. The goal is to always be pseuedo-draining the node - // with the fewest pods which is at least 15 minutes old. Sorting by oldest helps make this - // search deterministic -- we want to report the same node consistently unless there is a node - // with fewer pods. - sort.Slice(workloadNodes, func(i, j int) bool { - nodeI := workloadNodes[i] - podsI := getCachedPodCount(nodeI) - nodeJ := workloadNodes[j] - podsJ := getCachedPodCount(nodeJ) - if podsI < podsJ { - return true - } else if podsI == podsJ { - return workloadNodes[i].CreationTimestamp.Time.Before(workloadNodes[j].CreationTimestamp.Time) - } else { - return false - } - }) - - return workloadNodes, nil -} - -func (p *Prioritization) findNodesToPreclude(podClass PodClass) ([]*corev1.Node, error) { - nodeAvoidanceLock.Lock() - defer nodeAvoidanceLock.Unlock() - - workloadNodes, err := p.getWorkloadNodesInAvoidanceOrder(podClass) - - if err != nil { - return nil, fmt.Errorf("unable to get sorted workload nodes for %v: %w", podClass, err) - } - - if len(workloadNodes) <= 1 { - // A pod is about to be scheduled, there is no reason to try to avoid nodes - // if there is only 1 or 0 to consider (there may also be young nodes, - // but we ignore those for the purposes of avoidance). - return nil, nil - } - - precludeNodes := make([]*corev1.Node, 0) - - // this is the most likely node to be scaled down next. - // don't let pods schedule in order to help our scale - // down loop eliminate it. - precludeNodes = append(precludeNodes, workloadNodes[0]) - - return precludeNodes, nil -} - -func (p *Prioritization) getMachinePhase(machineNamespace string, machineName string) (machinePhase string, machineExists bool, machineObj *unstructured.Unstructured, err error) { - machineClient := p.dynamicClient.Resource(machineResource).Namespace(machineNamespace) - - machineObj, err = machineClient.Get(p.context, machineName, metav1.GetOptions{}) - if err != nil { - if kerrors.IsNotFound(err) { - return "", false, nil, nil - } - return "", true, nil, fmt.Errorf("unable to get machine for scale down machine %v: %#w", machineName, err) - } - - machinePhase, found, err := unstructured.NestedString(machineObj.UnstructuredContent(), "status", "phase") - if !found || err != nil { - return "", true, machineObj, fmt.Errorf("could not get machine phase machine %v: %#w", machineName, err) - } - - machinePhase = strings.ToLower(machinePhase) - return machinePhase, true, machineObj, nil -} - -// scaleDown should be called by only one thread at a time. It assesses a node which has been staged for -// safe scale down (e.g. is running with the NoSchedule taint). Final checks are performed. -func (p *Prioritization) scaleDown(podClass PodClass, node *corev1.Node) (machineSetNamespace string, machineSetName string, machineName string, err error) { - if _, ok := node.Labels[CiWorkloadLabelName]; !ok { - // Just a sanity check - return "", "", "", fmt.Errorf("will not scale down non-ci-workload node") - } - - machineKey, ok := node.Annotations[NodeMachineAnnotationKey] - if !ok { - return "", "", "", fmt.Errorf("could not find machine annotation associated with node: %v", node.Name) - } - components := strings.Split(machineKey, "/") - machineSetNamespace = components[0] - machineSetClient := p.dynamicClient.Resource(machineSetResource).Namespace(machineSetNamespace) - machineClient := p.dynamicClient.Resource(machineResource).Namespace(machineSetNamespace) - - machineName = components[1] - - _, machineExists, _, err := p.getMachinePhase(machineSetNamespace, machineName) - - if !machineExists { - return machineSetNamespace, "", machineName, nil - } - - if err != nil { - return machineSetNamespace, "", machineName, fmt.Errorf("error checking machine phase %v / node %v: %w", machineName, node.Name, err) - } - - for { - // Wait until terminated pods are X minutes old so that prow / ci-operator have a change to check final status - // extract logs / etc (they poll). - pods, err := p.getPodsUsingNode(node.Name, true, 5*time.Minute) - if err != nil { - klog.Errorf("Unable to query for pod age requirement. Encountered error: %v", err) - break - } - if len(pods) == 0 { - break - } - klog.Infof("Waiting for all terminated pods on machine %v / node %v to have been so for several minutes' %v remaining", machineName, node.Name, len(pods)) - time.Sleep(1 * time.Minute) - } - - _, machineExists, machineObj, err := p.getMachinePhase(machineSetNamespace, machineName) - - if !machineExists { - return machineSetNamespace, "", machineName, nil - } - - if err != nil { - return machineSetNamespace, "", machineName, fmt.Errorf("error checking machine phase %v / node %v: %w", machineName, node.Name, err) - } - - machineMetadata, found, err := unstructured.NestedMap(machineObj.UnstructuredContent(), "metadata") - if !found || err != nil { - return machineSetNamespace, "", machineName, fmt.Errorf("could not get machine metadata for node %v / machine %v: %#w", node.Name, machineName, err) - } - - machineOwnerReferencesInterface, ok := machineMetadata["ownerReferences"] - if !ok { - return machineSetNamespace, "", machineName, fmt.Errorf("could not find machineset ownerReferences associated with machine: %v node: %v", machineName, node.Name) - } - - machineOwnerReferences := machineOwnerReferencesInterface.([]interface{}) - - for _, ownerInterface := range machineOwnerReferences { - owner := ownerInterface.(map[string]interface{}) - ownerKind := owner["kind"].(string) - if ownerKind == "MachineSet" { - machineSetName = owner["name"].(string) - } - } - - if len(machineSetName) == 0 { - return machineSetNamespace, "", machineName, fmt.Errorf("unable to find machineset name in machine owner references: %v node: %v", machineName, node.Name) - } - - _, err = machineSetClient.Get(p.context, machineSetName, metav1.GetOptions{}) - if err != nil { - return machineSetNamespace, machineSetName, machineName, fmt.Errorf("unable to get machineset %v: %#w", machineSetName, err) - } - - // setting this Taint is the point of no return -- if successful, we will try to scale down indefinitely. - // This taint is set to work around a DNS bug where DNS pods need time to gracefully shutdown before a - // drain operation. Draining without a graceful termination period causes brief outages in DNS. - // https://issues.redhat.com/browse/OCPBUGS-488 is intended to fix this behavior. - err = p.setNoExecuteTaint(node.Name, podClass) - if err != nil { - return machineSetNamespace, machineSetName, machineName, fmt.Errorf("unable to set NoExecute node %v: %#w", node.Name, err) - } - - klog.Infof("Sleeping to allow graceful DNS pod termination on %v / %v", machineName, node.Name) - time.Sleep(40 * time.Second) - - attempt := 0 - for { - if attempt > 0 { - time.Sleep(10 * time.Second) - } - - klog.Infof("Setting machine deletion annotation on machine %v for node %v [attempt=%v]", machineName, node.Name, attempt) - deletionAnnotationsPatch := []interface{}{ - map[string]interface{}{ - "op": "add", - "path": "/metadata/annotations/" + strings.ReplaceAll(MachineDeleteAnnotationKey, "/", "~1"), - "value": "true", - }, - map[string]interface{}{ - "op": "add", - "path": "/metadata/annotations/" + strings.ReplaceAll(OldMachineDeleteAnnotationKey, "/", "~1"), - "value": "true", - }, - } - - deletionPayload, err := json.Marshal(deletionAnnotationsPatch) - if err != nil { - klog.Errorf("Unable to marshal machine %v annotation deletion patch: %#v", machineName, err) - continue - } - - _, err = machineClient.Patch(p.context, machineName, types.JSONPatchType, deletionPayload, metav1.PatchOptions{}) - if err != nil { - if kerrors.IsNotFound(err) { - klog.Warningf("Machine %v has disappeared -- canceling scaledown", machineName) - return machineSetNamespace, machineSetName, machineName, nil - } - klog.Errorf("Unable to apply machine %v annotation %v deletion patch: %#v", machineName, MachineDeleteAnnotationKey, err) - continue - } - - break - } - - // We will now interact with the machineset for this pod class. Hold a lock until we successfully - // get rid of this machine or initiate its deletion. - nodeClassScaleDownLock[podClass].Lock() - defer nodeClassScaleDownLock[podClass].Unlock() - - attempt = 0 - for { - if attempt > 0 { - time.Sleep(10 * time.Second) - } - - ms, err := machineSetClient.Get(p.context, machineSetName, metav1.GetOptions{}) - if err != nil { - if kerrors.IsNotFound(err) { - klog.Errorf("Machineset %v has disappeared -- canceling scaledown", machineSetName) - return machineSetNamespace, machineSetName, machineName, nil - } - klog.Errorf("Unable to get machineset %v: %#v", machineSetName, err) - continue - } - - klog.Infof("Trying to scale down machineset %v in order to eliminate machine %v / node %v [attempt %v]", machineSetName, machineName, node.Name, attempt) - attempt++ - - replicas, found, err := unstructured.NestedInt64(ms.UnstructuredContent(), "spec", "replicas") - if err != nil || !found { - klog.Errorf("unable to get current replicas in machineset %v: %#v", machineSetName, err) - continue - } - - // When replicas is reduced, the machine should through different phases (deleting / shutting down). Don't - // interact with machineset while the machine is not in the running state -- a previous scan may have already - // decremented the replica count. This check should prevent it from happening again while the - // machine shuts down. - - machinePhase, machineExists, _, err := p.getMachinePhase(machineSetNamespace, machineName) - - if err != nil { - klog.Errorf("Error trying to determine machine phase %v / node %v: %v", machineName, node.Name, err) - continue - } - - if machinePhase == "deleting" { - // This is treated as a successful scale down - klog.Infof("Machine is in deleting state %v / node %v", machineName, node.Name) - return machineSetNamespace, machineSetName, machineName, nil - } - - if !machineExists { - // This is also treated as a successful scale down - klog.Infof("Machine %v no longer exists according to API / node %v", machineName, node.Name) - return machineSetNamespace, machineSetName, machineName, nil - } - - if machinePhase != "running" { - klog.Infof("Waiting until machine phase is running or machine is deleted; machine %v / node %v is in phase %v", machineName, node.Name, machinePhase) - continue - } - - // There's no indication that the machine is scaling down. Commit to decrementing the replica count. - replicas-- - - if replicas < 0 { - // This is unexpected -- something has changed replicas and we don't think it was us. - klog.Errorf("computed replicas < 0 for machineset %v ; aborting this scale down due to race", machineSetName) - return machineSetNamespace, machineSetName, machineName, nil - } - - klog.Infof("Scaling down machineset %v to %v replicas in order to eliminate machine %v / node %v", machineSetName, replicas, machineName, node.Name) - - scaleDownPatch := []interface{}{ - map[string]interface{}{ - "op": "replace", - "path": "/spec/replicas", - "value": replicas, - }, - } - - scaleDownPayload, err := json.Marshal(scaleDownPatch) - if err != nil { - klog.Errorf("unable to marshal machineset scale down patch: %#v", err) - continue - } - - _, err = machineSetClient.Patch(p.context, machineSetName, types.JSONPatchType, scaleDownPayload, metav1.PatchOptions{}) - if err != nil { - klog.Errorf("unable to patch machineset %v with scale down patch: %#v", machineSetName, err) - continue - } - - // The machine was annotated for deletion and a corresponding decrement to the machineset was applied. - // This method is done. Returning from this method releases a lock which allows other machines in this - // class to scale down. Waiting too long in this method means that the number of cordoned machines may - // grow faster than they can be scaled done. - return machineSetNamespace, machineSetName, machineName, nil - } -} - -const TaintEffectNone corev1.TaintEffect = "None" - -func (p *Prioritization) getNodeAvoidanceState(node *corev1.Node) corev1.TaintEffect { - avoidanceState := TaintEffectNone - - for _, taint := range node.Spec.Taints { - if taint.Key == CiWorkloadPreferNoScheduleTaintName { - avoidanceState = corev1.TaintEffectPreferNoSchedule - break - } - } - - if node.Spec.Unschedulable { - avoidanceState = corev1.TaintEffectNoSchedule - } - - return avoidanceState -} - -func (p *Prioritization) setNodeCordoned(node *corev1.Node, cordoned bool) error { - if node.Spec.Unschedulable == cordoned { - // we are already at the desired state - return nil - } - - cordonPatch := []interface{}{ - map[string]interface{}{ - "op": "replace", - "path": "/spec/unschedulable", - "value": cordoned, - }, - } - - payloadBytes, _ := json.Marshal(cordonPatch) - _, err := p.k8sClientSet.CoreV1().Nodes().Patch(p.context, node.Name, types.JSONPatchType, payloadBytes, metav1.PatchOptions{}) - if err != nil { - return fmt.Errorf("failed to change cordoned state for node %v to %v: %#w", node.Name, cordoned, err) - } - - klog.Infof("Set node %v to cordoned=%v", node.Name, cordoned) - return nil -} - -func (p *Prioritization) setNodeAvoidanceState(node *corev1.Node, podClass PodClass, desiredEffect corev1.TaintEffect) error { - nodeTaints := node.Spec.Taints - if nodeTaints == nil { - nodeTaints = make([]corev1.Taint, 0) - } - - foundEffect := TaintEffectNone - - if node.Spec.Unschedulable { - foundEffect = corev1.TaintEffectNoSchedule - } - - if foundEffect == corev1.TaintEffectNoSchedule { - // Never uncordon nodes. This gets really complex if someone is manually cordoning nodes. - // Just avoid the complexity. - klog.Errorf("Attempt to new avoidance state %v for node %v targeted for scale down", desiredEffect, node.Name) - return nil - } - - // We enforce NoSchedule avoidance with cordon. CiWorkloadPreferNoScheduleTaintName - // will be set to unless desiredEffect == TaintEffectNone - _ = p.setNodeCordoned(node, desiredEffect == corev1.TaintEffectNoSchedule) - - // PreferNoSchedule is implemented as a custom taint. Depending on - // caller's request, add or remove that taint. - foundPreferNoScheduleIndex := -1 - for i, taint := range nodeTaints { - if taint.Key == CiWorkloadPreferNoScheduleTaintName { - foundPreferNoScheduleIndex = i - if !node.Spec.Unschedulable { - foundEffect = corev1.TaintEffectPreferNoSchedule - } - } - } - - modified := false // whether there is reason to patch the node taints - - if foundPreferNoScheduleIndex == -1 && desiredEffect != TaintEffectNone { - // Both non-none avoidance levels should set the PreferNoSchedule taint. - nodeTaints = append(nodeTaints, corev1.Taint{ - Key: CiWorkloadPreferNoScheduleTaintName, - Value: fmt.Sprintf("%v", podClass), - Effect: corev1.TaintEffectPreferNoSchedule, - }) - modified = true - } - - if foundPreferNoScheduleIndex >= 0 && desiredEffect == TaintEffectNone { - // remove our taint from the list - nodeTaints = append(nodeTaints[:foundPreferNoScheduleIndex], nodeTaints[foundPreferNoScheduleIndex+1:]...) - modified = true - } - - if modified { - taintMap := map[string][]corev1.Taint{ - "taints": nodeTaints, - } - unstructuredTaints, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&taintMap) - if err != nil { - return fmt.Errorf("error decoding modified taints to unstructured data: %w", err) - } - - patch := map[string]interface{}{ - "op": "add", - "path": "/spec/taints", - "value": unstructuredTaints["taints"], - } - - patchEntries := make([]map[string]interface{}, 0) - patchEntries = append(patchEntries, patch) - - payloadBytes, _ := json.Marshal(patchEntries) - _, err = p.k8sClientSet.CoreV1().Nodes().Patch(p.context, node.Name, types.JSONPatchType, payloadBytes, metav1.PatchOptions{}) - if err != nil { - return fmt.Errorf("failed to change avoidance taint (existing effect [%v]) to %v for node %v: %#w", foundEffect, desiredEffect, node.Name, err) - } - } - - if desiredEffect != foundEffect { - klog.Infof("Avoidance taint state changed (old effect [%v]) to %v for node: %v", foundEffect, desiredEffect, node.Name) - } - - return nil -} - -func (p *Prioritization) setNoExecuteTaint(nodeName string, podClass PodClass) error { - nodeObj, exists, err := nodesInformer.GetIndexer().GetByKey(nodeName) - - if err != nil { - return fmt.Errorf("error getting node to set NoExecute: %w", err) - } - - if !exists { - return fmt.Errorf("node targeted for NoExecute no longer exists") - } - - node := nodeObj.(*corev1.Node) - nodeTaints := node.Spec.Taints - if nodeTaints == nil { - nodeTaints = make([]corev1.Taint, 0) - } - - // See if NoExecute is already set - for _, taint := range nodeTaints { - if taint.Key == CiWorkloadPreferNoExecuteTaintName { - // Nothing to do if the taint exists - return nil - } - } - - nodeTaints = append(nodeTaints, corev1.Taint{ - Key: CiWorkloadPreferNoExecuteTaintName, - Value: fmt.Sprintf("%v", podClass), - Effect: corev1.TaintEffectNoExecute, - }) - - taintMap := map[string][]corev1.Taint{ - "taints": nodeTaints, - } - unstructuredTaints, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&taintMap) - if err != nil { - return fmt.Errorf("error decoding modified taints to unstructured data: %w", err) - } - - patch := map[string]interface{}{ - "op": "add", - "path": "/spec/taints", - "value": unstructuredTaints["taints"], - } - - patchEntries := make([]map[string]interface{}, 0) - patchEntries = append(patchEntries, patch) - - payloadBytes, _ := json.Marshal(patchEntries) - _, err = p.k8sClientSet.CoreV1().Nodes().Patch(p.context, node.Name, types.JSONPatchType, payloadBytes, metav1.PatchOptions{}) - if err != nil { - return fmt.Errorf("failed to change set NoExecute taint for node %v: %#w", node.Name, err) - } - - return nil -} - -func (p *Prioritization) findHostnamesToPreclude(podClass PodClass) []string { - hostnamesToPreclude := make([]string, 0) - nodesToPreclude, err := p.findNodesToPreclude(podClass) - if err != nil { - klog.Warningf("Error during node avoidance process: %#v", err) - } else { - for _, nodeToPreclude := range nodesToPreclude { - hostname := p.getNodeHostname(nodeToPreclude) - if len(hostname) == 0 { - klog.Errorf("Unable to get %v label for node: %v", KubernetesHostnameLabelName, nodeToPreclude.Name) - continue - } - hostnamesToPreclude = append(hostnamesToPreclude, hostname) - } - } - klog.Infof("Precluding hostnames for podClass %v: %v", podClass, hostnamesToPreclude) - return hostnamesToPreclude -} diff --git a/cmd/ci-scheduling-webhook/readme.md b/cmd/ci-scheduling-webhook/readme.md deleted file mode 100644 index b6b785f8b86..00000000000 --- a/cmd/ci-scheduling-webhook/readme.md +++ /dev/null @@ -1,84 +0,0 @@ - -# Why do we need this? - -## Workload segmentation -We have several workload types: builds, short-running tests, long-running tests, and prowjobs. They differ in duration and resource shape, so we segment them. - -### RuntimeClass and kubelet overhead -Builds stress the container runtime in ways the pod autoscaler does not fully model. You can address reserve CPU with `KubeletConfig` on self-managed OpenShift; we also use **`RuntimeClass`** so admission applies **overhead** CPU and memory per pool. That keeps build and test pods on different overhead profiles (for example, extra CPU headroom for builds) without relying on a single global kubelet recipe. - -Without enough CPU, the runtime can surface errors such as `context deadline exceeded` and builds can fail. - -### IOPS, instance types, analysis -Segmentation lets us tune IOPS and machine types per class and reason about capacity without mixing unrelated workloads on the same node pool. - -## Kubernetes scheduling and the cluster autoscaler -By default the scheduler tends to **spread** pods (least-allocated style). Our CI load is often bursty: many pods, then scale-out, then relatively sparse placement—so spread scheduling interacts badly with **machine** scale-out/scale-in. - -The **cluster autoscaler** scales **MachineSets up** when there are **unschedulable** pods that match a pool. It is conservative about evicting workloads that are not clearly safe to move; many CI pods are not owned by ReplicaSets in the usual way, which can leave nodes busy longer than we want. - -**This webhook does not replace the cluster autoscaler for scale-up.** It coordinates **scale-down** and placement pressure so we can reclaim nodes on a useful timeline (see [Scale-down control loop](#scale-down-control-loop)). In operation, the **ci-scheduling-webhook** service account may **patch `MachineSet` and node objects** as part of that path—distinct from the machine-controller’s behavior when **you** change `MachineSet` replicas. - -Tight **scheduling** vs **kubelet** resource accounting was a recurring issue in older Kubernetes releases ([kubernetes#106884](https://github.com/kubernetes/kubernetes/issues/106884#issuecomment-1005074672)). On **current** OpenShift/Kubernetes versions, treat that history as **background**: validate tight packing under your real jobs instead of assuming the same failure mode as pre–1.23-era clusters. - -## Cluster autoscaler, DNS (`dns-default`), and `enable-ds-eviction` -**DaemonSet eviction for DNS** is a **cluster-autoscaler** concern when it **removes or drains nodes**. OpenShift DNS pods can use annotations such as **`cluster-autoscaler.kubernetes.io/enable-ds-eviction`** so eviction goes through paths that honor graceful shutdown. That is **not** something this **admission webhook** applies to arbitrary pods—it is part of how **platform DNS / DaemonSets** interact with **node** lifecycle and CA. - -**This webhook is not the right lever to “make DNS respect `enable-ds-eviction`”**—that belongs to DNS/operator/CA configuration for the relevant **MachineSet** and DaemonSet, not to mutating CI workload pods here. - -# Design - -## Workload classes -Classes include tests, builds, longtests, and prowjobs. Each class has its own **MachineSet** (and usually **MachineAutoscaler** for scale-out). The webhook classifies pods and applies a **`RuntimeClass`** so they land on the right pool. - -## Cluster autoscaler scales up -The cluster autoscaler increases **MachineSet** replicas when **Pending** pods need capacity—normal behavior. - -## Scale-down control loop -Per workload class, the controller runs a **reconciliation loop about once per minute** (`pollNodeClassForScaleDown` in `prioritization.go`: initial evaluation, then `time.Tick(time.Minute)`). Each loop: - -1. **Cordoned candidates:** For nodes already in **NoSchedule** avoidance (cordoned), only consider nodes **at least ~15 minutes old** so initialization cordons are not mistaken for scale-down targets. If the node still has **no** class pods (DaemonSets ignored), spawn work to **actually scale down** the Machine / MachineSet; otherwise wait. -2. **Avoidance set:** Among schedulable workload nodes, mark roughly **the top 25%** (`ceil(n/4)`) for **PreferNoSchedule**-style avoidance so new CI work tends to land elsewhere; clear avoidance on nodes past that budget. -3. **When a node is empty** under the avoidance / cordon policy, a later pass **cordons** (NoSchedule) and, once still empty, triggers **machine removal** via the scale-down path (with checks that the MachineSet is reconciled). - -So “useful timeline” is **minute-scale loops** plus **15-minute minimum node age** for one part of the decision—not real-time per pod. The same path adjusts **nodes** and **MachineSet** objects so capacity is not left stranded longer than necessary. - -## Avoidance states -States include **none**, **PreferNoSchedule** (taint), and **NoSchedule** (cordon). When a node has no running class pods (DaemonSets ignored for this check), the webhook can cordon and eventually trigger removal of that machine. - -## Pod node affinity -Incoming pods can be given affinity that **excludes** a specific node the webhook is trying to empty, so scheduling pressure favors reclaiming that node. - -# Deploying -1. Create **MachineSet** and **MachineAutoscaler** per class; they are **cluster- and cloud-specific**. Copy from existing build-farm YAML in **`openshift/release`** (for example under `clusters/build-clusters/buildNN/ci-scheduling-webhook/`). Set **`minReplicas` / `maxReplicas`** per pool from those examples—there is **no** single fixed maximum (older docs mentioning a flat max were wrong). -2. Apply `cmd/ci-scheduling-webhook/res/admin.yaml`. -3. Apply `cmd/ci-scheduling-webhook/res/rbac.yaml`. -4. Apply `cmd/ci-scheduling-webhook/res/deployment.yaml`. -5. Apply `cmd/ci-scheduling-webhook/res/dns.yaml` so cluster DNS DaemonSets can schedule on **tainted** worker nodes where required. -6. Verify the deployment in namespace `ci-scheduling-webhook`. -7. Confirm **MachineSets** have at least one node. -8. Apply `cmd/ci-scheduling-webhook/res/webhook.yaml`. - -# Hack - -## Manual image builds -```shell -[ci-tools]$ CGO_ENABLED=0 go build -ldflags="-extldflags=-static" github.com/openshift/ci-tools/cmd/ci-scheduling-webhook -[ci-tools]$ podman build -t quay.io/jupierce/ci-scheduling-webhook:latest -f images/ci-scheduling-webhook/Dockerfile . -[ci-tools]$ podman push quay.io/jupierce/ci-scheduling-webhook:latest -``` - -## Local test -```shell -[ci-tools]$ export KUBECONFIG=~/.kube/config -[ci-tools]$ go run github.com/openshift/ci-tools/cmd/ci-scheduling-webhook --as system:admin --port 8443 -[ci-tools]$ cmd/ci-scheduling-webhook/testing/post-pods.sh -``` - -## Pushing to prod -```shell -[ci-tools]$ podman build -t quay.io/openshift/ci:ci_ci-scheduling-webhook_latest -f images/ci-scheduling-webhook/Dockerfile . -[ci-tools]$ podman push quay.io/openshift/ci:ci_ci-scheduling-webhook_latest -``` - -Restart or roll `ci-scheduling-webhook` pods on build farms so they pick up the new image. diff --git a/cmd/determinize-peribolos/main.go b/cmd/determinize-peribolos/main.go deleted file mode 100644 index 083a5625cbd..00000000000 --- a/cmd/determinize-peribolos/main.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import ( - "errors" - "flag" - "os" - - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/config/org" - "sigs.k8s.io/prow/pkg/logrusutil" - "sigs.k8s.io/yaml" - - "github.com/openshift/ci-tools/pkg/util/gzip" -) - -type options struct { - config string -} - -func parseOptions() options { - var o options - if err := o.parseArgs(flag.CommandLine, os.Args[1:]); err != nil { - logrus.Fatalf("Invalid flags: %v", err) - } - return o -} - -func (o *options) parseArgs(flags *flag.FlagSet, args []string) error { - flags.StringVar(&o.config, "config-path", "", "Path to org config.yaml") - if err := flags.Parse(args); err != nil { - return err - } - if o.config == "" { - return errors.New("--config-path required") - } - return nil -} - -func main() { - logrusutil.ComponentInit() - - o := parseOptions() - - raw, err := gzip.ReadFileMaybeGZIP(o.config) - if err != nil { - logrus.WithError(err).Fatal("Could not read --config-path file") - } - - var cfg org.FullConfig - if err := yaml.Unmarshal(raw, &cfg); err != nil { - logrus.WithError(err).Fatal("Failed to load configuration") - } - - out, err := yaml.Marshal(cfg) - if err != nil { - logrus.WithError(err).Fatal("Failed to marshal output.") - } - - if err := os.WriteFile(o.config, out, 0666); err != nil { - logrus.WithError(err).Fatal("Failed to write output.") - } - - logrus.Info("Finished formatting configuration.") -} diff --git a/cmd/gpu-scheduling-webhook/README.md b/cmd/gpu-scheduling-webhook/README.md deleted file mode 100644 index 18e50714a4f..00000000000 --- a/cmd/gpu-scheduling-webhook/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# gpu-scheduling-webhook - -## Motivation -Our clusters host some nodes that feature an Nvidia GPU. They are expensive to run workload on so -by using this mutating webhook we ensure that only the pods requesting a GPU actually run on those -nodes, leaving out everything else. - -## How it works -A node that features an Nvida GPU holds the following taint: - -```yaml -taints: -- effect: NoSchedule - key: nvidia.com/gpu - value: "true" -``` - -The webhook inspects a pod's container requests, both form the init containers and regular ones, and apply -this toleration: - -```yaml -tolerations: -- key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule -``` - -when it finds either such a request: - -```yaml -requests: - nvidia.com/gpu: -``` - -or the following limit: - -```yaml -limits: - nvidia.com/gpu: -``` diff --git a/cmd/gpu-scheduling-webhook/main.go b/cmd/gpu-scheduling-webhook/main.go deleted file mode 100644 index d7a91eab7b7..00000000000 --- a/cmd/gpu-scheduling-webhook/main.go +++ /dev/null @@ -1,240 +0,0 @@ -package main - -import ( - "context" - "fmt" - "net/http" - - "github.com/bombsimon/logrusr/v3" - "github.com/go-logr/logr" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/manager/signals" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - "github.com/openshift/ci-tools/pkg/api" -) - -var ( - nvidiaGPUToleration = corev1.Toleration{ - Key: api.NvidiaGPUResource, - Operator: corev1.TolerationOpEqual, - Value: "true", - Effect: corev1.TaintEffectNoSchedule, - } - - KVMVirtToleration = corev1.Toleration{ - Key: "ci-workload", - Operator: corev1.TolerationOpEqual, - Value: "virt-launcher", - Effect: corev1.TaintEffectNoSchedule, - } - - opts = options{} - - rootCmd = &cobra.Command{ - Use: "gpu-scheduling-webhook", - Short: "Controls where pods will be scheduled when they request a GPU", - Long: `Controls where pods will be scheduled when they request a GPU. - -Example: -$ gpu-scheduling-webhook --cert-dir= --port=443`, - RunE: RunE, - } -) - -func init() { - rootCmd.Flags().StringVar(&opts.certDir, "cert-dir", "", "A folder holding the server private key and and certicate for TLS") - rootCmd.Flags().StringVar(&opts.healthProbeAddr, "health-probe-addr", ":8081", "Health probe binding address :. Default to :8081") - rootCmd.Flags().IntVar(&opts.port, "port", 0, "Port the server will listen on") -} - -func setupLogger() logr.Logger { - innerLogger := logrus.New() - innerLogger.Formatter = &logrus.JSONFormatter{} - log.SetLogger(logrusr.New(innerLogger)) - return log.Log -} - -type options struct { - certDir string - port int - healthProbeAddr string -} - -type gpuTolerator struct{} - -func (*gpuTolerator) Default(ctx context.Context, obj runtime.Object) error { - logger := log.FromContext(ctx) - - pod, ok := obj.(*corev1.Pod) - if !ok { - return fmt.Errorf("expected a Pod but got a %T", obj) - } - - logger = logger.WithValues("pod", fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) - - // Check if a GPU required on resources from containers and init containers. - var gpuNeeded bool - gpuNeeded = hasNvidaGPURequest(logger, pod.Spec.InitContainers) - - var KVMVirtNeeded bool - KVMVirtNeeded = hasKVMVirtRequest(logger, pod.Spec.InitContainers) - - if !gpuNeeded { - gpuNeeded = hasNvidaGPURequest(logger, pod.Spec.Containers) - } - - if gpuNeeded { - addToleration(logger, pod) - } - - if !KVMVirtNeeded { - KVMVirtNeeded = hasKVMVirtRequest(logger, pod.Spec.Containers) - } - - if KVMVirtNeeded { - addKVMVirtToleration(logger, pod) - } - - return nil -} - -func hasNvidaGPURequest(logger logr.Logger, containers []corev1.Container) bool { - for i := range containers { - c := &containers[i] - if needNvidiaGPU(c.Resources) { - logger.Info("Request Nvidia GPU", "container", c.Name) - return true - } - } - return false -} - -func hasKVMVirtRequest(logger logr.Logger, containers []corev1.Container) bool { - for i := range containers { - c := &containers[i] - if needKVMVirt(c.Resources) { - logger.Info("Request KVM Virt", "container", c.Name) - return true - } - } - return false -} - -func needKVMVirt(requirement corev1.ResourceRequirements) bool { - _, requestExists := requirement.Requests["devices.kubevirt.io/kvm"] - _, limitExists := requirement.Limits["devices.kubevirt.io/kvm"] - return requestExists || limitExists -} - -// Allow a pod to be scheduled on the KVM Virt featured node by adding a toleration. -// Do nothing if the toleration has already been added. -func addKVMVirtToleration(logger logr.Logger, pod *corev1.Pod) { - var tolerationExists bool - for _, t := range pod.Spec.Tolerations { - if t == KVMVirtToleration { - tolerationExists = true - break - } - } - - if !tolerationExists { - pod.Spec.Tolerations = append(pod.Spec.Tolerations, KVMVirtToleration) - logger.Info("Add KVM Virt toleration") - } else { - logger.Info("KVM Virt toleration exists already") - } -} - -// Allow a pod to be scheduled on the GPU featured node by adding a toleration. -// Do nothing if the toleration has already been added. -func addToleration(logger logr.Logger, pod *corev1.Pod) { - var tolerationExists bool - for _, t := range pod.Spec.Tolerations { - if t == nvidiaGPUToleration { - tolerationExists = true - break - } - } - - if !tolerationExists { - pod.Spec.Tolerations = append(pod.Spec.Tolerations, nvidiaGPUToleration) - logger.Info("Add toleration") - } else { - logger.Info("Toleration exists already") - } -} - -func needNvidiaGPU(requirement corev1.ResourceRequirements) bool { - _, requestExists := requirement.Requests[api.NvidiaGPUResource] - _, limitExists := requirement.Limits[api.NvidiaGPUResource] - return requestExists || limitExists -} - -func startWebhookServer(ctx context.Context, logger *logr.Logger, o *options, cfg *rest.Config) error { - logger.Info("Setting up manager") - mgr, err := manager.New(cfg, manager.Options{ - HealthProbeBindAddress: o.healthProbeAddr, - WebhookServer: webhook.NewServer(webhook.Options{ - CertDir: o.certDir, - Port: o.port, - }), - }) - if err != nil { - logger.Error(err, "Unable to set up manager") - return err - } - - if err := mgr.AddHealthzCheck("healthz", func(req *http.Request) error { return nil }); err != nil { - logger.Error(err, "Unable to set up healthz endpoint") - return err - } - - if err := mgr.AddReadyzCheck("readyz", func(req *http.Request) error { return nil }); err != nil { - logger.Error(err, "Unable to set up readyz endpoint") - return err - } - - logger.WithValues("Addr", o.healthProbeAddr).Info("Serving healthiness probes") - - if err := builder.WebhookManagedBy(mgr). - For(&corev1.Pod{}). - WithDefaulter(&gpuTolerator{}). - Complete(); err != nil { - logger.Error(err, "Unable to build webhook") - return err - } - - logger.Info("Starting manager") - if err := mgr.Start(ctx); err != nil { - logger.Error(err, "Unable to start manager") - return err - } - - return nil -} - -func RunE(cmd *cobra.Command, args []string) error { - logger := setupLogger().WithName("gpu-scheduling") - logger.Info("Starting the webhook") - - cfg, err := config.GetConfig() - if err != nil { - return fmt.Errorf("get cluster config: %w", err) - } - - return startWebhookServer(cmd.Context(), &logger, &opts, cfg) -} - -func main() { - cobra.CheckErr(rootCmd.ExecuteContext(signals.SetupSignalHandler())) -} diff --git a/cmd/gpu-scheduling-webhook/main_test.go b/cmd/gpu-scheduling-webhook/main_test.go deleted file mode 100644 index 8b0e7474586..00000000000 --- a/cmd/gpu-scheduling-webhook/main_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package main - -import ( - "context" - "errors" - "testing" - - "github.com/google/go-cmp/cmp" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/openshift/ci-tools/pkg/api" -) - -func TestMutatePod(t *testing.T) { - for _, testCase := range []struct { - name string - pod runtime.Object - wantPod corev1.Pod - wantErr error - }{ - { - name: "Request a GPU therefore add toleration", - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "c1", - Command: []string{"cmd"}, - Image: "img", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - Limits: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - }, - }, - }, - }, - }, - wantPod: corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "c1", - Command: []string{"cmd"}, - Image: "img", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - Limits: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - }, - }, - }, - Tolerations: []corev1.Toleration{nvidiaGPUToleration}, - }, - }, - }, - { - name: "No GPU request a GPU, leave pod untouched", - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{Containers: []corev1.Container{{ - Name: "c1", - Command: []string{"cmd"}, - Image: "img", - }}}, - }, - wantPod: corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{Containers: []corev1.Container{{ - Name: "c1", - Command: []string{"cmd"}, - Image: "img", - }}}, - }, - }, - { - name: "Do not add the same toleration again", - pod: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "c1", - Command: []string{"cmd"}, - Image: "img", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - Limits: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - }, - }, - }, - Tolerations: []corev1.Toleration{nvidiaGPUToleration}, - }, - }, - wantPod: corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "c1", - Command: []string{"cmd"}, - Image: "img", - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - Limits: corev1.ResourceList{ - api.NvidiaGPUResource: resource.MustParse("1"), - }, - }, - }, - }, - Tolerations: []corev1.Toleration{nvidiaGPUToleration}, - }, - }, - }, - { - name: "Not a pod, return an error", - pod: &corev1.Node{}, - wantErr: errors.New("expected a Pod but got a *v1.Node"), - }, - } { - t.Run(testCase.name, func(t *testing.T) { - pgs := gpuTolerator{} - err := pgs.Default(context.TODO(), testCase.pod) - - if err != nil && testCase.wantErr == nil { - t.Fatalf("want err nil but got: %v", err) - } - if err == nil && testCase.wantErr != nil { - t.Fatalf("want err %v but nil", testCase.wantErr) - } - if err != nil && testCase.wantErr != nil { - if diff := cmp.Diff(testCase.wantErr.Error(), err.Error()); diff != "" { - t.Fatalf("unexpected error: %s", diff) - } - return - } - - pod, _ := testCase.pod.(*corev1.Pod) - if diff := cmp.Diff(testCase.wantPod, *pod); diff != "" { - t.Error(diff) - } - }) - } -} diff --git a/cmd/helpdesk-faq/main.go b/cmd/helpdesk-faq/main.go deleted file mode 100644 index de17248275d..00000000000 --- a/cmd/helpdesk-faq/main.go +++ /dev/null @@ -1,142 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "html/template" - "net/http" - "os" - "strconv" - "time" - - "github.com/sirupsen/logrus" - - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/prow/pkg/flagutil" - "sigs.k8s.io/prow/pkg/interrupts" - "sigs.k8s.io/prow/pkg/logrusutil" - - helpdeskfaq "github.com/openshift/ci-tools/pkg/helpdesk-faq" - "github.com/openshift/ci-tools/pkg/util" -) - -const ( - ci = "ci" -) - -type options struct { - logLevel string - port int - gracePeriod time.Duration - kubernetesOptions flagutil.KubernetesOptions -} - -type Page struct { - Data []helpdeskfaq.FaqItem `json:"data"` -} - -func gatherOptions() (options, error) { - o := options{} - fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - fs.StringVar(&o.logLevel, "log-level", "info", "Level at which to log output.") - fs.IntVar(&o.port, "port", 8080, "Port to run the server on") - o.kubernetesOptions.AddFlags(fs) - fs.DurationVar(&o.gracePeriod, "gracePeriod", time.Second*10, "Grace period for server shutdown") - if err := fs.Parse(os.Args[1:]); err != nil { - return o, fmt.Errorf("failed to parse flags: %w", err) - } - return o, nil -} - -func validateOptions(o options) error { - _, err := logrus.ParseLevel(o.logLevel) - if err != nil { - return fmt.Errorf("invalid --log-level: %w", err) - } - return o.kubernetesOptions.Validate(false) -} - -func router(client helpdeskfaq.FaqItemClient) *http.ServeMux { - handler := http.NewServeMux() - - handler.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - page := map[string]bool{"ok": true} - if err := json.NewEncoder(w).Encode(page); err != nil { - logrus.WithError(err).WithField("page", page).Error("failed to encode page") - } - }) - - handler.HandleFunc("/api/v1/faq-items", func(w http.ResponseWriter, r *http.Request) { - logrus.WithField("path", "/api/v1/faq-items").Info("serving") - - items, err := client.GetSerializedFAQItems() - if err != nil { - logrus.WithError(err).Fatal("unable to get helpdesk-faq items") - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - page := Page{} - for _, item := range items { - faqItem := &helpdeskfaq.FaqItem{} - if err := json.Unmarshal([]byte(item), faqItem); err != nil { - logrus.WithError(err).Fatal("unable to unmarshall faq item") - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - page.Data = append(page.Data, *faqItem) - } - - if callbackName := r.URL.Query().Get("callback"); callbackName != "" { - bytes, err := json.Marshal(page) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/javascript") - template.JSEscape(w, []byte(callbackName)) - if n, err := fmt.Fprintf(w, "(%s);", string(bytes)); err != nil { - logrus.WithError(err).WithField("n", n).Error("failed to write content") - } - } else { - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(page); err != nil { - logrus.WithError(err).WithField("page", page).Error("failed to encode page") - } - } - }) - - return handler -} - -func main() { - logrusutil.ComponentInit() - o, err := gatherOptions() - if err != nil { - logrus.WithError(err).Fatal("failed go gather options") - } - if err := validateOptions(o); err != nil { - logrus.WithError(err).Fatal("invalid options") - } - level, _ := logrus.ParseLevel(o.logLevel) - logrus.SetLevel(level) - - inClusterConfig, err := util.LoadClusterConfig() - if err != nil { - logrus.WithError(err).Fatal("Failed to load in-cluster config") - } - kubeClient, err := ctrlruntimeclient.New(inClusterConfig, ctrlruntimeclient.Options{}) - if err != nil { - logrus.WithError(err).Fatal("Failed to create client") - } - client := helpdeskfaq.NewCMClient(kubeClient, ci, logrus.WithField("client", "cm-client")) - server := &http.Server{ - Addr: ":" + strconv.Itoa(o.port), - Handler: router(&client), - } - interrupts.ListenAndServe(server, o.gracePeriod) - logrus.Debug("Server ready.") - interrupts.WaitForGracefulShutdown() -} diff --git a/cmd/pipeline-controller/config_data_provider.go b/cmd/pipeline-controller/config_data_provider.go deleted file mode 100644 index e709fa7d61e..00000000000 --- a/cmd/pipeline-controller/config_data_provider.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "sync" - "time" - - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/config" -) - -// RepoLister is a function that returns a list of "org/repo" strings that should be processed -type RepoLister func() []string - -type presubmitTests struct { - protected []config.Presubmit - alwaysRequired []config.Presubmit - conditionallyRequired []config.Presubmit - pipelineConditionallyRequired []config.Presubmit - pipelineSkipOnlyRequired []config.Presubmit -} - -type ConfigDataProvider struct { - configGetter config.Getter - repoLister RepoLister - updatedPresubmits map[string]presubmitTests - logger *logrus.Entry - m sync.Mutex -} - -func NewConfigDataProvider(configGetter config.Getter, repoLister RepoLister, logger *logrus.Entry) *ConfigDataProvider { - provider := &ConfigDataProvider{ - configGetter: configGetter, - repoLister: repoLister, - updatedPresubmits: make(map[string]presubmitTests), - logger: logger, - m: sync.Mutex{}, - } - // Initialize with first load - provider.gatherData() - return provider -} - -func (c *ConfigDataProvider) GetPresubmits(orgRepo string) presubmitTests { - c.m.Lock() - defer c.m.Unlock() - if presubmits, ok := c.updatedPresubmits[orgRepo]; ok { - return presubmits - } - return presubmitTests{} -} - -func (c *ConfigDataProvider) Run() { - for { - time.Sleep(10 * time.Minute) - // Always refresh job data to pick up added/removed tests - c.gatherData() - } -} - -func (c *ConfigDataProvider) gatherData() { - // Get the list of repositories from the pipeline controller configs (config + lgtm config) - orgRepos := c.repoLister() - c.gatherDataForRepos(orgRepos) -} - -// gatherDataForRepos processes the given list of repositories and updates the presubmits data -func (c *ConfigDataProvider) gatherDataForRepos(orgRepos []string) { - cfg := c.configGetter() - - updatedPresubmits := make(map[string]presubmitTests) - for _, orgRepo := range orgRepos { - // Skip if we've already processed this repo (avoid duplicates) - if _, exists := updatedPresubmits[orgRepo]; exists { - continue - } - presubmits := cfg.GetPresubmitsStatic(orgRepo) - - for _, p := range presubmits { - if !p.AlwaysRun && p.RunIfChanged == "" && p.SkipIfOnlyChanged == "" { - if val, ok := p.Annotations["pipeline_run_if_changed"]; ok && val != "" { - pre := updatedPresubmits[orgRepo] - pre.pipelineConditionallyRequired = append(pre.pipelineConditionallyRequired, p) - updatedPresubmits[orgRepo] = pre - continue - } - // Also check for pipeline_skip_if_only_changed annotation - if val, ok := p.Annotations["pipeline_skip_if_only_changed"]; ok && val != "" { - pre := updatedPresubmits[orgRepo] - pre.pipelineSkipOnlyRequired = append(pre.pipelineSkipOnlyRequired, p) - updatedPresubmits[orgRepo] = pre - continue - } - // Only categorize as protected if it doesn't have pipeline annotations - if !p.Optional { - if _, hasPipelineRun := p.Annotations["pipeline_run_if_changed"]; !hasPipelineRun { - if _, hasPipelineSkip := p.Annotations["pipeline_skip_if_only_changed"]; !hasPipelineSkip { - pre := updatedPresubmits[orgRepo] - pre.protected = append(pre.protected, p) - updatedPresubmits[orgRepo] = pre - } - } - } - } - if !p.Optional && p.AlwaysRun { - pre := updatedPresubmits[orgRepo] - pre.alwaysRequired = append(pre.alwaysRequired, p) - updatedPresubmits[orgRepo] = pre - continue - } - if !p.Optional && (p.RunIfChanged != "" || p.SkipIfOnlyChanged != "") { - pre := updatedPresubmits[orgRepo] - pre.conditionallyRequired = append(pre.conditionallyRequired, p) - updatedPresubmits[orgRepo] = pre - continue - } - } - } - - c.m.Lock() - defer c.m.Unlock() - c.updatedPresubmits = updatedPresubmits -} diff --git a/cmd/pipeline-controller/config_data_provider_test.go b/cmd/pipeline-controller/config_data_provider_test.go deleted file mode 100644 index 6d81c2729e9..00000000000 --- a/cmd/pipeline-controller/config_data_provider_test.go +++ /dev/null @@ -1,315 +0,0 @@ -package main - -import ( - "io" - "reflect" - "sync" - "testing" - - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/config" -) - -// testLogger creates a discarded logger for tests -func testLogger() *logrus.Entry { - logger := logrus.New() - logger.SetOutput(io.Discard) - return logrus.NewEntry(logger) -} - -func yes() *bool { - yes := true - return &yes -} - -func composeBPConfig() config.ProwConfig { - return config.ProwConfig{ - BranchProtection: config.BranchProtection{ - Orgs: map[string]config.Org{ - "org": { - Policy: config.Policy{}, - Repos: map[string]config.Repo{ - "repo": { - Policy: config.Policy{}, - Branches: make(map[string]config.Branch), - }, - }}, - }, - }, - } -} - -func decorateWithOrgPolicy(cfg config.ProwConfig) config.ProwConfig { - if org, ok := cfg.BranchProtection.Orgs["org"]; ok { - org.Policy.RequireManuallyTriggeredJobs = yes() - cfg.BranchProtection.Orgs["org"] = org - - } - return cfg -} - -func decorateWithRepoPolicy(cfg config.ProwConfig) config.ProwConfig { - if org, ok := cfg.BranchProtection.Orgs["org"]; ok { - if repo, ok := org.Repos["repo"]; ok { - repo.Policy.RequireManuallyTriggeredJobs = yes() - cfg.BranchProtection.Orgs["org"].Repos["repo"] = repo - } - } - return cfg -} - -func composeProtectedPresubmit(name string) config.Presubmit { //nolint:unparam // parameter allows flexibility for future test cases - return config.Presubmit{ - JobBase: config.JobBase{Name: name}, - AlwaysRun: false, - Optional: false, - RegexpChangeMatcher: config.RegexpChangeMatcher{ - SkipIfOnlyChanged: "", - RunIfChanged: "", - }, - } -} - -func composeRequiredPresubmit() config.Presubmit { - return config.Presubmit{ - JobBase: config.JobBase{Name: "ps2"}, - AlwaysRun: true, - Optional: false, - } -} - -func composeCondRequiredPresubmit(name string) config.Presubmit { - return config.Presubmit{ - JobBase: config.JobBase{Name: name}, - RegexpChangeMatcher: config.RegexpChangeMatcher{RunIfChanged: ".*"}, - } -} - -func composePipelineCondRequiredPresubmit(name string, optional bool, annotations map[string]string) config.Presubmit { - return config.Presubmit{ - AlwaysRun: false, - Optional: optional, - JobBase: config.JobBase{Name: name, Annotations: annotations}, - } -} - -func TestConfigDataProviderGatherData(t *testing.T) { - tests := []struct { - name string - configGetter config.Getter - repoLister RepoLister - expected presubmitTests - }{ - { - name: "Org policy requires manual trigger, repo policy does not", - configGetter: func() *config.Config { - cfs := config.Config{ - JobConfig: config.JobConfig{PresubmitsStatic: map[string][]config.Presubmit{ - "org/repo": { - composeProtectedPresubmit("ps1"), - composeRequiredPresubmit(), - composeCondRequiredPresubmit("ps3"), - composePipelineCondRequiredPresubmit("ps4", false, map[string]string{"pipeline_run_if_changed": ".*"}), - composePipelineCondRequiredPresubmit("ps5", true, map[string]string{"pipeline_run_if_changed": ".*"}), - composePipelineCondRequiredPresubmit("ps6", true, map[string]string{}), - }, - }}, - ProwConfig: decorateWithOrgPolicy(composeBPConfig()), - } - return &cfs - }, - repoLister: func() []string { - return []string{"org/repo"} - }, - expected: presubmitTests{ - protected: []config.Presubmit{composeProtectedPresubmit("ps1")}, - alwaysRequired: []config.Presubmit{composeRequiredPresubmit()}, - conditionallyRequired: []config.Presubmit{composeCondRequiredPresubmit("ps3")}, - pipelineConditionallyRequired: []config.Presubmit{ - composePipelineCondRequiredPresubmit("ps4", false, map[string]string{"pipeline_run_if_changed": ".*"}), - composePipelineCondRequiredPresubmit("ps5", true, map[string]string{"pipeline_run_if_changed": ".*"}), - }}, - }, - { - name: "Org policy and repo require manual trigger", - configGetter: func() *config.Config { - return &config.Config{ - JobConfig: config.JobConfig{PresubmitsStatic: map[string][]config.Presubmit{ - "org/repo": { - composeProtectedPresubmit("ps1"), - composeRequiredPresubmit(), - {JobBase: config.JobBase{Name: "ps3"}, Optional: true}, - }, - }}, - ProwConfig: decorateWithRepoPolicy(decorateWithOrgPolicy(composeBPConfig())), - } - }, - repoLister: func() []string { - return []string{"org/repo"} - }, - expected: presubmitTests{protected: []config.Presubmit{composeProtectedPresubmit("ps1")}, alwaysRequired: []config.Presubmit{composeRequiredPresubmit()}}, - }, - { - name: "No manual trigger required", - configGetter: func() *config.Config { - return &config.Config{ - JobConfig: config.JobConfig{PresubmitsStatic: map[string][]config.Presubmit{ - "org/repo": { - composeProtectedPresubmit("ps1"), - }, - }}, - ProwConfig: composeBPConfig(), - } - }, - repoLister: func() []string { - return []string{} // No repos configured - this should result in empty results - }, - expected: presubmitTests{}, - }, - { - name: "Jobs with pipeline_skip_if_only_changed are collected", - configGetter: func() *config.Config { - cfs := config.Config{ - JobConfig: config.JobConfig{PresubmitsStatic: map[string][]config.Presubmit{ - "org/repo": { - composeProtectedPresubmit("ps1"), - composeRequiredPresubmit(), - composePipelineCondRequiredPresubmit("ps3", false, map[string]string{"pipeline_skip_if_only_changed": "^docs/.*"}), - composePipelineCondRequiredPresubmit("ps4", true, map[string]string{"pipeline_skip_if_only_changed": "^test/.*"}), - composePipelineCondRequiredPresubmit("ps5", false, map[string]string{"pipeline_run_if_changed": `.*\.go`}), - }, - }}, - ProwConfig: decorateWithOrgPolicy(composeBPConfig()), - } - return &cfs - }, - repoLister: func() []string { - return []string{"org/repo"} - }, - expected: presubmitTests{ - protected: []config.Presubmit{composeProtectedPresubmit("ps1")}, - alwaysRequired: []config.Presubmit{composeRequiredPresubmit()}, - conditionallyRequired: []config.Presubmit{}, - pipelineConditionallyRequired: []config.Presubmit{ - composePipelineCondRequiredPresubmit("ps5", false, map[string]string{"pipeline_run_if_changed": `.*\.go`}), - }, - pipelineSkipOnlyRequired: []config.Presubmit{ - composePipelineCondRequiredPresubmit("ps3", false, map[string]string{"pipeline_skip_if_only_changed": "^docs/.*"}), - composePipelineCondRequiredPresubmit("ps4", true, map[string]string{"pipeline_skip_if_only_changed": "^test/.*"}), - }, - }, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - c := &ConfigDataProvider{ - configGetter: tc.configGetter, - repoLister: tc.repoLister, - updatedPresubmits: make(map[string]presubmitTests), - logger: testLogger(), - m: sync.Mutex{}, - } - c.gatherData() - actual := c.GetPresubmits("org/repo") - // Compare protected presubmits by name - if len(actual.protected) != len(tc.expected.protected) { - t.Errorf("protected length - expected %d, got %d", len(tc.expected.protected), len(actual.protected)) - } else { - for _, expected := range tc.expected.protected { - found := false - for _, actualItem := range actual.protected { - if expected.Name == actualItem.Name { - found = true - break - } - } - if !found { - t.Errorf("protected - expected to find job %s", expected.Name) - } - } - } - - // Compare always required presubmits by name - if len(actual.alwaysRequired) != len(tc.expected.alwaysRequired) { - t.Errorf("alwaysRequired length - expected %d, got %d", len(tc.expected.alwaysRequired), len(actual.alwaysRequired)) - } else { - for _, expected := range tc.expected.alwaysRequired { - found := false - for _, actualItem := range actual.alwaysRequired { - if expected.Name == actualItem.Name { - found = true - break - } - } - if !found { - t.Errorf("alwaysRequired - expected to find job %s", expected.Name) - } - } - } - - // Compare conditionally required presubmits by name - if len(actual.conditionallyRequired) != len(tc.expected.conditionallyRequired) { - t.Errorf("conditionallyRequired length - expected %d, got %d", len(tc.expected.conditionallyRequired), len(actual.conditionallyRequired)) - } else { - for _, expected := range tc.expected.conditionallyRequired { - found := false - for _, actualItem := range actual.conditionallyRequired { - if expected.Name == actualItem.Name { - found = true - break - } - } - if !found { - t.Errorf("conditionallyRequired - expected to find job %s", expected.Name) - } - } - } - // For pipelineConditionallyRequired, check length and then check each item exists - if len(actual.pipelineConditionallyRequired) != len(tc.expected.pipelineConditionallyRequired) { - t.Errorf("pipelineConditionallyRequired length - expected %d, got %d", - len(tc.expected.pipelineConditionallyRequired), - len(actual.pipelineConditionallyRequired)) - return - } - - // Check that each expected item exists in actual - for _, expected := range tc.expected.pipelineConditionallyRequired { - found := false - for _, actualItem := range actual.pipelineConditionallyRequired { - if expected.Name == actualItem.Name && - reflect.DeepEqual(expected.Annotations, actualItem.Annotations) && - expected.Optional == actualItem.Optional { - found = true - break - } - } - if !found { - t.Errorf("pipelineConditionallyRequired - expected to find job %s with annotations %v and optional=%v", - expected.Name, expected.Annotations, expected.Optional) - } - } - // For pipelineSkipOnlyRequired, check length and then check each item exists - if len(actual.pipelineSkipOnlyRequired) != len(tc.expected.pipelineSkipOnlyRequired) { - t.Errorf("pipelineSkipOnlyRequired length - expected %d, got %d", - len(tc.expected.pipelineSkipOnlyRequired), - len(actual.pipelineSkipOnlyRequired)) - } else { - for _, expectedJob := range tc.expected.pipelineSkipOnlyRequired { - found := false - for _, actualJob := range actual.pipelineSkipOnlyRequired { - if expectedJob.Name == actualJob.Name && - reflect.DeepEqual(expectedJob.Annotations, actualJob.Annotations) && - expectedJob.Optional == actualJob.Optional { - found = true - break - } - } - if !found { - t.Errorf("pipelineSkipOnlyRequired - expected job %s not found", expectedJob.Name) - } - } - } - }) - } -} diff --git a/cmd/pipeline-controller/config_watcher.go b/cmd/pipeline-controller/config_watcher.go deleted file mode 100644 index 7507951a3b1..00000000000 --- a/cmd/pipeline-controller/config_watcher.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "os" - "reflect" - "sync" - "time" - - "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" -) - -// RepoItem represents a repository configuration that can be either a string or an object -type RepoItem struct { - Name string - Branches []string - Mode struct { - Trigger string - } -} - -// UnmarshalYAML implements custom unmarshaling to support both string and object formats -func (r *RepoItem) UnmarshalYAML(unmarshal func(interface{}) error) error { - // Try to unmarshal as a string first (backwards compatibility) - var repoString string - if err := unmarshal(&repoString); err == nil { - r.Name = repoString - r.Mode.Trigger = "auto" // default to auto for backwards compatibility - return nil - } - - // If string unmarshaling failed, try as a struct - type rawRepo struct { - Name string `yaml:"name"` - Branches []string `yaml:"branches,omitempty"` - Mode struct { - Trigger string `yaml:"trigger"` - } `yaml:"mode,omitempty"` - } - var raw rawRepo - if err := unmarshal(&raw); err != nil { - return err - } - - r.Name = raw.Name - r.Branches = raw.Branches - r.Mode.Trigger = raw.Mode.Trigger - if r.Mode.Trigger == "" { - r.Mode.Trigger = "auto" // default to auto if not specified - } - return nil -} - -// enabled config struct represents the YAML file structure of enabled repos and orgs -type enabledConfig struct { - Orgs []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - } `yaml:"orgs"` -} - -// RepoConfig contains configuration for a single repository -type RepoConfig struct { - Trigger string - Branches []string // If empty, all branches are enabled -} - -// watcher struct encapsulates the file watcher and configuration -type watcher struct { - filePath string - config enabledConfig - mutex sync.Mutex - logger *logrus.Entry -} - -func newWatcher(filePath string, logger *logrus.Entry) *watcher { - watcher := &watcher{ - filePath: filePath, - logger: logger, - } - - return watcher -} - -func (w *watcher) watch() { - // Load initial config - if err := w.reloadConfig(); err != nil { - w.logger.WithError(err).Error("Failed to load initial config") - } - - // Use polling instead of fsnotify because git-sync doesn't trigger filesystem events - ticker := time.NewTicker(3 * time.Minute) - defer ticker.Stop() - - // Store previous config for comparison - prevConfig := w.getConfigCopy() - - for range ticker.C { - if err := w.reloadConfig(); err != nil { - w.logger.WithError(err).Error("Failed to reload config") - continue - } - - currentConfig := w.getConfigCopy() - if !reflect.DeepEqual(currentConfig, prevConfig) { - w.logger.Info("Config change detected, config reloaded successfully") - prevConfig = currentConfig - } - } -} - -// getConfigCopy returns a deep copy of the current config for comparison -func (w *watcher) getConfigCopy() enabledConfig { - w.mutex.Lock() - defer w.mutex.Unlock() - return w.config -} - -func (w *watcher) reloadConfig() error { - w.mutex.Lock() - defer w.mutex.Unlock() - - yamlFile, err := os.Open(w.filePath) - if err != nil { - return err - } - - defer yamlFile.Close() - - decoder := yaml.NewDecoder(yamlFile) - err = decoder.Decode(&w.config) - if err != nil { - return err - } - - return nil -} - -func (w *watcher) getConfig() map[string]map[string]RepoConfig { - w.mutex.Lock() - defer w.mutex.Unlock() - - ret := map[string]map[string]RepoConfig{} - for _, org := range w.config.Orgs { - repoConfigs := map[string]RepoConfig{} - for _, repo := range org.Repos { - repoConfigs[repo.Name] = RepoConfig{ - Trigger: repo.Mode.Trigger, - Branches: repo.Branches, - } - } - ret[org.Org] = repoConfigs - } - return ret -} diff --git a/cmd/pipeline-controller/config_watcher_test.go b/cmd/pipeline-controller/config_watcher_test.go deleted file mode 100644 index fec127c080c..00000000000 --- a/cmd/pipeline-controller/config_watcher_test.go +++ /dev/null @@ -1,190 +0,0 @@ -package main - -import ( - "os" - "testing" - - "gopkg.in/yaml.v2" -) - -func TestConfigBackwardsCompatibility(t *testing.T) { - tests := []struct { - name string - yamlContent string - expectedOrgs int - expectedRepos map[string]map[string]string // org -> repo -> trigger - }{ - { - name: "old format with string repos", - yamlContent: `orgs: - - org: openshift - repos: - - cluster-capi-operator - - installer -`, - expectedOrgs: 1, - expectedRepos: map[string]map[string]string{ - "openshift": { - "cluster-capi-operator": "auto", - "installer": "auto", - }, - }, - }, - { - name: "new format with object repos", - yamlContent: `orgs: - - org: openshift - repos: - - name: cluster-capi-operator - mode: - trigger: manual - - name: installer - mode: - trigger: auto -`, - expectedOrgs: 1, - expectedRepos: map[string]map[string]string{ - "openshift": { - "cluster-capi-operator": "manual", - "installer": "auto", - }, - }, - }, - { - name: "new format with missing trigger (should default to auto)", - yamlContent: `orgs: - - org: openshift - repos: - - name: cluster-capi-operator -`, - expectedOrgs: 1, - expectedRepos: map[string]map[string]string{ - "openshift": { - "cluster-capi-operator": "auto", - }, - }, - }, - { - name: "mixed format (not recommended but should work)", - yamlContent: `orgs: - - org: openshift - repos: - - installer - - name: cluster-capi-operator - mode: - trigger: manual -`, - expectedOrgs: 1, - expectedRepos: map[string]map[string]string{ - "openshift": { - "installer": "auto", - "cluster-capi-operator": "manual", - }, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - // Create temporary file with YAML content - tmpfile, err := os.CreateTemp("", "config-*.yaml") - if err != nil { - t.Fatal(err) - } - defer os.Remove(tmpfile.Name()) - - if _, err := tmpfile.Write([]byte(tc.yamlContent)); err != nil { - t.Fatal(err) - } - if err := tmpfile.Close(); err != nil { - t.Fatal(err) - } - - // Parse the config - var config enabledConfig - yamlFile, err := os.Open(tmpfile.Name()) - if err != nil { - t.Fatal(err) - } - defer yamlFile.Close() - - decoder := yaml.NewDecoder(yamlFile) - if err := decoder.Decode(&config); err != nil { - t.Fatalf("Failed to decode YAML: %v", err) - } - - // Verify the results - if len(config.Orgs) != tc.expectedOrgs { - t.Errorf("Expected %d orgs, got %d", tc.expectedOrgs, len(config.Orgs)) - } - - for _, org := range config.Orgs { - expectedRepos, ok := tc.expectedRepos[org.Org] - if !ok { - t.Errorf("Unexpected org: %s", org.Org) - continue - } - - if len(org.Repos) != len(expectedRepos) { - t.Errorf("For org %s, expected %d repos, got %d", org.Org, len(expectedRepos), len(org.Repos)) - } - - for _, repo := range org.Repos { - expectedTrigger, ok := expectedRepos[repo.Name] - if !ok { - t.Errorf("Unexpected repo %s in org %s", repo.Name, org.Org) - continue - } - - if repo.Mode.Trigger != expectedTrigger { - t.Errorf("For repo %s/%s, expected trigger %s, got %s", org.Org, repo.Name, expectedTrigger, repo.Mode.Trigger) - } - } - } - }) - } -} - -func TestWatcherGetConfig(t *testing.T) { - // Test that the watcher.getConfig() method properly converts the config - w := &watcher{ - config: enabledConfig{ - Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: "openshift", - Repos: []RepoItem{ - {Name: "repo1", Mode: struct{ Trigger string }{Trigger: "auto"}}, - {Name: "repo2", Mode: struct{ Trigger string }{Trigger: "manual"}}, - }, - }, - }, - }, - } - - config := w.getConfig() - - // Verify the results - if len(config) != 1 { - t.Errorf("Expected 1 org, got %d", len(config)) - } - - repos, ok := config["openshift"] - if !ok { - t.Error("Expected org 'openshift' not found") - } - - if len(repos) != 2 { - t.Errorf("Expected 2 repos, got %d", len(repos)) - } - - if repos["repo1"].Trigger != "auto" { - t.Errorf("Expected repo1 trigger to be 'auto', got %s", repos["repo1"].Trigger) - } - - if repos["repo2"].Trigger != "manual" { - t.Errorf("Expected repo2 trigger to be 'manual', got %s", repos["repo2"].Trigger) - } -} diff --git a/cmd/pipeline-controller/helpers.go b/cmd/pipeline-controller/helpers.go deleted file mode 100644 index 4b5d0fe96e1..00000000000 --- a/cmd/pipeline-controller/helpers.go +++ /dev/null @@ -1,191 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strings" - - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - v1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" - "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/kube" -) - -type minimalGhClient interface { - GetPullRequest(org, repo string, number int) (*github.PullRequest, error) - CreateComment(org, repo string, number int, comment string) error - GetPullRequestChanges(org string, repo string, number int) ([]github.PullRequestChange, error) - CreateStatus(org, repo, ref string, s github.Status) error - AddLabel(org, repo string, number int, label string) error - GetIssueLabels(org, repo string, number int) ([]github.Label, error) -} - -func sendComment(presubmits presubmitTests, pj *v1.ProwJob, ghc minimalGhClient, deleteIds func(), pjLister ctrlruntimeclient.Reader) error { - return sendCommentWithMode(presubmits, pj, ghc, deleteIds, pjLister, false) -} - -func sendCommentWithMode(presubmits presubmitTests, pj *v1.ProwJob, ghc minimalGhClient, deleteIds func(), pjLister ctrlruntimeclient.Reader, isExplicitCommand bool) error { - if pj.Spec.Refs == nil || len(pj.Spec.Refs.Pulls) == 0 { - deleteIds() - return fmt.Errorf("ProwJob %s does not have valid Refs.Pulls", pj.Name) - } - - // Combine pipelineConditionallyRequired and pipelineSkipOnlyRequired for processing - allConditionalTests := append([]config.Presubmit{}, presubmits.pipelineConditionallyRequired...) - allConditionalTests = append(allConditionalTests, presubmits.pipelineSkipOnlyRequired...) - - testContexts, manualControlMessage, err := acquireConditionalContexts(context.Background(), pj, allConditionalTests, ghc, deleteIds, pjLister, isExplicitCommand) - if err != nil { - deleteIds() - return err - } - - var comment string - - repoBaseRef := pj.Spec.Refs.Repo + "-" + pj.Spec.Refs.BaseRef - - // If it's an explicit /pipeline required command, ignore manual control message - // and proceed with scheduling tests - if manualControlMessage != "" && !isExplicitCommand { - comment = manualControlMessage - } else { - var protectedCommands string - for _, presubmit := range presubmits.protected { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - protectedCommands += "\n" + presubmit.RerunCommand - } - if protectedCommands != "" { - comment += "Scheduling required tests:" + protectedCommands - } - if testContexts != "" { - if protectedCommands != "" { - comment += "\n" - } - comment += "\nScheduling tests matching the `pipeline_run_if_changed` or not excluded by `pipeline_skip_if_only_changed` parameters:" - comment += testContexts - } - } - - // If no tests matched, send an informative comment instead of staying silent - if comment == "" { - comment = fmt.Sprintf("**Pipeline controller notification**\n\nNo second-stage tests were triggered for this PR.\n\nThis can happen when:\n- The changed files don't match any `pipeline_run_if_changed` patterns\n- All files match `pipeline_skip_if_only_changed` patterns\n- No pipeline-controlled jobs are defined for the `%s` branch\n\nUse `/test ?` to see all available tests.", pj.Spec.Refs.BaseRef) - } - - if err := ghc.CreateComment(pj.Spec.Refs.Org, pj.Spec.Refs.Repo, pj.Spec.Refs.Pulls[0].Number, comment); err != nil { - deleteIds() - return err - } - return nil -} - -func acquireConditionalContexts(ctx context.Context, pj *v1.ProwJob, pipelineConditionallyRequired []config.Presubmit, ghc minimalGhClient, deleteIds func(), pjLister ctrlruntimeclient.Reader, isExplicitCommand bool) (string, string, error) { - if pj.Spec.Refs == nil || len(pj.Spec.Refs.Pulls) == 0 { - return "", "", fmt.Errorf("ProwJob %s does not have valid Refs.Pulls", pj.Name) - } - - repoBaseRef := pj.Spec.Refs.Repo + "-" + pj.Spec.Refs.BaseRef - var testCommands string - if len(pipelineConditionallyRequired) != 0 { - cfp := config.NewGitHubDeferredChangedFilesProvider(ghc, pj.Spec.Refs.Org, pj.Spec.Refs.Repo, pj.Spec.Refs.Pulls[0].Number) - - // First, determine which tests should run based on file changes - var testsToRun []config.Presubmit - for _, presubmit := range pipelineConditionallyRequired { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - - shouldRun := false - // Check pipeline_run_if_changed first (takes precedence) - if run, ok := presubmit.Annotations["pipeline_run_if_changed"]; ok && run != "" { - psList := []config.Presubmit{presubmit} - psList[0].RegexpChangeMatcher = config.RegexpChangeMatcher{RunIfChanged: run} - if err := config.SetPresubmitRegexes(psList); err != nil { - deleteIds() - return "", "", err - } - _, shouldRunResult, err := psList[0].RegexpChangeMatcher.ShouldRun(cfp) - if err != nil { - deleteIds() - return "", "", err - } - shouldRun = shouldRunResult - } else if skip, ok := presubmit.Annotations["pipeline_skip_if_only_changed"]; ok && skip != "" { - // Check pipeline_skip_if_only_changed if pipeline_run_if_changed is not present - psList := []config.Presubmit{presubmit} - psList[0].RegexpChangeMatcher = config.RegexpChangeMatcher{SkipIfOnlyChanged: skip} - if err := config.SetPresubmitRegexes(psList); err != nil { - deleteIds() - return "", "", err - } - _, shouldRunResult, err := psList[0].RegexpChangeMatcher.ShouldRun(cfp) - if err != nil { - deleteIds() - return "", "", err - } - shouldRun = shouldRunResult - } else { - shouldRun = true - } - - if shouldRun { - testsToRun = append(testsToRun, presubmit) - } - } - - // Check if any of the tests that should run have already been manually triggered - // Skip this check if it's an explicit /pipeline required command - if len(testsToRun) > 0 && pjLister != nil && pj.Spec.Refs.Pulls[0].SHA != "" && !isExplicitCommand { - // Build label selector from ProwJob spec (same as in reconciler.go) - selector := map[string]string{ - kube.OrgLabel: pj.Spec.Refs.Org, - kube.RepoLabel: pj.Spec.Refs.Repo, - kube.PullLabel: fmt.Sprintf("%d", pj.Spec.Refs.Pulls[0].Number), - kube.BaseRefLabel: pj.Spec.Refs.BaseRef, - kube.ProwJobTypeLabel: string(v1.PresubmitJob), - } - - var pjs v1.ProwJobList - if err := pjLister.List(ctx, &pjs, ctrlruntimeclient.MatchingLabels(selector)); err != nil { - // If listing fails, skip check and proceed with creating comment - deleteIds() - testCommands := "" - for _, presubmit := range testsToRun { - testCommands += "\n" + presubmit.RerunCommand - } - return testCommands, "", nil - } - - // Check if any of the tests we want to run have already been triggered - // by looking for ProwJobs with matching job names and same SHA - repoBaseRef := pj.Spec.Refs.Repo + "-" + pj.Spec.Refs.BaseRef - for _, presubmit := range testsToRun { - testName := presubmit.Name - // Only check presubmits that match the repo-baseRef pattern (same as reconciler) - if !strings.Contains(testName, repoBaseRef) { - continue - } - for _, pjob := range pjs.Items { - // Check if this ProwJob matches the test we want to run - // and if it's for the same SHA - // If a job exists in ANY state, it means it was already triggered - // so we should not run it and inform the user - if pjob.Spec.Job == testName && - pjob.Spec.Refs != nil && - len(pjob.Spec.Refs.Pulls) > 0 && - pjob.Spec.Refs.Pulls[0].SHA == pj.Spec.Refs.Pulls[0].SHA { - return "", "Tests from second stage were triggered manually. Pipeline can be controlled only manually, until HEAD changes. Use command to trigger second stage.", nil - } - } - } - } - - for _, presubmit := range testsToRun { - testCommands += "\n" + presubmit.RerunCommand - } - } - return testCommands, "", nil -} diff --git a/cmd/pipeline-controller/helpers_test.go b/cmd/pipeline-controller/helpers_test.go deleted file mode 100644 index f5ac62a5d1a..00000000000 --- a/cmd/pipeline-controller/helpers_test.go +++ /dev/null @@ -1,479 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - v1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" - "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/kube" -) - -type fakeGhClientWithChanges struct { - changes []github.PullRequestChange - comment string -} - -func (f *fakeGhClientWithChanges) GetPullRequest(org, repo string, number int) (*github.PullRequest, error) { - return &github.PullRequest{State: github.PullRequestStateOpen}, nil -} - -func (f *fakeGhClientWithChanges) CreateComment(owner, repo string, number int, comment string) error { - f.comment = comment - return nil -} - -func (f *fakeGhClientWithChanges) GetPullRequestChanges(org, repo string, number int) ([]github.PullRequestChange, error) { - return f.changes, nil -} - -func (f *fakeGhClientWithChanges) CreateStatus(org, repo, ref string, s github.Status) error { - return nil -} - -func (f *fakeGhClientWithChanges) AddLabel(org, repo string, number int, label string) error { - return nil -} - -func (f *fakeGhClientWithChanges) GetIssueLabels(org, repo string, number int) ([]github.Label, error) { - return []github.Label{}, nil -} - -type fakeProwJobLister struct { - prowJobs []v1.ProwJob -} - -func (f *fakeProwJobLister) Get(ctx context.Context, key ctrlruntimeclient.ObjectKey, obj ctrlruntimeclient.Object, opts ...ctrlruntimeclient.GetOption) error { - return nil -} - -func (f *fakeProwJobLister) List(ctx context.Context, list ctrlruntimeclient.ObjectList, opts ...ctrlruntimeclient.ListOption) error { - pjList := list.(*v1.ProwJobList) - pjList.Items = f.prowJobs - return nil -} - -func TestAcquireConditionalContexts(t *testing.T) { - basePJ := &v1.ProwJob{ - Spec: v1.ProwJobSpec{ - Refs: &v1.Refs{ - Org: "org", - Repo: "repo", - BaseRef: "master", - Pulls: []v1.Pull{ - {Number: 123, SHA: "abc"}, - }, - }, - }, - } - - tests := []struct { - name string - pipelineConditionallyRequired []config.Presubmit - changes []github.PullRequestChange - existingProwJobs []v1.ProwJob - expectedTestCommands []string - expectedManualControlMessage string - expectedError string - }{ - { - name: "pipeline_run_if_changed matches files", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go", - }, - }, - Reporter: config.Reporter{ - Context: "test", - }, - RerunCommand: "/test test", - }, - }, - changes: []github.PullRequestChange{ - {Filename: "main.go"}, - {Filename: "README.md"}, - }, - existingProwJobs: []v1.ProwJob{}, - expectedTestCommands: []string{"/test test"}, - expectedManualControlMessage: "", - expectedError: "", - }, - { - name: "pipeline_run_if_changed does not match files", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go", - }, - }, - Reporter: config.Reporter{ - Context: "test", - }, - RerunCommand: "/test test", - Optional: false, - }, - }, - changes: []github.PullRequestChange{ - {Filename: "README.md"}, - {Filename: "docs/guide.md"}, - }, - existingProwJobs: []v1.ProwJob{}, - expectedTestCommands: []string{}, - expectedManualControlMessage: "", - expectedError: "", - }, - { - name: "pipeline_skip_if_only_changed skips when only matching files changed", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": "^docs/.*|.*\\.md", - }, - }, - Reporter: config.Reporter{ - Context: "test", - }, - RerunCommand: "/test test", - Optional: false, - }, - }, - changes: []github.PullRequestChange{ - {Filename: "docs/guide.md"}, - {Filename: "README.md"}, - }, - existingProwJobs: []v1.ProwJob{}, - expectedTestCommands: []string{}, - expectedManualControlMessage: "", - expectedError: "", - }, - { - name: "pipeline_skip_if_only_changed runs when other files are changed", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": "^docs/.*|.*\\.md", - }, - }, - Reporter: config.Reporter{ - Context: "test", - }, - RerunCommand: "/test test", - }, - }, - changes: []github.PullRequestChange{ - {Filename: "docs/guide.md"}, - {Filename: "main.go"}, - }, - existingProwJobs: []v1.ProwJob{}, - expectedTestCommands: []string{"/test test"}, - expectedManualControlMessage: "", - expectedError: "", - }, - { - name: "pipeline_run_if_changed takes precedence over pipeline_skip_if_only_changed", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go", - "pipeline_skip_if_only_changed": "^test/.*", - }, - }, - Reporter: config.Reporter{ - Context: "test", - }, - RerunCommand: "/test test", - }, - }, - changes: []github.PullRequestChange{ - {Filename: "test/test.go"}, - }, - existingProwJobs: []v1.ProwJob{}, - expectedTestCommands: []string{"/test test"}, - expectedManualControlMessage: "", - expectedError: "", - }, - { - name: "multiple jobs with different annotations", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test1", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go", - }, - }, - Reporter: config.Reporter{ - Context: "test1", - }, - RerunCommand: "/test test1", - }, - { - JobBase: config.JobBase{ - Name: "org-repo-master-test2", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": "^docs/.*", - }, - }, - Reporter: config.Reporter{ - Context: "test2", - }, - RerunCommand: "/test test2", - }, - }, - changes: []github.PullRequestChange{ - {Filename: "main.go"}, - {Filename: "docs/guide.md"}, - }, - existingProwJobs: []v1.ProwJob{}, - expectedTestCommands: []string{"/test test1", "/test test2"}, - expectedManualControlMessage: "", - expectedError: "", - }, - { - name: "job name does not contain repo-baseRef", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "different-job", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*", - }, - }, - Reporter: config.Reporter{ - Context: "different", - }, - RerunCommand: "/test different", - }, - }, - changes: []github.PullRequestChange{ - {Filename: "main.go"}, - }, - existingProwJobs: []v1.ProwJob{}, - expectedTestCommands: []string{}, - expectedManualControlMessage: "", - expectedError: "", - }, - { - name: "test already running - should return manual control message", - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go", - }, - }, - Reporter: config.Reporter{ - Context: "test", - }, - RerunCommand: "/test test", - }, - }, - changes: []github.PullRequestChange{ - {Filename: "main.go"}, - }, - existingProwJobs: []v1.ProwJob{ - { - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - kube.OrgLabel: "org", - kube.RepoLabel: "repo", - kube.PullLabel: "123", - kube.BaseRefLabel: "master", - kube.ProwJobTypeLabel: string(v1.PresubmitJob), - }, - }, - Spec: v1.ProwJobSpec{ - Job: "org-repo-master-test", - Refs: &v1.Refs{ - Org: "org", - Repo: "repo", - BaseRef: "master", - Pulls: []v1.Pull{ - {Number: 123, SHA: "abc"}, - }, - }, - }, - }, - }, - expectedTestCommands: []string{}, - expectedManualControlMessage: "Tests from second stage were triggered manually. Pipeline can be controlled only manually, until HEAD changes. Use command to trigger second stage.", - expectedError: "", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ghc := &fakeGhClientWithChanges{changes: tc.changes} - var pjLister ctrlruntimeclient.Reader - if len(tc.existingProwJobs) > 0 { - pjLister = &fakeProwJobLister{prowJobs: tc.existingProwJobs} - } - testCmds, manualControlMessage, err := acquireConditionalContexts(context.Background(), basePJ, tc.pipelineConditionallyRequired, ghc, func() {}, pjLister, false) - - // Check expected error - if tc.expectedError != "" { - if err == nil { - t.Errorf("expected error %q, got nil", tc.expectedError) - } else if err.Error() != tc.expectedError { - t.Errorf("expected error %q, got %q", tc.expectedError, err.Error()) - } - return - } - - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - // Check manual control message - if tc.expectedManualControlMessage != "" { - if manualControlMessage != tc.expectedManualControlMessage { - t.Errorf("expected manual control message %q, got %q", tc.expectedManualControlMessage, manualControlMessage) - } - return - } - - // Check test commands - for _, expected := range tc.expectedTestCommands { - if !strings.Contains(testCmds, expected) { - t.Errorf("expected test commands to contain %q, got %q", expected, testCmds) - } - } - - // Check that we don't have unexpected commands - if len(tc.expectedTestCommands) == 0 && testCmds != "" { - t.Errorf("expected no test commands, got %q", testCmds) - } - }) - } -} - -func TestSendCommentWithMode(t *testing.T) { - basePJ := &v1.ProwJob{ - Spec: v1.ProwJobSpec{ - Refs: &v1.Refs{ - Org: "org", - Repo: "repo", - BaseRef: "master", - Pulls: []v1.Pull{ - {Number: 123, SHA: "abc"}, - }, - }, - }, - } - - tests := []struct { - name string - presubmits presubmitTests - changes []github.PullRequestChange - expectedCommentContains []string - expectedCommentNotContains []string - }{ - { - name: "manual mode with pipeline jobs", - presubmits: presubmitTests{ - protected: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-protected", - }, - Reporter: config.Reporter{ - Context: "protected", - }, - RerunCommand: "/test protected", - }, - }, - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go", - }, - }, - Reporter: config.Reporter{ - Context: "test", - }, - RerunCommand: "/test test", - Optional: false, - }, - }, - }, - changes: []github.PullRequestChange{ - {Filename: "README.md"}, - }, - expectedCommentContains: []string{ - "Scheduling required tests:", - "/test protected", - }, - }, - { - name: "automatic mode with pipeline jobs", - presubmits: presubmitTests{ - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "org-repo-master-test", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": "^docs/.*", - }, - }, - Reporter: config.Reporter{ - Context: "test2", - }, - RerunCommand: "/test test2", - }, - }, - }, - changes: []github.PullRequestChange{ - {Filename: "main.go"}, - }, - expectedCommentContains: []string{ - "/test test2", - }, - expectedCommentNotContains: []string{ - "Pipeline controller response", - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ghc := &fakeGhClientWithChanges{changes: tc.changes} - - err := sendCommentWithMode(tc.presubmits, basePJ, ghc, func() {}, nil, false) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - comment := ghc.comment - - for _, expected := range tc.expectedCommentContains { - if !strings.Contains(comment, expected) { - t.Errorf("expected comment to contain %q, got:\n%s", expected, comment) - } - } - - for _, notExpected := range tc.expectedCommentNotContains { - if strings.Contains(comment, notExpected) { - t.Errorf("expected comment NOT to contain %q, got:\n%s", notExpected, comment) - } - } - }) - } -} diff --git a/cmd/pipeline-controller/main.go b/cmd/pipeline-controller/main.go deleted file mode 100644 index f05816a51cd..00000000000 --- a/cmd/pipeline-controller/main.go +++ /dev/null @@ -1,869 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "os" - "regexp" - "strings" - "sync" - "time" - - "github.com/bombsimon/logrusr/v3" - "github.com/sirupsen/logrus" - - "sigs.k8s.io/controller-runtime/pkg/cache" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" - v1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" - "sigs.k8s.io/prow/pkg/config/secret" - prowflagutil "sigs.k8s.io/prow/pkg/flagutil" - configflagutil "sigs.k8s.io/prow/pkg/flagutil/config" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/githubeventserver" - "sigs.k8s.io/prow/pkg/interrupts" - "sigs.k8s.io/prow/pkg/labels" - "sigs.k8s.io/prow/pkg/logrusutil" -) - -const pullRequestInfoComment = "**Pipeline controller notification**\nThis repo is configured to use the [pipeline controller](https://docs.ci.openshift.org/how-tos/creating-a-pipeline/). Second-stage tests will be triggered either automatically or after lgtm label is added, depending on the repository configuration. The pipeline controller will automatically detect which contexts are required and will utilize `/test` Prow commands to trigger the second stage.\n\nFor optional jobs, comment `/test ?` to see a list of all defined jobs. To trigger manually all jobs from second stage use `/pipeline required` command. \n\nThis repository is configured in: " - -const RepoNotConfiguredMessage = "This repository is not currently configured for [pipeline controller](https://docs.ci.openshift.org/how-tos/creating-a-pipeline/) support." - -const PipelinePendingMessage = "Waiting for pipeline condition to trigger this job" - -// PipelineAutoLabel is the label that marks a PR to behave as automatic mode -const PipelineAutoLabel = "pipeline-auto" - -type options struct { - client prowflagutil.KubernetesOptions - github prowflagutil.GitHubOptions - githubEventServerOptions githubeventserver.Options - config configflagutil.ConfigOptions - configFile string - lgtmConfigFile string - dryrun bool - webhookSecretFile string -} - -func (o *options) validate() error { - for _, opt := range []interface{ Validate(bool) error }{&o.client, &o.config} { - if err := opt.Validate(o.dryrun); err != nil { - return err - } - } - - return nil -} - -func (o *options) parseArgs(fs *flag.FlagSet, args []string) error { - fs.BoolVar(&o.dryrun, "dry-run", false, "Run in dry-run mode.") - fs.StringVar(&o.configFile, "config-file", "", "Config file with list of enabled orgs and repos.") - fs.StringVar(&o.lgtmConfigFile, "lgtm-config-file", "", "Config file with list of enabled orgs and repos with second stage triggered by lgtm label.") - fs.StringVar(&o.webhookSecretFile, "hmac-secret-file", "/etc/webhook/hmac", "Path to the file containing the GitHub HMAC secret.") - - o.config.AddFlags(fs) - o.github.AddFlags(fs) - o.client.AddFlags(fs) - o.githubEventServerOptions.Bind(fs) - - if err := fs.Parse(args); err != nil { - logrus.WithError(err).Fatal("Could not parse args.") - } - - if o.configFile == "" { - return fmt.Errorf("--config-file is mandatory") - } - if o.lgtmConfigFile == "" { - return fmt.Errorf("--lgtm-config-file is mandatory") - } - if err := o.githubEventServerOptions.DefaultAndValidate(); err != nil { - return err - } - - return o.validate() -} - -func parseOptions() options { - var o options - - if err := o.parseArgs(flag.CommandLine, os.Args[1:]); err != nil { - logrus.WithError(err).Fatal("invalid flag options") - } - - return o -} - -type clientWrapper struct { - ghc minimalGhClient - configDataProvider *ConfigDataProvider - watcher *watcher - lgtmWatcher *watcher - pjLister ctrlruntimeclient.Reader - pipelineAutoCache *PipelineAutoCache - mu sync.RWMutex // Protects against race conditions in event handling -} - -// isBranchEnabled checks if the branch is enabled for the given repo configuration -// If branches list is empty, all branches are enabled -func isBranchEnabled(branches []string, branch string) bool { - if len(branches) == 0 { - return true // Empty list means all branches are enabled - } - for _, b := range branches { - if b == branch { - return true - } - } - return false -} - -func (cw *clientWrapper) handlePullRequestCreation(l *logrus.Entry, event github.PullRequestEvent) { - cw.mu.Lock() - defer cw.mu.Unlock() - - logger := l.WithFields(logrus.Fields{ - "handler": "handlePullRequestCreation", - "action": event.Action, - "org": event.Repo.Owner.Login, - "repo": event.Repo.Name, - "pr": event.PullRequest.Number, - }) - - logger.Info("Processing pull request event") - - if github.PullRequestActionOpened == event.Action { - org := event.Repo.Owner.Login - repo := event.Repo.Name - number := event.PullRequest.Number - - logger = logger.WithFields(logrus.Fields{ - "org": org, - "repo": repo, - "pr": number, - }) - - logger.Info("Processing PR opened event") - - // Check if repo is in configuration (either manual/auto mode or LGTM mode) - currentCfg := cw.watcher.getConfig() - repos, orgExists := currentCfg[org] - repoConfig, repoExists := repos[repo] - - lgtmCfg := cw.lgtmWatcher.getConfig() - lgtmRepos, lgtmOrgExists := lgtmCfg[org] - _, lgtmRepoExists := lgtmRepos[repo] - - isInConfig := (orgExists && repoExists) || (lgtmOrgExists && lgtmRepoExists) - - logger.WithFields(logrus.Fields{ - "org_exists": orgExists, - "repo_exists": repoExists, - "lgtm_org_exists": lgtmOrgExists, - "lgtm_repo_exists": lgtmRepoExists, - "is_in_config": isInConfig, - }).Debug("Configuration check results") - - if !isInConfig { - logger.Debug("Repository not in configuration (neither regular nor LGTM), skipping") - return - } - - // Check if branch is enabled for this repo - baseBranch := event.PullRequest.Base.Ref - var branchEnabled bool - if orgExists && repoExists { - branchEnabled = isBranchEnabled(repoConfig.Branches, baseBranch) - } else if lgtmOrgExists && lgtmRepoExists { - lgtmRepoConfig := lgtmRepos[repo] - branchEnabled = isBranchEnabled(lgtmRepoConfig.Branches, baseBranch) - } - - if !branchEnabled { - logger.WithField("base_branch", baseBranch).Debug("Branch not enabled for pipeline controller, skipping") - return - } - - // Check if repo is in regular config to determine trigger mode - var isAutomaticPipeline bool - isInLGTMConfig := lgtmOrgExists && lgtmRepoExists - if orgExists && repoExists { - isAutomaticPipeline = repoConfig.Trigger == "auto" - logger.WithField("trigger_mode", repoConfig.Trigger).Debug("Repository trigger mode") - } - - logger.Debug("Getting presubmits from config data provider") - presubmits := cw.configDataProvider.GetPresubmits(org + "/" + repo) - - // Show pipeline info comment for automatic mode or LGTM mode - if isAutomaticPipeline || isInLGTMConfig { - hasPipelineJobs := len(presubmits.protected) > 0 || len(presubmits.alwaysRequired) > 0 || - len(presubmits.conditionallyRequired) > 0 || len(presubmits.pipelineConditionallyRequired) > 0 || - len(presubmits.pipelineSkipOnlyRequired) > 0 - - logger.WithField("has_pipeline_jobs", hasPipelineJobs).Debug("Checking for pipeline jobs") - - if hasPipelineJobs { - // Repo has pipeline-controlled jobs and is in automatic mode or LGTM mode, use pipeline info comment - modeStr := "automatic mode" - if isInLGTMConfig && !isAutomaticPipeline { - modeStr = "LGTM mode" - } - logger.WithField("mode", modeStr).Info("Creating pipeline info comment") - if err := cw.ghc.CreateComment(org, repo, number, pullRequestInfoComment+modeStr); err != nil { - logger.WithError(err).Error("failed to create comment") - } else { - logger.Info("Successfully created pipeline info comment") - } - } else { - logger.Debug("No pipeline jobs found, skipping comment creation") - } - } else { - // Manual mode: Check for non-always-run jobs - cfg := cw.configDataProvider.configGetter() - presubmits := cfg.GetPresubmitsStatic(org + "/" + repo) - - hasNonAlwaysRunJobs := false - for _, p := range presubmits { - if !p.AlwaysRun { - hasNonAlwaysRunJobs = true - break - } - } - - if hasNonAlwaysRunJobs { - comment := "There are test jobs defined for this repository which are not configured to run automatically. " + - "Comment `/test ?` to see a list of all defined jobs. Review these jobs and use `/test ` to manually trigger jobs most likely to be impacted by the proposed changes." + - "Comment `/pipeline required` to trigger all required & necessary jobs." - - if err := cw.ghc.CreateComment(org, repo, number, comment); err != nil { - logger.WithError(err).Error("failed to create comment") - } - } - } - } -} - -func (cw *clientWrapper) handleLabelAddition(l *logrus.Entry, event github.PullRequestEvent) { - cw.mu.Lock() - defer cw.mu.Unlock() - - logger := l.WithFields(logrus.Fields{ - "handler": "handleLabelAddition", - "action": event.Action, - "org": event.Repo.Owner.Login, - "repo": event.Repo.Name, - "pr": event.PullRequest.Number, - "label": event.Label.Name, - }) - - logger.Info("Processing label addition event") - - if github.PullRequestActionLabeled == event.Action && event.Label.Name == labels.LGTM { - org := event.Repo.Owner.Login - repo := event.Repo.Name - - logger = logger.WithFields(logrus.Fields{ - "org": org, - "repo": repo, - "pr": event.PullRequest.Number, - }) - - logger.Info("Processing LGTM label addition") - - logger.Debug("Getting LGTM configuration from watcher") - currentCfg := cw.lgtmWatcher.getConfig() - repos, orgExists := currentCfg[org] - _, repoExists := repos[repo] - - logger.WithFields(logrus.Fields{ - "org_exists": orgExists, - "repo_exists": repoExists, - }).Debug("LGTM configuration check results") - - if !orgExists || !repoExists { - logger.Debug("Repository not in LGTM configuration, skipping") - return - } - - // Check if pipeline-auto label is present - if so, skip (reconciler will handle auto triggering) - for _, label := range event.PullRequest.Labels { - if label.Name == PipelineAutoLabel { - logger.Info("PR has pipeline-auto label, skipping LGTM trigger (reconciler will handle automatic triggering)") - return - } - } - - // Check if branch is enabled for this repo - baseBranch := event.PullRequest.Base.Ref - repoConfig := repos[repo] - if !isBranchEnabled(repoConfig.Branches, baseBranch) { - logger.WithField("base_branch", baseBranch).Debug("Branch not enabled for pipeline controller, skipping") - return - } - - prowJob := &v1.ProwJob{ - Spec: v1.ProwJobSpec{ - Refs: &v1.Refs{ - Org: org, - Repo: repo, - BaseRef: event.PullRequest.Base.Ref, - Pulls: []v1.Pull{ - {Number: event.PullRequest.Number, SHA: event.PullRequest.Head.SHA}, - }, - }, - }, - } - - // If SHA is missing, log a warning but continue (status check will be skipped) - if event.PullRequest.Head.SHA == "" { - logger.Warn("PR head SHA is empty, status check will be skipped") - } - - logger.WithFields(logrus.Fields{ - "org": org, - "repo": repo, - "pr_number": event.PullRequest.Number, - "sha": event.PullRequest.Head.SHA, - }).Debug("ProwJob created for LGTM label addition") - logger.Debug("Getting presubmits from config data provider") - presubmits := cw.configDataProvider.GetPresubmits(prowJob.Spec.Refs.Org + "/" + prowJob.Spec.Refs.Repo) - - logger.WithFields(logrus.Fields{ - "protected_count": len(presubmits.protected), - "always_required_count": len(presubmits.alwaysRequired), - "conditionally_required_count": len(presubmits.conditionallyRequired), - "pipeline_conditionally_required_count": len(presubmits.pipelineConditionallyRequired), - "pipeline_skip_only_required_count": len(presubmits.pipelineSkipOnlyRequired), - }).Debug("Presubmits retrieved for LGTM handler") - - hasPresubmits := len(presubmits.protected) > 0 || len(presubmits.alwaysRequired) > 0 || - len(presubmits.conditionallyRequired) > 0 || len(presubmits.pipelineConditionallyRequired) > 0 || - len(presubmits.pipelineSkipOnlyRequired) > 0 - - if !hasPresubmits { - logger.Debug("No presubmits found, skipping comment") - return - } - - logger.Info("Sending comment for LGTM label addition") - if err := sendComment(presubmits, prowJob, cw.ghc, func() {}, cw.pjLister); err != nil { - logger.WithError(err).Error("failed to send a comment") - } else { - logger.Info("Successfully sent comment for LGTM label addition") - } - } -} - -func (cw *clientWrapper) handleIssueComment(l *logrus.Entry, event github.IssueCommentEvent) { - cw.mu.Lock() - defer cw.mu.Unlock() - - logger := l.WithFields(logrus.Fields{ - "handler": "handleIssueComment", - "org": event.Repo.Owner.Login, - "repo": event.Repo.Name, - "issue": event.Issue.Number, - "comment_id": event.Comment.ID, - }) - - // Only handle issue comments on PRs - if !event.Issue.IsPullRequest() { - return - } - - // Check if the comment contains "/pipeline required" or "/pipeline auto" as a command (at start of line) - // Use (?m) for multiline mode so ^ matches start of any line, not just start of string - pipelineRequiredRegex := regexp.MustCompile(`(?im)^/pipeline\s+required`) - pipelineAutoRegex := regexp.MustCompile(`(?im)^/pipeline\s+auto`) - - matchesRequired := pipelineRequiredRegex.MatchString(event.Comment.Body) - matchesAuto := pipelineAutoRegex.MatchString(event.Comment.Body) - - if !matchesRequired && !matchesAuto { - return - } - - org := event.Repo.Owner.Login - repo := event.Repo.Name - number := event.Issue.Number - - logger = logger.WithFields(logrus.Fields{ - "org": org, - "repo": repo, - "pr": number, - }) - - if matchesAuto { - logger.Info("Processing /pipeline auto comment") - } else { - logger.Info("Processing /pipeline required comment") - } - - // Get presubmits for this repo - logger.Debug("Getting presubmits from config data provider") - presubmits := cw.configDataProvider.GetPresubmits(org + "/" + repo) - - // Check if there are any pipeline-controlled jobs - if len(presubmits.protected) == 0 && len(presubmits.alwaysRequired) == 0 && - len(presubmits.conditionallyRequired) == 0 && len(presubmits.pipelineConditionallyRequired) == 0 && - len(presubmits.pipelineSkipOnlyRequired) == 0 { - return - } - - // Check if repo is in configuration (either manual/auto mode or LGTM mode) - currentCfg := cw.watcher.getConfig() - repos, orgExists := currentCfg[org] - _, repoExists := repos[repo] - - lgtmCfg := cw.lgtmWatcher.getConfig() - lgtmRepos, lgtmOrgExists := lgtmCfg[org] - _, lgtmRepoExists := lgtmRepos[repo] - - if (!orgExists || !repoExists) && (!lgtmOrgExists || !lgtmRepoExists) { - if err := cw.ghc.CreateComment(org, repo, number, RepoNotConfiguredMessage); err != nil { - logger.WithError(err).Error("failed to create comment") - } - return - } - - // Fetch PR details - pr, err := cw.ghc.GetPullRequest(org, repo, number) - if err != nil { - logger.WithError(err).Error("failed to get PR details") - return - } - - // Check if branch is enabled for this repo - baseBranch := pr.Base.Ref - var branchEnabled bool - if orgExists && repoExists { - repoConfig := repos[repo] - branchEnabled = isBranchEnabled(repoConfig.Branches, baseBranch) - } else if lgtmOrgExists && lgtmRepoExists { - lgtmRepoConfig := lgtmRepos[repo] - branchEnabled = isBranchEnabled(lgtmRepoConfig.Branches, baseBranch) - } - - if !branchEnabled { - logger.WithField("base_branch", baseBranch).Debug("Branch not enabled for pipeline controller, skipping") - return - } - - // If this is a /pipeline auto command, add the pipeline-auto label and return - // The reconciler will trigger second-stage tests when first-stage tests pass - if matchesAuto { - // /pipeline auto is only available for LGTM-configured repos - if !lgtmOrgExists || !lgtmRepoExists { - comment := "The `/pipeline auto` command is only available for LGTM-mode repositories. " + - "For repositories in automatic mode, second-stage tests are already triggered automatically." - if err := cw.ghc.CreateComment(org, repo, number, comment); err != nil { - logger.WithError(err).Error("failed to create comment") - } - return - } - - if err := cw.ghc.AddLabel(org, repo, number, PipelineAutoLabel); err != nil { - logger.WithError(err).Error("failed to add pipeline-auto label") - return - } - logger.Info("Successfully added pipeline-auto label") - // Cache that this PR has pipeline-auto label - if cw.pipelineAutoCache != nil { - cw.pipelineAutoCache.Set(org, repo, number) - } - comment := "**Pipeline controller notification**\n\nThe `pipeline-auto` label has been added to this PR. Second-stage tests will be triggered automatically when all first-stage tests pass." - if err := cw.ghc.CreateComment(org, repo, number, comment); err != nil { - logger.WithError(err).Error("failed to create confirmation comment") - } - return - } - - // For /pipeline required, trigger tests immediately - // Create a fake ProwJob to reuse existing logic - prowJob := &v1.ProwJob{ - Spec: v1.ProwJobSpec{ - Refs: &v1.Refs{ - Org: org, - Repo: repo, - BaseRef: pr.Base.Ref, - Pulls: []v1.Pull{ - {Number: number, SHA: pr.Head.SHA}, - }, - }, - }, - } - - // Generate the comment with test/override commands - // Pass true for isExplicitCommand since this is an explicit /pipeline required command - if err := sendCommentWithMode(presubmits, prowJob, cw.ghc, func() {}, cw.pjLister, true); err != nil { - logger.WithError(err).Error("failed to send comment in response to /pipeline required") - } -} - -// handlePipelineContextCreation handles PR events (open, push, reopen) and creates contexts for matching tests -func (cw *clientWrapper) handlePipelineContextCreation(l *logrus.Entry, event github.PullRequestEvent) { - cw.mu.Lock() - defer cw.mu.Unlock() - - logger := l.WithFields(logrus.Fields{ - "handler": "handlePipelineContextCreation", - "action": event.Action, - "org": event.Repo.Owner.Login, - "repo": event.Repo.Name, - "pr": event.PullRequest.Number, - }) - - if event.Action != github.PullRequestActionOpened && - event.Action != github.PullRequestActionSynchronize && - event.Action != github.PullRequestActionReopened { - return - } - - org := event.Repo.Owner.Login - repo := event.Repo.Name - number := event.PullRequest.Number - sha := event.PullRequest.Head.SHA - - presubmits := cw.configDataProvider.GetPresubmits(org + "/" + repo) - - if len(presubmits.pipelineConditionallyRequired) == 0 && len(presubmits.pipelineSkipOnlyRequired) == 0 && - len(presubmits.protected) == 0 { - return - } - - // Check if repo is in configuration (either manual/auto mode or LGTM mode) - currentCfg := cw.watcher.getConfig() - repos, orgExists := currentCfg[org] - _, repoExists := repos[repo] - - lgtmCfg := cw.lgtmWatcher.getConfig() - lgtmRepos, lgtmOrgExists := lgtmCfg[org] - _, lgtmRepoExists := lgtmRepos[repo] - - isInConfig := (orgExists && repoExists) || (lgtmOrgExists && lgtmRepoExists) - - if !isInConfig { - return - } - - // Check if branch is enabled for this repo - baseBranch := event.PullRequest.Base.Ref - var branchEnabled bool - if orgExists && repoExists { - repoConfig := repos[repo] - branchEnabled = isBranchEnabled(repoConfig.Branches, baseBranch) - } else if lgtmOrgExists && lgtmRepoExists { - lgtmRepoConfig := lgtmRepos[repo] - branchEnabled = isBranchEnabled(lgtmRepoConfig.Branches, baseBranch) - } - - if !branchEnabled { - logger.WithField("base_branch", baseBranch).Debug("Branch not enabled for pipeline controller, skipping") - return - } - - logger = logger.WithFields(logrus.Fields{ - "org": org, - "repo": repo, - "pr": number, - "sha": sha, - }) - - // Get changed files for this PR - changedFiles, err := cw.ghc.GetPullRequestChanges(org, repo, number) - if err != nil { - logger.WithError(err).Error("failed to get PR changes") - return - } - - filenames := make([]string, 0, len(changedFiles)) - for _, change := range changedFiles { - filenames = append(filenames, change.Filename) - } - - // Filter tests by branch - only process tests that match the target branch - repoBaseRef := repo + "-" + baseBranch - - // Evaluate pipeline_run_if_changed tests - for _, presubmit := range presubmits.pipelineConditionallyRequired { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - if pattern, ok := presubmit.Annotations["pipeline_run_if_changed"]; ok && pattern != "" { - if shouldRun, err := matchesPattern(pattern, filenames); err != nil { - logger.WithError(err).WithField("test", presubmit.Name).WithField("pattern", pattern).Error("failed to evaluate pattern") - continue - } else if shouldRun { - if err := cw.createContext(org, repo, sha, presubmit.Context, "pending", PipelinePendingMessage); err != nil { - logger.WithError(err).WithField("test", presubmit.Name).Error("failed to create context") - } else { - logger.WithField("test", presubmit.Name).Info("created pending context for pipeline test") - } - } - } - } - - // Evaluate pipeline_skip_if_only_changed tests - for _, presubmit := range presubmits.pipelineSkipOnlyRequired { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - if pattern, ok := presubmit.Annotations["pipeline_skip_if_only_changed"]; ok && pattern != "" { - if shouldSkip, err := allFilesMatchPattern(pattern, filenames); err != nil { - logger.WithError(err).WithField("test", presubmit.Name).WithField("pattern", pattern).Error("failed to evaluate skip pattern") - continue - } else if !shouldSkip { - // If not all files match the skip pattern, we should run the test - if err := cw.createContext(org, repo, sha, presubmit.Context, "pending", PipelinePendingMessage); err != nil { - logger.WithError(err).WithField("test", presubmit.Name).Error("failed to create context") - } else { - logger.WithField("test", presubmit.Name).Info("created pending context for pipeline test") - } - } - } - } - - // Create contexts for protected jobs (always_run: false, optional: false, no run conditions) - for _, presubmit := range presubmits.protected { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - if err := cw.createContext(org, repo, sha, presubmit.Context, "pending", PipelinePendingMessage); err != nil { - logger.WithError(err).WithField("test", presubmit.Name).Error("failed to create context") - } else { - logger.WithField("test", presubmit.Name).Info("created pending context for protected test") - } - } -} - -// createContext creates a GitHub status context -func (cw *clientWrapper) createContext(org, repo, sha, context, state, description string) error { - return cw.ghc.CreateStatus(org, repo, sha, github.Status{ - Context: context, - State: state, - Description: description, - }) -} - -// matchesPattern checks if any of the filenames match the given regex pattern -func matchesPattern(pattern string, filenames []string) (bool, error) { - if pattern == "" { - return false, nil - } - - regex, err := regexp.Compile(pattern) - if err != nil { - return false, fmt.Errorf("invalid regex pattern %q: %w", pattern, err) - } - - for _, filename := range filenames { - if regex.MatchString(filename) { - return true, nil - } - } - - return false, nil -} - -// allFilesMatchPattern checks if ALL filenames match the given regex pattern -func allFilesMatchPattern(pattern string, filenames []string) (bool, error) { - if pattern == "" { - return false, nil - } - - if len(filenames) == 0 { - return false, nil - } - - regex, err := regexp.Compile(pattern) - if err != nil { - return false, fmt.Errorf("invalid regex pattern %q: %w", pattern, err) - } - - for _, filename := range filenames { - if !regex.MatchString(filename) { - return false, nil - } - } - - return true, nil -} - -func main() { - logrusutil.ComponentInit() - logger := logrus.WithField("component", "pipeline-controller") - ctrlruntimelog.SetLogger(logrusr.New(logger)) - - o := parseOptions() - - configAgent, err := o.config.ConfigAgent() - if err != nil { - logger.WithError(err).Fatal("error starting config agent") - } - cfg := configAgent.Config - - restCfg, err := o.client.InfrastructureClusterConfig(o.dryrun) - if err != nil { - logger.WithError(err).Fatal("failed to get kubeconfig") - } - mgr, err := manager.New(restCfg, manager.Options{ - Cache: cache.Options{ - DefaultNamespaces: map[string]cache.Config{ - cfg().ProwJobNamespace: {}, - }, - }, - Metrics: server.Options{ - BindAddress: "0", - }, - }) - if err != nil { - logger.WithError(err).Fatal("failed to create manager") - } - - if err := o.client.AddKubeconfigChangeCallback(func() { - logger.Info("kubeconfig changed, exiting to trigger a restart") - interrupts.Terminate() - }); err != nil { - logger.WithError(err).Fatal("failed to register kubeconfig callback") - } - - githubClient, err := o.github.GitHubClient(o.dryrun) - if err != nil { - logger.WithError(err).Fatal("error getting GitHub client") - } - - watcher := newWatcher(o.configFile, logger) - go watcher.watch() - - lgtmWatcher := newWatcher(o.lgtmConfigFile, logger) - go lgtmWatcher.watch() - - // Create a function that returns repos from both config and lgtm config - repoLister := func() []string { - var repos []string - - // Get repos from main config - mainConfig := watcher.getConfig() - for org, repoConfigs := range mainConfig { - for repo := range repoConfigs { - repos = append(repos, org+"/"+repo) - } - } - - // Get repos from lgtm config - lgtmConfig := lgtmWatcher.getConfig() - for org, repoConfigs := range lgtmConfig { - for repo := range repoConfigs { - orgRepo := org + "/" + repo - // Avoid duplicates - found := false - for _, existing := range repos { - if existing == orgRepo { - found = true - break - } - } - if !found { - repos = append(repos, orgRepo) - } - } - } - - // If no repos found, retry once after a short delay - if len(repos) == 0 { - time.Sleep(100 * time.Millisecond) - - // Retry getting configs - mainConfig = watcher.getConfig() - lgtmConfig = lgtmWatcher.getConfig() - - for org, repoConfigs := range mainConfig { - for repo := range repoConfigs { - repos = append(repos, org+"/"+repo) - } - } - - for org, repoConfigs := range lgtmConfig { - for repo := range repoConfigs { - orgRepo := org + "/" + repo - found := false - for _, existing := range repos { - if existing == orgRepo { - found = true - break - } - } - if !found { - repos = append(repos, orgRepo) - } - } - } - } - - return repos - } - - configDataProvider := NewConfigDataProvider(cfg, repoLister, logger.WithField("component", "config-data-provider")) - go configDataProvider.Run() - - // Wait for config data provider to be ready - logger.Info("Waiting for config data provider to be ready...") - time.Sleep(2 * time.Second) // Give it time to load initial data - logger.Info("Config data provider should be ready") - - pipelineAutoCache := NewPipelineAutoCache() - go func() { - ticker := time.NewTicker(1 * time.Hour) - defer ticker.Stop() - for range ticker.C { - pipelineAutoCache.CleanExpired() - } - }() - - reconciler, err := NewReconciler(mgr, configDataProvider, githubClient, logger, watcher, lgtmWatcher, pipelineAutoCache) - if err != nil { - logger.WithError(err).Fatal("failed to construct github reporter controller") - } - go reconciler.cleanOldIds(24 * time.Hour) - - if err = secret.Add(o.webhookSecretFile); err != nil { - logger.WithError(err).Fatal("error starting secrets agent") - } - webhookTokenGenerator := secret.GetTokenGenerator(o.webhookSecretFile) - - cw := &clientWrapper{ - ghc: githubClient, - configDataProvider: configDataProvider, - watcher: watcher, - lgtmWatcher: lgtmWatcher, - pjLister: mgr.GetCache(), - pipelineAutoCache: pipelineAutoCache, - } - - eventServer := githubeventserver.New(o.githubEventServerOptions, webhookTokenGenerator, logger) - - // Register event handlers with proper logging - logger.Info("Registering event handlers") - eventServer.RegisterHandlePullRequestEvent(cw.handlePullRequestCreation) - eventServer.RegisterHandlePullRequestEvent(cw.handleLabelAddition) - eventServer.RegisterHandlePullRequestEvent(cw.handlePipelineContextCreation) - eventServer.RegisterHandleIssueCommentEvent(cw.handleIssueComment) - - logger.Info("All event handlers registered successfully") - - interrupts.OnInterrupt(func() { - eventServer.GracefulShutdown() - }) - - interrupts.ListenAndServe(eventServer, time.Second*30) - interrupts.Run(func(ctx context.Context) { - if err := mgr.Start(ctx); err != nil { - logger.WithError(err).Fatal("controller manager exited with error") - } - }) - interrupts.WaitForGracefulShutdown() -} diff --git a/cmd/pipeline-controller/main_test.go b/cmd/pipeline-controller/main_test.go deleted file mode 100644 index a28341597fe..00000000000 --- a/cmd/pipeline-controller/main_test.go +++ /dev/null @@ -1,948 +0,0 @@ -package main - -import ( - "errors" - "io" - "strings" - "testing" - - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/labels" -) - -// testLoggerMain creates a discarded logger for tests -func testLoggerMain() *logrus.Entry { - logger := logrus.New() - logger.SetOutput(io.Discard) - return logrus.NewEntry(logger) -} - -// Test constants for expected comment messages -const ( - testPipelineRequiredResponse = "Scheduling required tests:" -) - -// fakeGhClient is a fake GitHub client for testing -type fakeGhClientWithComment struct { - comment string - error error - pr *github.PullRequest - labelsAdded []string - existingLabels []github.Label -} - -func (f *fakeGhClientWithComment) GetPullRequest(org, repo string, number int) (*github.PullRequest, error) { - if f.error != nil { - return nil, f.error - } - if f.pr != nil { - return f.pr, nil - } - return &github.PullRequest{State: github.PullRequestStateOpen}, nil -} -func (f *fakeGhClientWithComment) CreateComment(owner, repo string, number int, comment string) error { - f.comment = comment - return nil -} -func (f *fakeGhClientWithComment) GetPullRequestChanges(org, repo string, number int) ([]github.PullRequestChange, error) { - return []github.PullRequestChange{}, nil -} - -func (f *fakeGhClientWithComment) CreateStatus(org, repo, ref string, s github.Status) error { - return nil -} - -func (f *fakeGhClientWithComment) AddLabel(org, repo string, number int, label string) error { - f.labelsAdded = append(f.labelsAdded, label) - return nil -} - -func (f *fakeGhClientWithComment) GetIssueLabels(org, repo string, number int) ([]github.Label, error) { - return f.existingLabels, nil -} - -// Helper function to create repo config for tests -func createRepoConfig(name string, trigger string) RepoItem { - item := RepoItem{ - Name: name, - } - item.Mode.Trigger = trigger - return item -} - -func TestHandleLabelAddition_RealFunctions(t *testing.T) { - org := "openshift" - repo := "assisted-installer" - baseRef := "master" - prNumber := 1 - - basicEvent := github.PullRequestEvent{ - Action: github.PullRequestActionLabeled, - Label: github.Label{Name: labels.LGTM}, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - PullRequest: github.PullRequest{ - Number: prNumber, - Base: github.PullRequestBranch{Ref: baseRef}, - }, - } - - tests := []struct { - name string - event github.PullRequestEvent - configData map[string]presubmitTests - existingLabels []github.Label - expectCommentCall bool - expectedComment string - }{ - { - name: "action not labeled: do nothing", - event: github.PullRequestEvent{ - Action: "opened", - }, - expectCommentCall: false, - }, - { - name: "label not LGTM: do nothing", - event: github.PullRequestEvent{ - Action: github.PullRequestActionLabeled, - Label: github.Label{Name: "not-lgtm"}, - }, - expectCommentCall: false, - }, - { - name: "presubmits exist, sendComment succeeds", - event: basicEvent, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{Name: "pull-ci-openshift-assisted-installer-master-dummy-test"}, - RerunCommand: "/test dummy-test", - } - p.Context = "dummy-test" - return []config.Presubmit{p} - }(), - }, - }, - expectCommentCall: true, - expectedComment: "dummy-test", - }, - { - name: "pipeline-auto label present: skip LGTM trigger", - event: github.PullRequestEvent{ - Action: github.PullRequestActionLabeled, - Label: github.Label{Name: labels.LGTM}, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - PullRequest: github.PullRequest{ - Number: prNumber, - Base: github.PullRequestBranch{Ref: baseRef}, - Labels: []github.Label{{Name: PipelineAutoLabel}}, // PR already has pipeline-auto label - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{Name: "pull-ci-openshift-assisted-installer-master-dummy-test"}, - RerunCommand: "/test dummy-test", - } - p.Context = "dummy-test" - return []config.Presubmit{p} - }(), - }, - }, - expectCommentCall: false, // Should skip because pipeline-auto is present - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ghc := &fakeGhClientWithComment{existingLabels: tc.existingLabels} - - cw := &clientWrapper{ - lgtmWatcher: &watcher{config: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{ - createRepoConfig(repo, "auto"), - }, - }, - }}}, - configDataProvider: &ConfigDataProvider{ - updatedPresubmits: tc.configData, - logger: testLoggerMain(), - }, - ghc: ghc, - } - - entry := logrus.NewEntry(logrus.New()) - cw.handleLabelAddition(entry, tc.event) - - if tc.expectCommentCall { - if ghc.comment == "" { - t.Errorf("expected CreateComment to be called, but no comment was recorded") - } else { - if !strings.Contains(ghc.comment, tc.expectedComment) { - t.Errorf("expected comment to contain %q, got %q", tc.expectedComment, ghc.comment) - } - } - } else { - if ghc.comment != "" { - t.Errorf("expected CreateComment not to be called, but got comment %q", ghc.comment) - } - } - }) - } -} - -func TestHandleIssueComment(t *testing.T) { - org := "openshift" - repo := "test-repo" - prNumber := 123 - - basicEvent := github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/pipeline required", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, // Non-nil means it's a PR - }, - } - - tests := []struct { - name string - event github.IssueCommentEvent - configData map[string]presubmitTests - watcherConfig enabledConfig - ghPR *github.PullRequest - ghError error - expectCommentCall bool - expectedComment string - }{ - { - name: "not a PR: do nothing", - event: github.IssueCommentEvent{ - Issue: github.Issue{ - PullRequest: nil, - }, - }, - expectCommentCall: false, - }, - { - name: "comment doesn't contain /pipeline required: do nothing", - event: github.IssueCommentEvent{ - Comment: github.IssueComment{ - Body: "This is just a regular comment", - }, - Issue: github.Issue{ - PullRequest: &struct{}{}, - }, - }, - expectCommentCall: false, - }, - { - name: "comment contains /pipeline required in middle of text: do nothing", - event: github.IssueCommentEvent{ - Comment: github.IssueComment{ - Body: "Comment `/pipeline required` to trigger all required & necessary jobs.", - }, - Issue: github.Issue{ - PullRequest: &struct{}{}, - }, - }, - expectCommentCall: false, - }, - { - name: "whitespace variation: double space", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/pipeline required", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectCommentCall: true, - expectedComment: testPipelineRequiredResponse, - }, - { - name: "command on new line in multiline comment: should trigger", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "Some text here\n/pipeline required", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectCommentCall: true, - expectedComment: testPipelineRequiredResponse, - }, - { - name: "whitespace variation: tab", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/pipeline\trequired", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectCommentCall: true, - expectedComment: testPipelineRequiredResponse, - }, - { - name: "case insensitive: uppercase", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/PIPELINE REQUIRED", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectCommentCall: true, - expectedComment: testPipelineRequiredResponse, - }, - { - name: "no pipeline-controlled jobs: do nothing", - event: basicEvent, - configData: map[string]presubmitTests{ - org + "/" + repo: {}, - }, - expectCommentCall: false, - }, - { - name: "has pipeline jobs, responds with test and override commands", - event: basicEvent, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: repo + "-master-test", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go", - }, - }, - RerunCommand: "/test test", - Reporter: config.Reporter{ - Context: "test", - }, - }, - }, - }, - }, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectCommentCall: true, - expectedComment: testPipelineRequiredResponse, - }, - { - name: "error getting PR: do nothing", - event: basicEvent, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - ghError: errors.New("failed to get PR"), - expectCommentCall: false, - }, - { - name: "repo not configured: responds with not configured message", - event: basicEvent, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - watcherConfig: enabledConfig{}, // Empty config means repo not configured - expectCommentCall: true, - expectedComment: RepoNotConfiguredMessage, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ghc := &fakeGhClientWithComment{} - if tc.ghError != nil { - ghc = &fakeGhClientWithComment{error: tc.ghError} - } else if tc.ghPR != nil { - ghc = &fakeGhClientWithComment{pr: tc.ghPR} - } - - cw := &clientWrapper{ - configDataProvider: &ConfigDataProvider{ - updatedPresubmits: tc.configData, - logger: testLoggerMain(), - }, - ghc: ghc, - watcher: &watcher{config: tc.watcherConfig}, - lgtmWatcher: &watcher{config: enabledConfig{}}, - } - - // If no watcherConfig is provided, use a default config with the repo configured - if len(tc.watcherConfig.Orgs) == 0 && tc.expectedComment != RepoNotConfiguredMessage { - cw.watcher.config = enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{ - createRepoConfig(repo, "auto"), - }, - }, - }} - } - - entry := logrus.NewEntry(logrus.New()) - cw.handleIssueComment(entry, tc.event) - - if tc.expectCommentCall { - if ghc.comment == "" { - t.Errorf("expected CreateComment to be called, but no comment was recorded") - } else { - if !strings.Contains(ghc.comment, tc.expectedComment) { - t.Errorf("expected comment to contain %q, got %q", tc.expectedComment, ghc.comment) - } - // Check for protected job listing format if not the "not configured" message - if tc.expectedComment != RepoNotConfiguredMessage && - !strings.Contains(ghc.comment, "Scheduling required tests:") { - t.Errorf("expected comment to contain 'Scheduling required tests:', got %q", ghc.comment) - } - } - } else { - if ghc.comment != "" { - t.Errorf("expected CreateComment not to be called, but got comment %q", ghc.comment) - } - } - }) - } -} - -func TestHandleIssueCommentPipelineAuto(t *testing.T) { - org := "openshift" - repo := "test-repo" - prNumber := 123 - - tests := []struct { - name string - event github.IssueCommentEvent - configData map[string]presubmitTests - watcherConfig enabledConfig - lgtmWatcherConfig enabledConfig - ghPR *github.PullRequest - expectLabelAdded bool - expectCommentCall bool - expectedComment string - notExpectedComment string - }{ - { - name: "/pipeline auto adds label and confirmation comment", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/pipeline auto", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - lgtmWatcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{{Name: repo}}, - }, - }}, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectLabelAdded: true, - expectCommentCall: true, - expectedComment: "pipeline-auto", - notExpectedComment: "Scheduling required tests", - }, - { - name: "/pipeline auto with uppercase", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/PIPELINE AUTO", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - lgtmWatcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{{Name: repo}}, - }, - }}, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectLabelAdded: true, - expectCommentCall: true, - expectedComment: "pipeline-auto", - }, - { - name: "/pipeline auto does not trigger tests immediately", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/pipeline auto", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - lgtmWatcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{{Name: repo}}, - }, - }}, - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectLabelAdded: true, - expectCommentCall: true, - expectedComment: "will be triggered automatically when all first-stage tests pass", - notExpectedComment: "/test protected-test", - }, - { - name: "repo not configured: responds with not configured message", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/pipeline auto", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - watcherConfig: enabledConfig{}, - lgtmWatcherConfig: enabledConfig{}, - expectLabelAdded: false, - expectCommentCall: true, - expectedComment: RepoNotConfiguredMessage, - }, - { - name: "/pipeline auto on auto-mode repo (not LGTM): should be rejected", - event: github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: "/pipeline auto", - }, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - Issue: github.Issue{ - Number: prNumber, - PullRequest: &struct{}{}, - }, - }, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - watcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{{Name: repo, Mode: struct{ Trigger string }{Trigger: "auto"}}}, - }, - }}, - lgtmWatcherConfig: enabledConfig{}, // Not in LGTM config - ghPR: &github.PullRequest{ - Base: github.PullRequestBranch{Ref: "master"}, - Head: github.PullRequestBranch{SHA: "abc123"}, - }, - expectLabelAdded: false, - expectCommentCall: true, - expectedComment: "only available for LGTM-mode repositories", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ghc := &fakeGhClientWithComment{} - if tc.ghPR != nil { - ghc = &fakeGhClientWithComment{pr: tc.ghPR} - } - - cw := &clientWrapper{ - configDataProvider: &ConfigDataProvider{ - updatedPresubmits: tc.configData, - logger: testLoggerMain(), - }, - ghc: ghc, - watcher: &watcher{config: tc.watcherConfig}, - lgtmWatcher: &watcher{config: tc.lgtmWatcherConfig}, - } - - entry := logrus.NewEntry(logrus.New()) - cw.handleIssueComment(entry, tc.event) - - // Check label was added - if tc.expectLabelAdded { - found := false - for _, label := range ghc.labelsAdded { - if label == PipelineAutoLabel { - found = true - break - } - } - if !found { - t.Errorf("expected pipeline-auto label to be added, but it wasn't") - } - } else { - for _, label := range ghc.labelsAdded { - if label == PipelineAutoLabel { - t.Errorf("expected pipeline-auto label NOT to be added, but it was") - } - } - } - - // Check comment - if tc.expectCommentCall { - if ghc.comment == "" { - t.Errorf("expected CreateComment to be called, but no comment was recorded") - } else { - if tc.expectedComment != "" && !strings.Contains(ghc.comment, tc.expectedComment) { - t.Errorf("expected comment to contain %q, got %q", tc.expectedComment, ghc.comment) - } - if tc.notExpectedComment != "" && strings.Contains(ghc.comment, tc.notExpectedComment) { - t.Errorf("expected comment NOT to contain %q, got %q", tc.notExpectedComment, ghc.comment) - } - } - } else { - if ghc.comment != "" { - t.Errorf("expected CreateComment not to be called, but got comment %q", ghc.comment) - } - } - }) - } -} - -func TestHandlePullRequestCreation(t *testing.T) { - org := "openshift" - repo := "test-repo" - prNumber := 123 - - basicEvent := github.PullRequestEvent{ - Action: github.PullRequestActionOpened, - Repo: github.Repo{ - Owner: github.User{Login: org}, - Name: repo, - }, - PullRequest: github.PullRequest{ - Number: prNumber, - }, - } - - tests := []struct { - name string - event github.PullRequestEvent - watcherConfig enabledConfig - configData map[string]presubmitTests - presubmits []config.Presubmit - configGetter config.Getter - expectCommentCall bool - expectedComment string - }{ - { - name: "action not opened: do nothing", - event: github.PullRequestEvent{ - Action: "closed", - }, - expectCommentCall: false, - }, - { - name: "automatic pipeline repo with jobs: shows pipeline info", - event: basicEvent, - watcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{ - createRepoConfig(repo, "auto"), - }, - }, - }}, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - expectCommentCall: true, - expectedComment: pullRequestInfoComment, - }, - { - name: "automatic pipeline repo without jobs: no comment", - event: basicEvent, - watcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{ - createRepoConfig(repo, "auto"), - }, - }, - }}, - configData: map[string]presubmitTests{ - org + "/" + repo: {}, - }, - expectCommentCall: false, - }, - { - name: "non-configured repo: no comment regardless of jobs", - event: basicEvent, - watcherConfig: enabledConfig{}, // Empty config means not configured - configGetter: func() *config.Config { - return &config.Config{ - JobConfig: config.JobConfig{PresubmitsStatic: map[string][]config.Presubmit{ - org + "/" + repo: { - { - JobBase: config.JobBase{Name: "manual-test"}, - AlwaysRun: false, - }, - }, - }}, - } - }, - expectCommentCall: false, - }, - { - name: "manual pipeline repo with jobs: no comment for manual mode", - event: basicEvent, - watcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{ - createRepoConfig(repo, "manual"), - }, - }, - }}, - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - expectCommentCall: false, - }, - { - name: "LGTM mode repo with jobs: shows pipeline info", - event: basicEvent, - watcherConfig: enabledConfig{}, // Empty regular config - configData: map[string]presubmitTests{ - org + "/" + repo: { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "pull-ci-openshift-test-repo-master-protected-test"}, RerunCommand: "/test protected-test"}}, - }, - }, - expectCommentCall: true, - expectedComment: pullRequestInfoComment, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ghc := &fakeGhClientWithComment{} - - configGetter := tc.configGetter - if configGetter == nil { - configGetter = func() *config.Config { - return &config.Config{} - } - } - - lgtmConfig := enabledConfig{} - if tc.name == "LGTM mode repo with jobs: shows pipeline info" { - lgtmConfig = enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: org, - Repos: []RepoItem{{Name: repo}}, - }, - }} - } - - cw := &clientWrapper{ - watcher: &watcher{config: tc.watcherConfig}, - lgtmWatcher: &watcher{config: lgtmConfig}, - configDataProvider: &ConfigDataProvider{ - updatedPresubmits: tc.configData, - configGetter: configGetter, - logger: testLoggerMain(), - }, - ghc: ghc, - } - - entry := logrus.NewEntry(logrus.New()) - cw.handlePullRequestCreation(entry, tc.event) - - if tc.expectCommentCall { - if ghc.comment == "" { - t.Errorf("expected CreateComment to be called, but no comment was recorded") - } else { - if !strings.Contains(ghc.comment, tc.expectedComment) { - t.Errorf("expected comment to contain %q, got %q", tc.expectedComment, ghc.comment) - } - } - } else { - if ghc.comment != "" { - t.Errorf("expected CreateComment not to be called, but got comment %q", ghc.comment) - } - } - }) - } -} diff --git a/cmd/pipeline-controller/pipeline_auto_cache.go b/cmd/pipeline-controller/pipeline_auto_cache.go deleted file mode 100644 index 54595c9dfbb..00000000000 --- a/cmd/pipeline-controller/pipeline_auto_cache.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -import ( - "fmt" - "sync" - "time" -) - -// PipelineAutoCache caches which PRs have the pipeline-auto label -// Key format: "org/repo/number" -// Entries are automatically cleaned up after 48 hours -type PipelineAutoCache struct { - cache sync.Map -} - -type pipelineAutoCacheEntry struct { - addedAt time.Time -} - -const pipelineAutoCacheTTL = 48 * time.Hour - -func NewPipelineAutoCache() *PipelineAutoCache { - return &PipelineAutoCache{} -} - -func (c *PipelineAutoCache) composePRKey(org, repo string, number int) string { - return fmt.Sprintf("%s/%s/%d", org, repo, number) -} - -// Set marks a PR as having the pipeline-auto label -func (c *PipelineAutoCache) Set(org, repo string, number int) { - c.cache.Store(c.composePRKey(org, repo, number), pipelineAutoCacheEntry{addedAt: time.Now()}) -} - -// Has checks if a PR is known to have the pipeline-auto label -func (c *PipelineAutoCache) Has(org, repo string, number int) bool { - val, ok := c.cache.Load(c.composePRKey(org, repo, number)) - if !ok { - return false - } - entry := val.(pipelineAutoCacheEntry) - // Check if entry is still valid (not expired) - if time.Since(entry.addedAt) > pipelineAutoCacheTTL { - // Entry expired, remove it - c.cache.Delete(c.composePRKey(org, repo, number)) - return false - } - return true -} - -// CleanExpired removes all expired entries from the cache -func (c *PipelineAutoCache) CleanExpired() { - c.cache.Range(func(key, value interface{}) bool { - entry := value.(pipelineAutoCacheEntry) - if time.Since(entry.addedAt) > pipelineAutoCacheTTL { - c.cache.Delete(key) - } - return true - }) -} diff --git a/cmd/pipeline-controller/pipeline_context_test.go b/cmd/pipeline-controller/pipeline_context_test.go deleted file mode 100644 index 5210e058d81..00000000000 --- a/cmd/pipeline-controller/pipeline_context_test.go +++ /dev/null @@ -1,720 +0,0 @@ -package main - -import ( - "fmt" - "io" - "testing" - - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/github" -) - -// testLoggerPipeline creates a discarded logger for tests -func testLoggerPipeline() *logrus.Entry { - logger := logrus.New() - logger.SetOutput(io.Discard) - return logrus.NewEntry(logger) -} - -func TestMatchesPattern(t *testing.T) { - tests := []struct { - name string - pattern string - filenames []string - expected bool - expectErr bool - }{ - { - name: "empty pattern returns false", - pattern: "", - filenames: []string{"file.go"}, - expected: false, - expectErr: false, - }, - { - name: "simple go file pattern matches", - pattern: ".*\\.go$", - filenames: []string{"main.go", "test.txt"}, - expected: true, - expectErr: false, - }, - { - name: "go file pattern doesn't match txt files", - pattern: ".*\\.go$", - filenames: []string{"readme.txt", "docs.md"}, - expected: false, - expectErr: false, - }, - { - name: "docs pattern matches docs directory", - pattern: "^docs/.*", - filenames: []string{"docs/readme.md", "src/main.go"}, - expected: true, - expectErr: false, - }, - { - name: "invalid regex pattern returns error", - pattern: "[", - filenames: []string{"file.go"}, - expected: false, - expectErr: true, - }, - { - name: "empty filenames list returns false", - pattern: ".*\\.go$", - filenames: []string{}, - expected: false, - expectErr: false, - }, - { - name: "multiple patterns - first matches", - pattern: "(cmd|pkg)/.*\\.go$", - filenames: []string{"cmd/main.go", "docs/readme.md"}, - expected: true, - expectErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := matchesPattern(tt.pattern, tt.filenames) - - if tt.expectErr && err == nil { - t.Errorf("expected error but got none") - return - } - if !tt.expectErr && err != nil { - t.Errorf("unexpected error: %v", err) - return - } - if result != tt.expected { - t.Errorf("matchesPattern() = %v, expected %v", result, tt.expected) - } - }) - } -} - -type fakeGhClientForContext struct { - createdStatuses []github.Status - changedFiles []github.PullRequestChange - pullReqChangesErr error -} - -func (f *fakeGhClientForContext) GetPullRequest(org, repo string, number int) (*github.PullRequest, error) { - return &github.PullRequest{State: github.PullRequestStateOpen}, nil -} - -func (f *fakeGhClientForContext) CreateComment(org, repo string, number int, comment string) error { - return nil -} - -func (f *fakeGhClientForContext) GetPullRequestChanges(org string, repo string, number int) ([]github.PullRequestChange, error) { - if f.pullReqChangesErr != nil { - return nil, f.pullReqChangesErr - } - return f.changedFiles, nil -} - -func (f *fakeGhClientForContext) CreateStatus(org, repo, ref string, s github.Status) error { - f.createdStatuses = append(f.createdStatuses, s) - return nil -} - -func (f *fakeGhClientForContext) AddLabel(org, repo string, number int, label string) error { - return nil -} - -func (f *fakeGhClientForContext) GetIssueLabels(org, repo string, number int) ([]github.Label, error) { - return []github.Label{}, nil -} - -func TestHandlePipelineContextCreation(t *testing.T) { - tests := []struct { - name string - action github.PullRequestEventAction - presubmits presubmitTests - changedFiles []github.PullRequestChange - expectedContexts int - }{ - { - name: "PR opened with matching pipeline_run_if_changed", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - pipelineConditionallyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-test-go", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go$", - }, - }, - } - p.Context = "ci/prow/test-go" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "main.go"}, - {Filename: "readme.md"}, - }, - expectedContexts: 1, - }, - { - name: "PR synchronized with non-matching pipeline_run_if_changed", - action: github.PullRequestActionSynchronize, - presubmits: presubmitTests{ - pipelineConditionallyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-test-go", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go$", - }, - }, - } - p.Context = "ci/prow/test-go" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "readme.md"}, - {Filename: "docs.txt"}, - }, - expectedContexts: 0, - }, - { - name: "PR reopened with pipeline_skip_only_if_changed - should run", - action: github.PullRequestActionReopened, - presubmits: presubmitTests{ - pipelineSkipOnlyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-test-important", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": ".*\\.md$", - }, - }, - } - p.Context = "ci/prow/test-important" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "main.go"}, - {Filename: "readme.md"}, - }, - expectedContexts: 1, // Should run because non-md files changed - }, - { - name: "PR opened with pipeline_skip_only_if_changed - should skip", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - pipelineSkipOnlyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-test-important", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": ".*\\.md$", - }, - }, - } - p.Context = "ci/prow/test-important" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "readme.md"}, - {Filename: "docs.md"}, - }, - expectedContexts: 0, // Should skip because only md files changed - }, - { - name: "wrong action - should do nothing", - action: github.PullRequestActionClosed, - presubmits: presubmitTests{ - pipelineConditionallyRequired: []config.Presubmit{ - { - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-test-go", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*", - }, - }, - }, - }, - }, - changedFiles: []github.PullRequestChange{ - {Filename: "main.go"}, - }, - expectedContexts: 0, - }, - { - name: "PR opened with protected jobs - always create contexts", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - protected: func() []config.Presubmit { - p1 := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-main-protected-1"}} - p1.Context = "ci/prow/protected-1" - p2 := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-main-protected-2"}} - p2.Context = "ci/prow/protected-2" - return []config.Presubmit{p1, p2} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "main.go"}, - }, - expectedContexts: 2, - }, - { - name: "PR opened with mixed job types", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - protected: func() []config.Presubmit { - p := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-main-protected"}} - p.Context = "ci/prow/protected" - return []config.Presubmit{p} - }(), - pipelineConditionallyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-pipeline", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go$", - }, - }, - } - p.Context = "ci/prow/pipeline" - return []config.Presubmit{p} - }(), - pipelineSkipOnlyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-skip-docs", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": ".*\\.md$", - }, - }, - } - p.Context = "ci/prow/skip-docs" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "main.go"}, - {Filename: "readme.md"}, - }, - expectedContexts: 3, // protected + pipeline (matches go) + skip (mixed files) - }, - { - name: "PR synchronized with only protected jobs", - action: github.PullRequestActionSynchronize, - presubmits: presubmitTests{ - protected: func() []config.Presubmit { - p := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-main-always-required"}} - p.Context = "ci/prow/always-required" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "config.yaml"}, - }, - expectedContexts: 1, - }, - { - name: "PR opened with job from different branch - should not create context", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - protected: func() []config.Presubmit { - // This job is for release-4.16 branch, but PR is targeting main - p := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-release-4.16-e2e-aws"}} - p.Context = "ci/prow/e2e-aws" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "main.go"}, - }, - expectedContexts: 0, // Job doesn't match branch - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fakeClient := &fakeGhClientForContext{ - changedFiles: tt.changedFiles, - } - - configProvider := &ConfigDataProvider{ - updatedPresubmits: map[string]presubmitTests{ - "test-org/test-repo": tt.presubmits, - }, - logger: testLoggerPipeline(), - } - - watcherVar := &watcher{ - config: enabledConfig{ - Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: "test-org", - Repos: []RepoItem{{Name: "test-repo"}}, - }, - }, - }, - } - - lgtmWatcher := &watcher{config: enabledConfig{}} - cw := &clientWrapper{ - ghc: fakeClient, - configDataProvider: configProvider, - watcher: watcherVar, - lgtmWatcher: lgtmWatcher, - } - - event := github.PullRequestEvent{ - Action: tt.action, - Repo: github.Repo{ - Owner: github.User{Login: "test-org"}, - Name: "test-repo", - }, - PullRequest: github.PullRequest{ - Number: 123, - Head: github.PullRequestBranch{SHA: "abc123"}, - Base: github.PullRequestBranch{Ref: "main"}, - }, - } - - logger := logrus.NewEntry(logrus.New()) - cw.handlePipelineContextCreation(logger, event) - - if len(fakeClient.createdStatuses) != tt.expectedContexts { - t.Errorf("expected %d contexts created, got %d", tt.expectedContexts, len(fakeClient.createdStatuses)) - } - - // Verify that created contexts have correct state - for _, status := range fakeClient.createdStatuses { - if status.State != "pending" { - t.Errorf("expected status state 'pending', got '%s'", status.State) - } - if status.Description != PipelinePendingMessage { - t.Errorf("unexpected status description: %s", status.Description) - } - } - }) - } -} - -func TestAllFilesMatchPattern(t *testing.T) { - tests := []struct { - name string - pattern string - filenames []string - expected bool - expectErr bool - }{ - { - name: "empty pattern returns false", - pattern: "", - filenames: []string{"file.go"}, - expected: false, - expectErr: false, - }, - { - name: "all files match pattern", - pattern: ".*\\.md$", - filenames: []string{"readme.md", "docs.md"}, - expected: true, - expectErr: false, - }, - { - name: "not all files match pattern", - pattern: ".*\\.md$", - filenames: []string{"readme.md", "main.go"}, - expected: false, - expectErr: false, - }, - { - name: "empty filenames list returns false", - pattern: ".*\\.md$", - filenames: []string{}, - expected: false, - expectErr: false, - }, - { - name: "invalid regex pattern returns error", - pattern: "[", - filenames: []string{"file.md"}, - expected: false, - expectErr: true, - }, - { - name: "single file matches", - pattern: "^docs/.*\\.md$", - filenames: []string{"docs/readme.md"}, - expected: true, - expectErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := allFilesMatchPattern(tt.pattern, tt.filenames) - - if tt.expectErr && err == nil { - t.Errorf("expected error but got none") - return - } - if !tt.expectErr && err != nil { - t.Errorf("unexpected error: %v", err) - return - } - if result != tt.expected { - t.Errorf("allFilesMatchPattern() = %v, expected %v", result, tt.expected) - } - }) - } -} - -func TestHandlePipelineContextCreationEdgeCases(t *testing.T) { - tests := []struct { - name string - action github.PullRequestEventAction - presubmits presubmitTests - changedFiles []github.PullRequestChange - pullReqChangesErr error - expectedContexts int - expectedNames []string - shouldSkipExecution bool - }{ - { - name: "no presubmits at all - should do nothing", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - protected: []config.Presubmit{}, - pipelineConditionallyRequired: []config.Presubmit{}, - pipelineSkipOnlyRequired: []config.Presubmit{}, - }, - changedFiles: []github.PullRequestChange{{Filename: "main.go"}}, - expectedContexts: 0, - shouldSkipExecution: true, - }, - { - name: "empty changed files list", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - pipelineConditionallyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-test-go", - Annotations: map[string]string{ - "pipeline_run_if_changed": ".*\\.go$", - }, - }, - } - p.Context = "ci/prow/test-go" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{}, - expectedContexts: 0, - }, - { - name: "GitHub API error getting changed files", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - protected: func() []config.Presubmit { - p := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-main-protected"}} - p.Context = "ci/prow/protected" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{}, - pullReqChangesErr: fmt.Errorf("GitHub API error"), - expectedContexts: 0, - shouldSkipExecution: true, - }, - { - name: "invalid regex pattern in pipeline_run_if_changed", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - pipelineConditionallyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-invalid-pattern", - Annotations: map[string]string{ - "pipeline_run_if_changed": "[invalid-regex", - }, - }, - } - p.Context = "ci/prow/invalid-pattern" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{{Filename: "main.go"}}, - expectedContexts: 0, // Should not create context due to invalid pattern - }, - { - name: "invalid regex pattern in pipeline_skip_only_if_changed", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - pipelineSkipOnlyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-invalid-skip-pattern", - Annotations: map[string]string{ - "pipeline_skip_if_only_changed": "[invalid-regex", - }, - }, - } - p.Context = "ci/prow/invalid-skip-pattern" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{{Filename: "main.go"}}, - expectedContexts: 0, // Should not create context due to invalid pattern - }, - { - name: "complex regex patterns", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - pipelineConditionallyRequired: func() []config.Presubmit { - p := config.Presubmit{ - JobBase: config.JobBase{ - Name: "pull-ci-test-org-test-repo-main-complex-go-pattern", - Annotations: map[string]string{ - "pipeline_run_if_changed": "^(cmd|pkg)/.*\\.go$|^go\\.(mod|sum)$", - }, - }, - } - p.Context = "ci/prow/complex-go-pattern" - return []config.Presubmit{p} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "cmd/main.go"}, - {Filename: "docs/readme.md"}, - }, - expectedContexts: 1, - expectedNames: []string{"ci/prow/complex-go-pattern"}, - }, - { - name: "protected jobs with specific names verification", - action: github.PullRequestActionOpened, - presubmits: presubmitTests{ - protected: func() []config.Presubmit { - p1 := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-main-test-1"}} - p1.Context = "ci/prow/test-1" - p2 := config.Presubmit{JobBase: config.JobBase{Name: "pull-ci-test-org-test-repo-main-test-2"}} - p2.Context = "ci/prow/test-2" - return []config.Presubmit{p1, p2} - }(), - }, - changedFiles: []github.PullRequestChange{ - {Filename: "config.yaml"}, - }, - expectedContexts: 2, - expectedNames: []string{"ci/prow/test-1", "ci/prow/test-2"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fakeClient := &fakeGhClientForContext{ - changedFiles: tt.changedFiles, - pullReqChangesErr: tt.pullReqChangesErr, - } - - configProvider := &ConfigDataProvider{ - updatedPresubmits: map[string]presubmitTests{ - "test-org/test-repo": tt.presubmits, - }, - logger: testLoggerPipeline(), - } - - watcherVar := &watcher{ - config: enabledConfig{ - Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: "test-org", - Repos: []RepoItem{{Name: "test-repo"}}, - }, - }, - }, - } - - lgtmWatcher := &watcher{config: enabledConfig{}} - cw := &clientWrapper{ - ghc: fakeClient, - configDataProvider: configProvider, - watcher: watcherVar, - lgtmWatcher: lgtmWatcher, - } - - event := github.PullRequestEvent{ - Action: tt.action, - Repo: github.Repo{ - Owner: github.User{Login: "test-org"}, - Name: "test-repo", - }, - PullRequest: github.PullRequest{ - Number: 123, - Head: github.PullRequestBranch{SHA: "abc123"}, - Base: github.PullRequestBranch{Ref: "main"}, - }, - } - - logger := logrus.NewEntry(logrus.New()) - cw.handlePipelineContextCreation(logger, event) - - if tt.shouldSkipExecution { - // For error cases or early returns, just verify no contexts were created - if len(fakeClient.createdStatuses) != 0 { - t.Errorf("expected 0 contexts created due to early return, got %d", len(fakeClient.createdStatuses)) - } - return - } - - if len(fakeClient.createdStatuses) != tt.expectedContexts { - t.Errorf("expected %d contexts created, got %d", tt.expectedContexts, len(fakeClient.createdStatuses)) - for i, status := range fakeClient.createdStatuses { - t.Logf("Created context %d: %s", i, status.Context) - } - } - - // Verify that created contexts have correct state and names - createdNames := make([]string, len(fakeClient.createdStatuses)) - for i, status := range fakeClient.createdStatuses { - if status.State != "pending" { - t.Errorf("expected status state 'pending', got '%s'", status.State) - } - if status.Description != PipelinePendingMessage { - t.Errorf("unexpected status description: %s", status.Description) - } - createdNames[i] = status.Context - } - - // Verify expected context names were created (if specified) - if len(tt.expectedNames) > 0 { - for _, expectedName := range tt.expectedNames { - found := false - for _, createdName := range createdNames { - if createdName == expectedName { - found = true - break - } - } - if !found { - t.Errorf("expected context '%s' was not found. Created contexts: %v", expectedName, createdNames) - } - } - } - }) - } -} diff --git a/cmd/pipeline-controller/reconciler.go b/cmd/pipeline-controller/reconciler.go deleted file mode 100644 index bfadbeb56af..00000000000 --- a/cmd/pipeline-controller/reconciler.go +++ /dev/null @@ -1,318 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strings" - "sync" - "time" - - "github.com/sirupsen/logrus" - - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/builder" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - v1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/kube" -) - -const ( - retention = 24 * time.Hour -) - -type pullRequest struct { - closed bool - checkTime time.Time -} - -type closedPRsCache struct { - prs map[string]pullRequest - m sync.Mutex - ghc minimalGhClient - clearTime time.Time -} - -func composePRIdentifier(refs *v1.Refs) string { - return fmt.Sprintf("%s/%s/%d", refs.Org, refs.Repo, refs.Pulls[0].Number) -} - -// isPRClosed quieries either github or short-term cache to determine if PR is closed. Draft PRs are -// also quialified as closed due to potential, unexpected side effects -func (c *closedPRsCache) isPRClosed(refs *v1.Refs) (bool, error) { - id := composePRIdentifier(refs) - c.m.Lock() - defer c.m.Unlock() - c.clearCache() - pr, ok := c.prs[id] - if ok && (time.Since(pr.checkTime) < 5*time.Minute || pr.closed) { - return pr.closed, nil - } - ghPr, err := c.ghc.GetPullRequest(refs.Org, refs.Repo, refs.Pulls[0].Number) - if err != nil { - return false, fmt.Errorf("error getting pull request: %w", err) - } - c.prs[id] = pullRequest{closed: ghPr.State != github.PullRequestStateOpen || ghPr.Draft, checkTime: time.Now()} - return ghPr.State != github.PullRequestStateOpen || ghPr.Draft, nil -} - -func (c *closedPRsCache) clearCache() { - if time.Since(c.clearTime) < retention { - return - } - for k, v := range c.prs { - if time.Since(v.checkTime) >= retention { - delete(c.prs, k) - } - } - c.clearTime = time.Now() -} - -func composeKey(refs *v1.Refs) string { - return fmt.Sprintf("%s/%s/%d/%s/%s", refs.Org, refs.Repo, refs.Pulls[0].Number, refs.BaseRef, refs.Pulls[0].SHA) -} - -type reconciler struct { - pjclientset ctrlruntimeclient.Client - lister ctrlruntimeclient.Reader - configDataProvider *ConfigDataProvider - ghc minimalGhClient - closedPRsCache closedPRsCache - ids sync.Map - logger *logrus.Entry - watcher *watcher - lgtmWatcher *watcher - pipelineAutoCache *PipelineAutoCache -} - -func NewReconciler( - mgr manager.Manager, - configDataProvider *ConfigDataProvider, - ghc github.Client, - logger *logrus.Entry, - w *watcher, - lgtmW *watcher, - pipelineAutoCache *PipelineAutoCache, -) (*reconciler, error) { - reconciler := &reconciler{ - pjclientset: mgr.GetClient(), - lister: mgr.GetCache(), - configDataProvider: configDataProvider, - ghc: ghc, - ids: sync.Map{}, - logger: logger, - watcher: w, - lgtmWatcher: lgtmW, - pipelineAutoCache: pipelineAutoCache, - closedPRsCache: closedPRsCache{ - prs: map[string]pullRequest{}, - m: sync.Mutex{}, - ghc: ghc, - clearTime: time.Now(), - }, - } - if err := builder. - ControllerManagedBy(mgr). - Named("pipeline-controller"). - For(&v1.ProwJob{}). - WithOptions(controller.Options{MaxConcurrentReconciles: 1}). - Complete(reconciler); err != nil { - return nil, fmt.Errorf("failed to construct controller: %w", err) - } - return reconciler, nil -} - -func (r *reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - log := r.logger.WithField("key", req.String()).WithField("prowjob", req.Name) - err := r.reconcile(ctx, req) - if err != nil { - log.WithError(err).Error("reconciliation failed") - } - return reconcile.Result{}, err -} - -func (r *reconciler) cleanOldIds(interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for range ticker.C { - r.ids.Range(func(key, value interface{}) bool { - if time.Since(value.(time.Time)) >= interval { - r.ids.Delete(key) - } - return true - }) - } -} - -func (r *reconciler) reconcile(ctx context.Context, req reconcile.Request) error { - var pj v1.ProwJob - if err := r.pjclientset.Get(ctx, req.NamespacedName, &pj); err != nil { - if errors.IsNotFound(err) { - return nil - } - - return fmt.Errorf("failed to get prowjob %s: %w", req.String(), err) - } - - if pj.Spec.Refs == nil || pj.Spec.Type != v1.PresubmitJob { - return nil - } - - presubmits := r.configDataProvider.GetPresubmits(pj.Spec.Refs.Org + "/" + pj.Spec.Refs.Repo) - if len(presubmits.protected) == 0 && len(presubmits.alwaysRequired) == 0 && - len(presubmits.conditionallyRequired) == 0 && len(presubmits.pipelineConditionallyRequired) == 0 { - return nil - } - - currentCfg := r.watcher.getConfig() - repos, orgExists := currentCfg[pj.Spec.Refs.Org] - repoConfig, repoExists := repos[pj.Spec.Refs.Repo] - - // Also check LGTM config for pipeline-auto label support - lgtmCfg := r.lgtmWatcher.getConfig() - lgtmRepos, lgtmOrgExists := lgtmCfg[pj.Spec.Refs.Org] - lgtmRepoConfig, lgtmRepoExists := lgtmRepos[pj.Spec.Refs.Repo] - - isInMainConfig := orgExists && repoExists - isInLgtmConfig := lgtmOrgExists && lgtmRepoExists - - if !isInMainConfig && !isInLgtmConfig { - return nil - } - - // Check if branch is enabled for this repo - var branchEnabled bool - if isInMainConfig { - branchEnabled = isBranchEnabled(repoConfig.Branches, pj.Spec.Refs.BaseRef) - } else if isInLgtmConfig { - branchEnabled = isBranchEnabled(lgtmRepoConfig.Branches, pj.Spec.Refs.BaseRef) - } - - if !branchEnabled { - if r.logger != nil { - log := r.logger.WithFields(logrus.Fields{ - "org": pj.Spec.Refs.Org, - "repo": pj.Spec.Refs.Repo, - "branch": pj.Spec.Refs.BaseRef, - "prowjob": pj.Name, - }) - log.Debug("Branch not enabled for pipeline controller, skipping reconcile") - } - return nil - } - - // Determine if we should proceed with automatic triggering - // Case 1: Repo is in main config with trigger mode "auto" - // Case 2: Repo is in LGTM config and has the pipeline-auto label - shouldProceedAuto := false - - if isInMainConfig && repoConfig.Trigger == "auto" { - shouldProceedAuto = true - } else if isInLgtmConfig && len(pj.Spec.Refs.Pulls) > 0 { - // Check if PR has the pipeline-auto label - // First check cache (populated when /pipeline auto is used) - prNumber := pj.Spec.Refs.Pulls[0].Number - if r.pipelineAutoCache != nil && r.pipelineAutoCache.Has(pj.Spec.Refs.Org, pj.Spec.Refs.Repo, prNumber) { - shouldProceedAuto = true - } else { - // Cache miss - query GitHub API - labels, err := r.ghc.GetIssueLabels(pj.Spec.Refs.Org, pj.Spec.Refs.Repo, prNumber) - if err != nil { - if r.logger != nil { - r.logger.WithError(err).WithField("prowjob", pj.Name).Debug("Failed to get PR labels, skipping") - } - return nil - } - for _, label := range labels { - if label.Name == PipelineAutoLabel { - shouldProceedAuto = true - // Add to cache for future lookups - if r.pipelineAutoCache != nil { - r.pipelineAutoCache.Set(pj.Spec.Refs.Org, pj.Spec.Refs.Repo, prNumber) - } - break - } - } - } - } - - if !shouldProceedAuto { - return nil - } - - status, err := r.reportSuccessOnPR(ctx, &pj, presubmits) - if err != nil || !status { - return err - } - - return sendComment(presubmits, &pj, r.ghc, func() { r.ids.Delete(composeKey(pj.Spec.Refs)) }, r.lister) -} - -func (r *reconciler) reportSuccessOnPR(ctx context.Context, pj *v1.ProwJob, presubmits presubmitTests) (bool, error) { - if pj == nil || pj.Spec.Refs == nil || len(pj.Spec.Refs.Pulls) != 1 { - return false, nil - } - selector := map[string]string{} - for _, l := range []string{kube.OrgLabel, kube.RepoLabel, kube.PullLabel, kube.BaseRefLabel} { - selector[l] = pj.ObjectMeta.Labels[l] - } - // Only list presubmit jobs - postsubmits, periodics, and batch jobs are not relevant - selector[kube.ProwJobTypeLabel] = string(v1.PresubmitJob) - var pjs v1.ProwJobList - if err := r.lister.List(ctx, &pjs, ctrlruntimeclient.MatchingLabels(selector)); err != nil { - return false, fmt.Errorf("cannot list prowjob using selector %v", selector) - } - - latestBatch := make(map[string]v1.ProwJob) - for _, pjob := range pjs.Items { - // Skip jobs with missing refs or pulls to avoid nil pointer dereference - if pjob.Spec.Refs == nil || len(pjob.Spec.Refs.Pulls) == 0 { - continue - } - if pjob.Spec.Refs.Pulls[0].SHA == pj.Spec.Refs.Pulls[0].SHA { - if existing, ok := latestBatch[pjob.Spec.Job]; !ok { - latestBatch[pjob.Spec.Job] = pjob - } else if pjob.CreationTimestamp.After(existing.CreationTimestamp.Time) { - latestBatch[pjob.Spec.Job] = pjob - } - } - } - - repoBaseRef := pj.Spec.Refs.Repo + "-" + pj.Spec.Refs.BaseRef - for _, presubmit := range presubmits.protected { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - if _, ok := latestBatch[presubmit.Name]; ok { - return false, nil - } - } - for _, presubmit := range presubmits.alwaysRequired { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - if pjob, ok := latestBatch[presubmit.Name]; !ok || (ok && pjob.Status.State != v1.SuccessState) { - return false, nil - } - } - for _, presubmit := range presubmits.conditionallyRequired { - if !strings.Contains(presubmit.Name, repoBaseRef) { - continue - } - if pjob, ok := latestBatch[presubmit.Name]; ok && pjob.Status.State != v1.SuccessState { - return false, nil - } - } - if closed, err := r.closedPRsCache.isPRClosed(pj.Spec.Refs); err != nil || closed { - return false, err - } - - if _, loaded := r.ids.LoadOrStore(composeKey(pj.Spec.Refs), time.Now()); loaded { - return false, nil - } - return true, nil -} diff --git a/cmd/pipeline-controller/reconciler_test.go b/cmd/pipeline-controller/reconciler_test.go deleted file mode 100644 index 5bc0bce2623..00000000000 --- a/cmd/pipeline-controller/reconciler_test.go +++ /dev/null @@ -1,645 +0,0 @@ -package main - -import ( - "context" - "errors" - "io" - "sync" - "testing" - "time" - - "github.com/sirupsen/logrus" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - v1 "sigs.k8s.io/prow/pkg/apis/prowjobs/v1" - "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/kube" -) - -// testLoggerReconciler creates a discarded logger for tests -func testLoggerReconciler() *logrus.Entry { - logger := logrus.New() - logger.SetOutput(io.Discard) - return logrus.NewEntry(logger) -} - -type fakeGhClient struct { - closed sets.Int -} - -func (c fakeGhClient) GetPullRequest(org, repo string, number int) (*github.PullRequest, error) { - if c.closed.Has(number) { - return &github.PullRequest{State: github.PullRequestStateClosed}, nil - } - return &github.PullRequest{State: github.PullRequestStateOpen}, nil - -} - -func (c fakeGhClient) CreateComment(owner, repo string, number int, comment string) error { - return nil -} - -func (c fakeGhClient) GetPullRequestChanges(org string, repo string, number int) ([]github.PullRequestChange, error) { - return []github.PullRequestChange{}, nil -} - -func (c fakeGhClient) CreateStatus(org, repo, ref string, s github.Status) error { - return nil -} - -func (c fakeGhClient) AddLabel(org, repo string, number int, label string) error { - return nil -} - -func (c fakeGhClient) GetIssueLabels(org, repo string, number int) ([]github.Label, error) { - return []github.Label{}, nil -} - -type FakeReader struct { - pjs v1.ProwJobList -} - -func (tr FakeReader) Get(ctx context.Context, n ctrlruntimeclient.ObjectKey, o ctrlruntimeclient.Object, opts ...ctrlruntimeclient.GetOption) error { - return nil -} - -func (tr FakeReader) List(ctx context.Context, list ctrlruntimeclient.ObjectList, opts ...ctrlruntimeclient.ListOption) error { - pjList, ok := list.(*v1.ProwJobList) - if !ok { - return errors.New("conversion to pj list error") - } - pjList.Items = tr.pjs.Items - return nil -} - -type fakeGhClientWithTracking struct { - closed sets.Int - commentSent bool - labels []github.Label -} - -func (c *fakeGhClientWithTracking) GetPullRequest(org, repo string, number int) (*github.PullRequest, error) { - if c.closed.Has(number) { - return &github.PullRequest{State: github.PullRequestStateClosed}, nil - } - return &github.PullRequest{State: github.PullRequestStateOpen}, nil -} - -func (c *fakeGhClientWithTracking) CreateComment(owner, repo string, number int, comment string) error { - c.commentSent = true - return nil -} - -func (c *fakeGhClientWithTracking) GetPullRequestChanges(org string, repo string, number int) ([]github.PullRequestChange, error) { - return []github.PullRequestChange{}, nil -} - -func (c *fakeGhClientWithTracking) CreateStatus(org, repo, ref string, s github.Status) error { - return nil -} - -func (c *fakeGhClientWithTracking) AddLabel(org, repo string, number int, label string) error { - return nil -} - -func (c *fakeGhClientWithTracking) GetIssueLabels(org, repo string, number int) ([]github.Label, error) { - return c.labels, nil -} - -func composePresubmit(name string, state v1.ProwJobState, sha string) v1.ProwJob { - timeNow := time.Now().Truncate(time.Hour) - pj := v1.ProwJob{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - kube.ProwJobTypeLabel: "presubmit", - kube.OrgLabel: "org", - kube.RepoLabel: "repo", - kube.PullLabel: "123", - }, - CreationTimestamp: metav1.Time{ - Time: timeNow.Add(-3 * time.Hour), - }, - ResourceVersion: "999", - }, - Status: v1.ProwJobStatus{ - State: state, - }, - Spec: v1.ProwJobSpec{ - Type: v1.PresubmitJob, - Refs: &v1.Refs{ - BaseRef: "master", - Repo: "repo", - Pulls: []v1.Pull{ - { - Number: 123, - SHA: sha, - }, - }, - }, - Job: name, - Report: true, - }, - } - if state == v1.SuccessState || state == v1.FailureState || state == v1.AbortedState { - pj.Status.CompletionTime = &metav1.Time{Time: timeNow.Add(-2 * time.Hour)} - } - return pj -} - -func Test_reconciler_reportSuccessOnPR(t *testing.T) { - var objs []runtime.Object - fakeClient := fakectrlruntimeclient.NewClientBuilder().WithRuntimeObjects(objs...).Build() - baseSha := "sha" - dummyPJ := composePresubmit("org-repo-master-ps1", v1.SuccessState, baseSha) - defaultGhClient := fakeGhClient{closed: sets.NewInt()} - - type fields struct { - lister FakeReader - ghc minimalGhClient - } - type args struct { - ctx context.Context - presubmits presubmitTests - } - tests := []struct { - name string - fields fields - args args - want bool - wantErr bool - }{ - { - name: "all tests are required and passed successfully, trigger protected", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps2", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps3", v1.SuccessState, baseSha), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}, {JobBase: config.JobBase{Name: "org-repo-other-branch-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: true, - wantErr: false, - }, - { - name: "all tests are required and passed successfully, do not trigger protected as PR is closed", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps2", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps3", v1.SuccessState, baseSha), - }}}, - ghc: fakeGhClient{closed: sets.NewInt([]int{123}...)}, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: false, - wantErr: false, - }, - { - name: "all required complete but conditionally required failed", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps2", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps3", v1.FailureState, baseSha), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: false, - wantErr: false, - }, - { - name: "all required complete only some of cond required executed", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps2", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps3", v1.SuccessState, baseSha), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}, {JobBase: config.JobBase{Name: "org-repo-master-ps4"}}, {JobBase: config.JobBase{Name: "org-repo-master-ps5"}}}, - }, - }, - want: true, - wantErr: false, - }, - { - name: "all required complete but always required is aborted", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps2", v1.AbortedState, baseSha), - composePresubmit("org-repo-master-ps3", v1.SuccessState, baseSha), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: false, - wantErr: false, - }, - { - name: "do not trigger as user is manually triggering", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps1", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps2", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps3", v1.SuccessState, baseSha), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}, {JobBase: config.JobBase{Name: "org-repo-master-ps4"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: false, - wantErr: false, - }, - { - name: "do not trigger as required are not complete", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps2", v1.PendingState, baseSha), - composePresubmit("org-repo-master-ps3", v1.TriggeredState, baseSha), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: false, - wantErr: false, - }, - { - name: "only protected tests exist", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{}}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - alwaysRequired: []config.Presubmit{}, - conditionallyRequired: []config.Presubmit{}, - }, - }, - want: true, - wantErr: false, - }, - { - name: "batch with one sha is analyzed but different sha has already passed", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps2", v1.SuccessState, "other-sha"), - composePresubmit("org-repo-master-ps3", v1.SuccessState, "other-sha"), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: false, - wantErr: false, - }, - { - name: "do not trigger as user is manually triggering", - fields: fields{ - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - composePresubmit("org-repo-master-ps1", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps2", v1.SuccessState, baseSha), - composePresubmit("org-repo-master-ps3", v1.SuccessState, baseSha), - }}}, - ghc: defaultGhClient, - }, - args: args{ - ctx: context.Background(), - presubmits: presubmitTests{ - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps1"}}, {JobBase: config.JobBase{Name: "org-repo-master-ps4"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - conditionallyRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps3"}}}, - }, - }, - want: false, - wantErr: false, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - r := &reconciler{ - pjclientset: fakeClient, - lister: tc.fields.lister, - configDataProvider: &ConfigDataProvider{ - logger: testLoggerReconciler(), - }, - ghc: tc.fields.ghc, - ids: sync.Map{}, - closedPRsCache: closedPRsCache{prs: map[string]pullRequest{}, m: sync.Mutex{}, ghc: tc.fields.ghc, clearTime: time.Now()}, - lgtmWatcher: &watcher{config: enabledConfig{}}, - pipelineAutoCache: NewPipelineAutoCache(), - } - got, err := r.reportSuccessOnPR(tc.args.ctx, &dummyPJ, tc.args.presubmits) - if (err != nil) != tc.wantErr { - t.Errorf("reconciler.reportSuccessOnPR() error = %v, wantErr %v", err, tc.wantErr) - return - } - if got != tc.want { - t.Errorf("reconciler.reportSuccessOnPR() = %v, want %v", got, tc.want) - } - }) - } -} - -func Test_reconciler_reconcile_with_modes(t *testing.T) { - baseSha := "sha" - // Create a ProwJob with all required fields - dummyPJ := v1.ProwJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: "org-repo-master-ps1", - Namespace: "test-namespace", - Labels: map[string]string{ - kube.ProwJobTypeLabel: "presubmit", - kube.OrgLabel: "org", - kube.RepoLabel: "repo", - kube.PullLabel: "123", - kube.BaseRefLabel: "master", - }, - CreationTimestamp: metav1.Time{ - Time: time.Now(), - }, - ResourceVersion: "999", - }, - Status: v1.ProwJobStatus{ - State: v1.SuccessState, - }, - Spec: v1.ProwJobSpec{ - Type: v1.PresubmitJob, - Refs: &v1.Refs{ - Org: "org", - Repo: "repo", - BaseRef: "master", - Pulls: []v1.Pull{ - { - Number: 123, - SHA: baseSha, - }, - }, - }, - Job: "org-repo-master-ps1", - Report: true, - }, - } - - type fields struct { - watcherConfig enabledConfig - lgtmWatcherConfig enabledConfig - labels []github.Label - presubmits map[string]presubmitTests - expectSendComment bool - } - tests := []struct { - name string - fields fields - }{ - { - name: "auto mode: should send comment", - fields: fields{ - watcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: "org", - Repos: []RepoItem{ - { - Name: "repo", - Mode: struct { - Trigger string - }{ - Trigger: "auto", - }, - }, - }, - }, - }}, - presubmits: map[string]presubmitTests{ - "org/repo": { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps-protected"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - }, - }, - expectSendComment: true, - }, - }, - { - name: "manual mode: should not send comment", - fields: fields{ - watcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: "org", - Repos: []RepoItem{ - { - Name: "repo", - Mode: struct { - Trigger string - }{ - Trigger: "manual", - }, - }, - }, - }, - }}, - presubmits: map[string]presubmitTests{ - "org/repo": { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps-protected"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - }, - }, - expectSendComment: false, - }, - }, - { - name: "LGTM mode with pipeline-auto label: should send comment", - fields: fields{ - watcherConfig: enabledConfig{}, // Not in main config - lgtmWatcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: "org", - Repos: []RepoItem{ - {Name: "repo"}, - }, - }, - }}, - labels: []github.Label{{Name: PipelineAutoLabel}}, - presubmits: map[string]presubmitTests{ - "org/repo": { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps-protected"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - }, - }, - expectSendComment: true, - }, - }, - { - name: "LGTM mode without pipeline-auto label: should not send comment", - fields: fields{ - watcherConfig: enabledConfig{}, // Not in main config - lgtmWatcherConfig: enabledConfig{Orgs: []struct { - Org string `yaml:"org"` - Repos []RepoItem `yaml:"repos"` - }{ - { - Org: "org", - Repos: []RepoItem{ - {Name: "repo"}, - }, - }, - }}, - labels: []github.Label{}, // No pipeline-auto label - presubmits: map[string]presubmitTests{ - "org/repo": { - protected: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps-protected"}}}, - alwaysRequired: []config.Presubmit{{JobBase: config.JobBase{Name: "org-repo-master-ps2"}}}, - }, - }, - expectSendComment: false, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ghc := &fakeGhClientWithTracking{closed: sets.NewInt(), labels: tc.fields.labels} - - // Create a successful always-required job in the lister - successfulJob := v1.ProwJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: "org-repo-master-ps2", - Namespace: "test-namespace", - Labels: map[string]string{ - kube.ProwJobTypeLabel: "presubmit", - kube.OrgLabel: "org", - kube.RepoLabel: "repo", - kube.PullLabel: "123", - kube.BaseRefLabel: "master", - }, - CreationTimestamp: metav1.Time{ - Time: time.Now(), - }, - }, - Status: v1.ProwJobStatus{ - State: v1.SuccessState, - }, - Spec: v1.ProwJobSpec{ - Type: v1.PresubmitJob, - Job: "org-repo-master-ps2", - Refs: &v1.Refs{ - Org: "org", - Repo: "repo", - BaseRef: "master", - Pulls: []v1.Pull{ - { - Number: 123, - SHA: baseSha, - }, - }, - }, - }, - } - - r := &reconciler{ - pjclientset: fakectrlruntimeclient.NewClientBuilder().WithRuntimeObjects(&dummyPJ).Build(), - lister: FakeReader{pjs: v1.ProwJobList{Items: []v1.ProwJob{ - successfulJob, - }}}, - configDataProvider: &ConfigDataProvider{ - updatedPresubmits: tc.fields.presubmits, - logger: testLoggerReconciler(), - }, - ghc: ghc, - ids: sync.Map{}, - watcher: &watcher{config: tc.fields.watcherConfig}, - lgtmWatcher: &watcher{config: tc.fields.lgtmWatcherConfig}, - pipelineAutoCache: NewPipelineAutoCache(), - closedPRsCache: closedPRsCache{ - prs: map[string]pullRequest{}, - m: sync.Mutex{}, - ghc: ghc, - clearTime: time.Now(), - }, - } - - ctx := context.Background() - err := r.reconcile(ctx, reconcile.Request{ - NamespacedName: ctrlruntimeclient.ObjectKey{ - Namespace: dummyPJ.Namespace, - Name: dummyPJ.Name, - }, - }) - - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - if ghc.commentSent != tc.fields.expectSendComment { - t.Errorf("expected comment sent = %v, got %v", tc.fields.expectSendComment, ghc.commentSent) - } - }) - } -} diff --git a/cmd/pr-reminder/README.md b/cmd/pr-reminder/README.md deleted file mode 100644 index c41a948fff2..00000000000 --- a/cmd/pr-reminder/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# PR Reminder -A tool to remind Test Platform team members of active PR review requests in the repos the team cares about. -The tool utilizes the following configuration: -- `config-path`: The location of the tool's config file; containing: `teamMembers`, `teamName`, and `repos` -- `github-users-file` The location of the users' information file. This file contains the GitHub username and Kerberos ID for each user. -- `slack-token-path`: The location of a file containing the slack token. -- `validate-only`: Run in `validate` mode. This simply validates that the config is correct, and will not send any messages or check for PR review requests. - -## Overview -Each of the `teams` in the config will have all of their `teamMember's` `slack id` and `github id` resolved utilizing their inferred email (`{kerberosId}@redhat.com`), and the users' config respectively. -PRs will then be gathered via the github API for each of the `repos` in that `team's` config, and added to users based on the `requested_reviewers` and `requested_teams` attributes. -Finally, a slack message will be sent to each of the `teamMember's` containing information about each PR review request. - -## Local Development -A script, `hack/local-pr-reminder.sh`, exists for running the tool locally. This script takes no arguments, but the user must be logged into the `app.ci` cluster. -You will want to run the script as `USR="my-kerberos-id" hack/local-pr-reminder.sh` to include your own kerberos ID to receive the message in the testing space. -The cluster is utilized to obtain the production `github-users-file` file and the `slack-token` for the alpha slack instance. -The script will run the tool, and message corresponding slack users in the `dptp-robot-testing` space. diff --git a/cmd/pr-reminder/main.go b/cmd/pr-reminder/main.go deleted file mode 100644 index bbeaf910c5a..00000000000 --- a/cmd/pr-reminder/main.go +++ /dev/null @@ -1,695 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "regexp" - "sort" - "strings" - "time" - - "github.com/sirupsen/logrus" - "github.com/slack-go/slack" - - kerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/yaml" - "sigs.k8s.io/prow/pkg/config/secret" - "sigs.k8s.io/prow/pkg/flagutil" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/labels" - "sigs.k8s.io/prow/pkg/logrusutil" - - "github.com/openshift/ci-tools/pkg/rover" -) - -type options struct { - config string - githubUsers string - slackTokenPath string - validateOnly bool - logLevel string - - flagutil.GitHubOptions -} - -func (o *options) validate() error { - _, err := logrus.ParseLevel(o.logLevel) - if err != nil { - return fmt.Errorf("invalid --log-level: %w", err) - } - - if o.config == "" { - return fmt.Errorf("--config-path is required") - } - - if o.githubUsers == "" { - return fmt.Errorf("--rover-groups-config-path is required") - } - - if o.slackTokenPath == "" { - return fmt.Errorf("--slack-token-path is required") - } - - return o.GitHubOptions.Validate(false) -} - -func parseOptions() (options, error) { - var o options - fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - fs.StringVar(&o.config, "config-path", "", "The config file location") - fs.StringVar(&o.githubUsers, "github-users-file", "", "The GitHub users' info file location") - fs.StringVar(&o.slackTokenPath, "slack-token-path", "", "Path to the file containing the Slack token to use.") - fs.BoolVar(&o.validateOnly, "validate-only", false, "Run the tool in validate-only mode. This will simply validate the config.") - fs.StringVar(&o.logLevel, "log-level", "info", "Level at which to log output.") - - o.GitHubOptions.AddFlags(fs) - return o, fs.Parse(os.Args[1:]) -} - -type config struct { - Teams []team `json:"teams"` -} - -// getInterestedLabels returns a set of those labels we are interested in when using the PR reminder -func getInterestedLabels() sets.Set[string] { - var prLabels = sets.Set[string]{} - prLabels.Insert("approved") - prLabels.Insert("lgtm") - prLabels.Insert("do-not-merge/hold") - return prLabels -} - -// getUnactionablePrLabels returns a set of those labels that mark a PR which can't be reviewed in its current state -func getUnactionablePrLabels() sets.Set[string] { - var prLabels = sets.Set[string]{} - prLabels.Insert(labels.WorkInProgress, labels.NeedsRebase) - return prLabels -} - -var orgRepoFormat = regexp.MustCompile(`\w+/\w+`) - -func (c *config) validate(gtk githubToKerberos, slackClient slackClient) error { - var errors []error - for i, t := range c.Teams { - if len(t.TeamMembers) == 0 { - errors = append(errors, fmt.Errorf("teams[%d] doesn't contain any teamMembers", i)) - } - - for _, r := range t.Repos { - if !orgRepoFormat.MatchString(r) { - errors = append(errors, fmt.Errorf("teams[%d] has improperly formatted org/repo: %s", i, r)) - } - } - } - - _, err := c.createUsers(gtk, slackClient) - if err != nil { - errors = append(errors, err) - } - - return kerrors.NewAggregate(errors) -} - -func (c *config) createUsers(gtk githubToKerberos, slackClient slackClient) (map[string]user, error) { - users := make(map[string]user) - var errors []error - for _, team := range c.Teams { - for _, member := range team.TeamMembers { - u, exists := users[member] - if exists { - u.TeamNames.Insert(team.TeamNames...) - u.Repos.Insert(team.Repos...) - } else { - email := fmt.Sprintf("%s@redhat.com", member) - slackUser, err := slackClient.GetUserByEmail(email) - var slackId string - if err != nil { - // Even though we won't be able to find PRs for this user we should leave them in the list for now to determine if there is a github ID found - errors = append(errors, fmt.Errorf("could not get slack id for: %s: %w", member, err)) - } else { - slackId = slackUser.ID - } - u = user{ - KerberosId: member, - TeamNames: sets.New[string](team.TeamNames...), - SlackId: slackId, - Repos: sets.New[string](team.Repos...), - } - } - users[member] = u - } - } - - for githubId, kerberosId := range gtk { - userInfo, exists := users[kerberosId] - if exists { - userInfo.GithubId = githubId - users[kerberosId] = userInfo - } - } - - for id, userInfo := range users { - if userInfo.GithubId == "" { - errors = append(errors, fmt.Errorf("no githubId found for: %v", id)) - delete(users, id) - } - if userInfo.SlackId == "" { - // The error was already found and added, but we don't want to include this user - delete(users, id) - } - } - - return users, kerrors.NewAggregate(errors) -} - -type repoChannel struct { - orgRepo string - omitBots bool -} - -func (c *config) channels() map[string][]repoChannel { - reposByChannel := map[string][]repoChannel{} - for _, team := range c.Teams { - if team.Channel != "" && len(team.Repos) > 0 { - for _, orgRepo := range team.Repos { - reposByChannel[team.Channel] = append(reposByChannel[team.Channel], repoChannel{orgRepo: orgRepo, omitBots: team.OmitBots}) - } - } - } - return reposByChannel -} - -type team struct { - TeamMembers []string `json:"teamMembers"` - TeamNames []string `json:"teamNames"` - Repos []string `json:"repos"` - - // Channel is the optional Slack channel to which the messages about unassigned pull requests from the - // repos will be sent. This does not change the messages sent to the team members. - Channel string `json:"channel,omitempty"` - - // OmitBots determines if we will report on pull requests created by GitHub robots to the channel configured above. - OmitBots bool `json:"omitBots,omitempty"` -} - -type githubToKerberos map[string]string - -type user struct { - KerberosId string - GithubId string - SlackId string - TeamNames sets.Set[string] - Repos sets.Set[string] - PrRequests []prRequest -} - -func (u *user) requestedToReview(pr github.PullRequest) bool { - // only check PRs that the user is not the author of, as they could have requested their own team - if u.GithubId != pr.User.Login { - for _, team := range pr.RequestedTeams { - for _, teamName := range sets.List(u.TeamNames) { - if teamName == team.Slug { - return true - } - } - } - - for _, reviewer := range pr.RequestedReviewers { - if u.GithubId == reviewer.Login { - return true - } - } - - for _, assignee := range pr.Assignees { - if u.GithubId == assignee.Login { - return true - } - } - } - - return false -} - -type prRequest struct { - Repo string - Number int - Url string - Title string - Author string - Created time.Time - LastUpdated time.Time - Labels []string -} - -func (p prRequest) link() string { - return fmt.Sprintf("<%s|*%s#%d*>: %s - by: *%s*", p.Url, p.Repo, p.Number, p.Title, p.Author) -} - -func (p prRequest) createdUpdatedMessage() string { - message := fmt.Sprintf("%s Created: %s | Updated: %s", - p.recency(), - p.Created.Format(time.RFC1123), - p.LastUpdated.Format(time.RFC1123)) - - if time.Since(p.LastUpdated).Hours() <= 24 { - message = fmt.Sprintf("%s %s", newUpdate, message) - } - return message -} - -const ( - recent = ":large_green_circle:" - normal = ":large_orange_circle:" - old = ":red_circle:" - newUpdate = ":new:" - twoDays = time.Hour * 24 * 2 - oneWeek = time.Hour * 24 * 7 -) - -func (p prRequest) recency() string { - now := time.Now() - if p.Created.After(now.Add(-twoDays)) { - return recent - } else if p.Created.After(now.Add(-oneWeek)) { - return normal - } else { - return old - } -} - -type ghClient interface { - prClient - reviewClient -} - -type prClient interface { - GetPullRequests(org, repo string) ([]github.PullRequest, error) - ListPullRequestCommits(org, repo string, number int) ([]github.RepositoryCommit, error) -} - -type slackClient interface { - GetUserByEmail(email string) (*slack.User, error) - PostMessage(channelID string, options ...slack.MsgOption) (string, string, error) -} - -func main() { - logrusutil.ComponentInit() - - o, err := parseOptions() - if err != nil { - logrus.WithError(err).Fatal("cannot parse args: ", os.Args[1:]) - } - - if err = o.validate(); err != nil { - logrus.WithError(err).Fatal("validation failed") - } - - level, _ := logrus.ParseLevel(o.logLevel) - logrus.SetLevel(level) - - var c config - if err = loadConfig(o.config, &c); err != nil { - logrus.WithError(err).Fatal("failed to load config") - } - - roverUsers := []rover.User{} - if err = loadConfig(o.githubUsers, &roverUsers); err != nil { - logrus.WithError(err).Fatal("failed to load rover groups config") - } - gtk := githubToKerberos(rover.MapGithubToKerberos(roverUsers)) - - if err := secret.Add(o.slackTokenPath); err != nil { - logrus.WithError(err).Fatal("failed to start secrets agent") - } - slackClient := slack.New(string(secret.GetSecret(o.slackTokenPath))) - - if o.validateOnly { - if err := c.validate(gtk, slackClient); err != nil { - logrus.WithError(err).Fatal("validation failed") - } else { - logrus.Infof("config is valid") - } - } else { - users, err := c.createUsers(gtk, slackClient) - if err != nil { - logrus.WithError(err).Error("failed to create some users") - } - - channels := c.channels() - - if len(users) > 0 || len(channels) > 0 { - ghClient, err := o.GitHubOptions.GitHubClient(false) - if err != nil { - logrus.WithError(err).Fatal("failed to create github client") - } - - unassigned, assinged := findPRs(users, channels, ghClient) - var errs []error - for _, user := range assinged { - logrus.Infof("%d PRs were found for user: %s", len(user.PrRequests), user.KerberosId) - if len(user.PrRequests) > 0 { - // sort by most recent update first - sort.Slice(user.PrRequests, func(i, j int) bool { - return user.PrRequests[i].LastUpdated.After(user.PrRequests[j].LastUpdated) - }) - - logger := logrus.WithFields(logrus.Fields{ - "kerberosId": user.KerberosId, - }) - if err = sendMessage(logger, user.SlackId, user.PrRequests, slackClient); err != nil { - logger.WithError(err).Error("failed to message user") - errs = append(errs, err) - } - } - } - - for channel, prs := range unassigned { - logrus.Infof("%d unassigned PRs were found for channel: %s", len(prs), channel) - if len(prs) > 0 { - // sort by most recent update first - sort.Slice(prs, func(i, j int) bool { - return prs[i].LastUpdated.After(prs[j].LastUpdated) - }) - - logger := logrus.WithFields(logrus.Fields{ - "channel": channel, - }) - if err = sendMessage(logger, channel, prs, slackClient); err != nil { - logger.WithError(err).Error("failed to message user") - errs = append(errs, err) - } - } - } - if len(errs) > 0 { - logrus.WithError(kerrors.NewAggregate(errs)).Fatal("Failed to message users") - } - } - } -} - -// findPRs finds the yet-to-be-reviewed PRs that should be broadcast to each channel as well as the PRs requiring -// a reminder for each team -func findPRs(users map[string]user, channels map[string][]repoChannel, ghClient ghClient) (map[string][]prRequest, map[string]user) { - repos := sets.New[string]() - for _, u := range users { - repos.Insert(sets.List(u.Repos)...) - } - - logrus.Infof("finding PRs for %d users in %d repos", len(users), len(repos)) - - repoToPRs := make(map[string][]github.PullRequest, len(repos)) - for _, orgRepo := range sets.List(repos) { - split := strings.Split(orgRepo, "/") - org, repo := split[0], split[1] - - prs, err := ghClient.GetPullRequests(org, repo) - if err != nil { - logrus.Errorf("failed to get pull requests for: %s: %v", repo, err) - } - repoToPRs[orgRepo] = prs - } - - for i, u := range users { - for _, orgRepo := range sets.List(u.Repos) { - split := strings.Split(orgRepo, "/") - org, repo := split[0], split[1] - - for _, pr := range repoToPRs[orgRepo] { - if !hasUnactionableLabels(pr.Labels) && !isReadyToMerge(pr.Labels) && u.requestedToReview(pr) && requiresAttention(org, repo, pr, ghClient, u) { - u.PrRequests = append(u.PrRequests, requestFor(orgRepo, pr)) - users[i] = u - } - } - } - } - - logrus.Infof("finding unassigned PRs for %d channels", len(channels)) - - channelToPRs := map[string][]prRequest{} - for channel, repos := range channels { - for _, cfg := range repos { - split := strings.Split(cfg.orgRepo, "/") - org, repo := split[0], split[1] - - for _, pr := range repoToPRs[cfg.orgRepo] { - if isUnreviewed(org, repo, pr, ghClient) && !hasUnactionableLabels(pr.Labels) && !(cfg.omitBots && pr.User.Type == github.UserTypeBot) { - if _, recorded := channelToPRs[channel]; !recorded { - channelToPRs[channel] = []prRequest{} - } - channelToPRs[channel] = append(channelToPRs[channel], requestFor(cfg.orgRepo, pr)) - } - } - } - } - - return channelToPRs, users -} - -func requestFor(repo string, pr github.PullRequest) prRequest { - return prRequest{ - Repo: repo, - Number: pr.Number, - Url: pr.HTMLURL, - Title: pr.Title, - Author: pr.User.Login, - Created: pr.CreatedAt, - LastUpdated: pr.UpdatedAt, - Labels: filterLabels(pr.Labels, getInterestedLabels()), - } -} - -// filterLabels filters out those labels from the PR we are not interested in -// and returns only those that are included in the interestedLabels set -func filterLabels(labels []github.Label, interestedLabels sets.Set[string]) []string { - var result []string - for _, label := range labels { - if interestedLabels.Has(label.Name) { - result = append(result, label.Name) - } - } - sort.Strings(result) - return result -} - -func loadConfig(filename string, config interface{}) error { - configData, err := os.ReadFile(filename) - if err != nil { - return fmt.Errorf("failed to read config: %w", err) - } - if err = yaml.Unmarshal(configData, &config); err != nil { - return fmt.Errorf("failed to unmarshall config: %w", err) - } - return nil -} - -func splitPRs(prs []prRequest, chunkSize int) [][]prRequest { - var chunks [][]prRequest - totalPRs := len(prs) - if totalPRs > chunkSize { - logrus.Warnf("Too many PRs (%d) to send in a single message, splitting into multiple messages", totalPRs) - } - // Split the PRs into chunks of chunkSize - // This is to avoid hitting Slack's message size limit - // and to make it easier to read the messages - for i := 0; i < totalPRs; i += chunkSize { - end := i + chunkSize - if end > totalPRs { - end = totalPRs - } - chunks = append(chunks, prs[i:end]) - } - return chunks -} - -func sendMessage(logger *logrus.Entry, channel string, prs []prRequest, slackClient slackClient) error { - header := func(prsNum int, title string) []slack.Block { - return []slack.Block{ - &slack.HeaderBlock{ - Type: slack.MBTHeader, - Text: &slack.TextBlockObject{ - Type: slack.PlainTextType, - Text: title, - }, - }, - &slack.SectionBlock{ - Type: slack.MBTSection, - Text: &slack.TextBlockObject{ - Type: slack.PlainTextType, - Text: fmt.Sprintf("You have %d PR(s) to review:", prsNum), - }, - }, - &slack.ContextBlock{ - Type: slack.MBTContext, - ContextElements: slack.ContextElements{ - Elements: []slack.MixedElement{ - &slack.TextBlockObject{ - Type: slack.MarkdownType, - Text: fmt.Sprintf("%s: created in the last 2 days", recent), - }, - &slack.TextBlockObject{ - Type: slack.MarkdownType, - Text: fmt.Sprintf("%s: created in the last week", normal), - }, - &slack.TextBlockObject{ - Type: slack.MarkdownType, - Text: fmt.Sprintf("%s: created more than a week ago", old), - }, - &slack.TextBlockObject{ - Type: slack.MarkdownType, - Text: fmt.Sprintf("%s: updated in the last 24 hours", newUpdate), - }, - }, - }, - }, - &slack.DividerBlock{Type: slack.MBTDivider}, - } - } - - var errors []error - prChunks := splitPRs(prs, 40) - prChunksLen := len(prChunks) - - for chunkIdx, prChunk := range prChunks { - headerTitle := "PR Review Reminders." - if len(prChunks) > 1 { - headerTitle = fmt.Sprintf("PR Review Reminders (%d/%d)", chunkIdx+1, prChunksLen) - } - message := header(len(prChunk), headerTitle) - - for _, pr := range prChunk { - prBlock := &slack.ContextBlock{ - Type: slack.MBTContext, - ContextElements: slack.ContextElements{ - Elements: []slack.MixedElement{ - &slack.TextBlockObject{ - Type: slack.MarkdownType, - Text: pr.link(), - }, - &slack.TextBlockObject{ - Type: slack.MarkdownType, - Text: pr.createdUpdatedMessage(), - }, - }, - }, - } - if len(pr.Labels) > 0 { - prBlock.ContextElements.Elements = append(prBlock.ContextElements.Elements, &slack.TextBlockObject{ - Type: slack.MarkdownType, - Text: getLabelMessage(pr.Labels), - }) - } - message = append(message, prBlock) - } - - responseChannel, responseTimestamp, err := slackClient.PostMessage(channel, - slack.MsgOptionText(headerTitle, true), - slack.MsgOptionBlocks(message...)) - if err != nil { - logger.WithError(err).WithField("message", message).Debug("Failed to message user about PR review reminder") - errors = append(errors, fmt.Errorf("failed to message channel %s about PR review reminder: %w", channel, err)) - } else { - logger.Infof("Posted PR review reminder in channel: %s at: %s", responseChannel, responseTimestamp) - } - } - - return kerrors.NewAggregate(errors) -} - -// getLabelMessage returns a string listing te PR's labels -func getLabelMessage(labels []string) string { - return fmt.Sprintf(":label: labeled: *%v*", strings.Join(labels[:], ", ")) -} - -// hasUnactionableLabels returns whether a PR has any labels which mark a PR -// that can't be reviewed in its current state -func hasUnactionableLabels(labels []github.Label) bool { - unactionableLabels := getUnactionablePrLabels() - for _, label := range labels { - if unactionableLabels.Has(label.Name) { - return true - } - } - return false -} - -// isReadyToMerge returns whether a PR has all the labels it needs to merge, which likely means it -// does not need to be looked at again -func isReadyToMerge(labels []github.Label) bool { - existing := sets.Set[string]{} - for _, label := range labels { - existing.Insert(label.Name) - } - return existing.HasAll("lgtm", "approved") -} - -type reviewClient interface { - ListReviews(org, repo string, number int) ([]github.Review, error) -} - -// isUnreviewed checks if we should post about this new PR to the channel, generally when the PR needs someone to look -// at it and nobody has done so yet - even though Prow will auto-assign the PR to someone, it's not super useful to have -// all those PRs get direct-message pinged to those folks as often some high-level developer gets auto-assigned to everything -func isUnreviewed(org, repo string, pr github.PullRequest, client reviewClient) bool { - // if we're LGTM + approved, we're not interested in bugging anyone - if isReadyToMerge(pr.Labels) { - return false - } - - // if it's assigned to someone, there's already been action taken on it - if len(pr.Assignees) > 0 { - return false - } - - // otherwise, post it on the channel if nobody's reviewed it - reviews, err := client.ListReviews(org, repo, pr.Number) - if err != nil { - logrus.WithError(err).Errorf("Failed to list PR reviews") - } - return len(reviews) == 0 -} - -// requiresAttention determines if the user needs to be reminded for the pull request in question by determining when -// the user last interacted with the PR and if any changes have been pushed or comments posted by the author since then -func requiresAttention(org, repo string, pr github.PullRequest, client ghClient, u user) bool { - reviews, err := client.ListReviews(org, repo, pr.Number) - if err != nil { - logrus.WithError(err).Errorf("Failed to list PR reviews") - } - - var lastReview time.Time - for _, review := range reviews { - if review.User.Login == u.GithubId { - lastReview = review.SubmittedAt - } - } - - if lastReview.IsZero() { - // the user has never reviewed it, so it requires attention - return true - } - - commits, err := client.ListPullRequestCommits(org, repo, pr.Number) - if err != nil { - logrus.WithError(err).Errorf("Failed to list PR commits") - } - - var lastCommit time.Time - for _, commit := range commits { - for _, date := range []time.Time{commit.Commit.Committer.Date, commit.Commit.Author.Date} { - if date.After(lastCommit) { - lastCommit = date - } - } - } - - // n.b. a more exhaustive approach to determining whether the PR requires attention would furthermore - // look at issue comments and try to catch the cases where no new commits have been pushed but comments - // have been posted from the author in response to reviews, but this gets tricky to do as we'd need to - // follow each review comment thread as well as the overall issue, and this would quickly eat up a lot of - // GitHub API tokens. The existing heuristic is easy to cache and if it's good enough, we can omit the - // harder follow-up. - - return lastCommit.After(lastReview) -} diff --git a/cmd/pr-reminder/main_test.go b/cmd/pr-reminder/main_test.go deleted file mode 100644 index 87f7c04a591..00000000000 --- a/cmd/pr-reminder/main_test.go +++ /dev/null @@ -1,969 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/slack-go/slack" - - "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/labels" - - "github.com/openshift/ci-tools/pkg/testhelper" -) - -func Test_user_requestedToReview(t *testing.T) { - testCases := []struct { - name string - user user - pr github.PullRequest - expected bool - }{ - { - name: "user requested", - user: user{GithubId: "some-id"}, - pr: github.PullRequest{ - RequestedReviewers: []github.User{ - { - Login: "some-id", - }, - }, - }, - expected: true, - }, - { - name: "user assigned", - user: user{GithubId: "some-id"}, - pr: github.PullRequest{ - Assignees: []github.User{ - { - Login: "some-id", - }, - }, - }, - expected: true, - }, - { - name: "team requested", - user: user{GithubId: "some-id", TeamNames: sets.New[string]("some-team")}, - pr: github.PullRequest{ - RequestedTeams: []github.Team{ - { - Slug: "some-team", - }, - }, - }, - expected: true, - }, - { - name: "team requested while user is author", - user: user{GithubId: "some-id", TeamNames: sets.New[string]("some-team")}, - pr: github.PullRequest{ - User: github.User{ - Login: "some-id", - }, - RequestedTeams: []github.Team{ - { - Slug: "some-team", - }, - }, - }, - expected: false, - }, - { - name: "not requested", - user: user{GithubId: "some-id"}, - pr: github.PullRequest{ - RequestedReviewers: []github.User{ - { - Login: "a-different-id", - }, - }, - RequestedTeams: []github.Team{ - { - Slug: "some-other-team", - }, - }, - }, - expected: false, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - requested := tc.user.requestedToReview(tc.pr) - if requested != tc.expected { - t.Fatalf("requestedToReview returned %v, expected %v", requested, tc.expected) - } - }) - } -} - -func Test_prRequest_recency(t *testing.T) { - testCases := []struct { - name string - prRequest prRequest - expected string - }{ - { - name: "recent PR", - prRequest: prRequest{ - Created: time.Now().Add(-1 * time.Hour), - }, - expected: recent, - }, - { - name: "5 day old PR", - prRequest: prRequest{ - Created: time.Now().Add(-24 * 5 * time.Hour), - }, - expected: normal, - }, - { - name: "old PR", - prRequest: prRequest{ - Created: time.Now().Add(-24 * 30 * time.Hour), - }, - expected: old, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - recency := tc.prRequest.recency() - if diff := cmp.Diff(tc.expected, recency); diff != "" { - t.Fatalf("recency resulted didn't match expected, diff: %s", diff) - } - }) - } -} - -type fakeGithubClient struct { - prs map[string][]github.PullRequest - reviews map[string]map[int][]github.Review - commits map[string]map[int][]github.RepositoryCommit -} - -func (c fakeGithubClient) GetPullRequests(org, repo string) ([]github.PullRequest, error) { - orgRepo := fmt.Sprintf("%s/%s", org, repo) - return c.prs[orgRepo], nil -} - -func (c fakeGithubClient) ListReviews(org, repo string, number int) ([]github.Review, error) { - orgRepo := fmt.Sprintf("%s/%s", org, repo) - if prs, ok := c.reviews[orgRepo]; ok { - return prs[number], nil - } - return nil, nil -} - -func (c fakeGithubClient) ListPullRequestCommits(org, repo string, number int) ([]github.RepositoryCommit, error) { - orgRepo := fmt.Sprintf("%s/%s", org, repo) - if prs, ok := c.commits[orgRepo]; ok { - return prs[number], nil - } - return nil, nil -} - -func TestFindPRs(t *testing.T) { - now := time.Now() - client := fakeGithubClient{prs: map[string][]github.PullRequest{ - "org/repo-1": { - { - Number: 1, - HTMLURL: "github.com/org/repo-1/1", - Title: "Some PR", - User: github.User{Login: "a-user"}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "id-1", - }, - }, - RequestedTeams: []github.Team{ - { - Slug: "some-team", - }, - }, - Assignees: []github.User{ - { - Login: "random", - }, - }, - }, - { - Number: 2, - HTMLURL: "github.com/org/repo-1/2", - Title: "Some Other PR", - User: github.User{Login: "some-user"}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "id-2", - }, - }, - RequestedTeams: []github.Team{ - { - Slug: "some-other-team", - }, - }, - Assignees: []github.User{ - { - Login: "random", - }, - }, - }, - { - Number: 3, - HTMLURL: "github.com/org/repo-1/3", - Title: "Reviewed", - User: github.User{Login: "some-user"}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "other", - }, - }, - }, - { - Number: 4, - HTMLURL: "github.com/org/repo-1/4", - Title: "Brand New", - User: github.User{Login: "some-user"}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "other", - }, - }, - }, - { - Number: 12, - HTMLURL: "github.com/org/repo-1/12", - Title: "Brand New From Bot", - User: github.User{Login: "some-bot", Type: github.UserTypeBot}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "other", - }, - }, - }, - { - Number: 5, - HTMLURL: "github.com/org/repo-1/5", - Title: "Brand New But Approved", - User: github.User{Login: "some-user"}, - CreatedAt: now, - UpdatedAt: now, - Labels: []github.Label{{Name: "approved"}, {Name: "lgtm"}}, - RequestedReviewers: []github.User{ - { - Login: "other", - }, - }, - }, - { - Number: 6, - HTMLURL: "github.com/org/repo-1/6", - Title: "Brand New But WIP", - User: github.User{Login: "some-user"}, - CreatedAt: now, - UpdatedAt: now, - Labels: []github.Label{{Name: "do-not-merge/work-in-progress"}}, - RequestedReviewers: []github.User{ - { - Login: "other", - }, - }, - }, - { - Number: 7, - HTMLURL: "github.com/org/repo-1/7", - Title: "Doesn't Need Attention Yet", - User: github.User{Login: "some-user"}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "id-2", - }, - }, - RequestedTeams: []github.Team{ - { - Slug: "some-other-team", - }, - }, - Assignees: []github.User{ - { - Login: "random", - }, - }, - }, - }, - "org/repo-2": { - { - Number: 66, - HTMLURL: "github.com/org/repo-2/66", - Title: "Some PR in this repo", - User: github.User{Login: "a-user"}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "a-different-id", - }, - }, - RequestedTeams: []github.Team{ - { - Slug: "some-team", - }, - }, - Assignees: []github.User{ - { - Login: "random", - }, - }, - }, - { - Number: 67, - HTMLURL: "github.com/org/repo-2/67", - Title: "Ready to merge", - User: github.User{Login: "a-user"}, - CreatedAt: now, - UpdatedAt: now, - Labels: []github.Label{{Name: "approved"}, {Name: "lgtm"}}, - RequestedReviewers: []github.User{ - { - Login: "a-different-id", - }, - }, - RequestedTeams: []github.Team{ - { - Slug: "some-team", - }, - }, - Assignees: []github.User{ - { - Login: "random", - }, - }, - }, - { - Number: 12, - HTMLURL: "github.com/org/repo-2/12", - Title: "Brand New From Bot", - User: github.User{Login: "some-user", Type: github.UserTypeBot}, - CreatedAt: now, - UpdatedAt: now, - RequestedReviewers: []github.User{ - { - Login: "other", - }, - }, - }, - }, - }, - reviews: map[string]map[int][]github.Review{ - "org/repo-1": { - 3: {{ID: 2}}, - 7: {{ID: 2, User: github.User{Login: "id-2"}, SubmittedAt: now}}, - }, - }, - commits: map[string]map[int][]github.RepositoryCommit{ - "org/repo-1": { - 7: {{Commit: github.GitCommit{Committer: github.CommitAuthor{Date: now.Add(-1 * time.Hour)}}}}, - }, - }, - } - - testCases := []struct { - name string - users map[string]user - expected map[string]user - - channels map[string][]repoChannel - expectedChannels map[string][]prRequest - }{ - { - name: "PRs exist", - users: map[string]user{ - "someuser": { - KerberosId: "someuser", - GithubId: "id-1", - TeamNames: sets.New[string]("some-team"), - Repos: sets.New[string]("org/repo-1", "org/repo-2"), - }, - "user-b": { - KerberosId: "user-b", - GithubId: "id-2", - TeamNames: sets.New[string]("some-team"), - Repos: sets.New[string]("org/repo-1", "org/repo-2"), - }, - }, - channels: map[string][]repoChannel{ - "channel": { - {orgRepo: "org/repo-1"}, - {orgRepo: "org/repo-2", omitBots: true}, - }, - }, - expected: map[string]user{ - "someuser": { - KerberosId: "someuser", - GithubId: "id-1", - TeamNames: sets.New[string]("some-team"), - Repos: sets.New[string]("org/repo-1", "org/repo-2"), - PrRequests: []prRequest{ - { - Repo: "org/repo-1", - Number: 1, - Url: "github.com/org/repo-1/1", - Title: "Some PR", - Author: "a-user", - Created: now, - LastUpdated: now, - }, - { - Repo: "org/repo-2", - Number: 66, - Url: "github.com/org/repo-2/66", - Title: "Some PR in this repo", - Author: "a-user", - Created: now, - LastUpdated: now, - }, - }, - }, - "user-b": { - KerberosId: "user-b", - GithubId: "id-2", - TeamNames: sets.New[string]("some-team"), - Repos: sets.New[string]("org/repo-1", "org/repo-2"), - PrRequests: []prRequest{ - { - Repo: "org/repo-1", - Number: 1, - Url: "github.com/org/repo-1/1", - Title: "Some PR", - Author: "a-user", - Created: now, - LastUpdated: now, - }, - { - Repo: "org/repo-1", - Number: 2, - Url: "github.com/org/repo-1/2", - Title: "Some Other PR", - Author: "some-user", - Created: now, - LastUpdated: now, - }, - { - Repo: "org/repo-2", - Number: 66, - Url: "github.com/org/repo-2/66", - Title: "Some PR in this repo", - Author: "a-user", - Created: now, - LastUpdated: now, - }, - }, - }, - }, - expectedChannels: map[string][]prRequest{ - "channel": {{ - Repo: "org/repo-1", - Number: 4, - Url: "github.com/org/repo-1/4", - Title: "Brand New", - Author: "some-user", - Created: now, - LastUpdated: now, - }, { - Repo: "org/repo-1", - Number: 12, - Url: "github.com/org/repo-1/12", - Title: "Brand New From Bot", - Author: "some-bot", - Created: now, - LastUpdated: now, - }}, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - unassigned, prs := findPRs(tc.users, tc.channels, client) - if diff := cmp.Diff(tc.expected, prs); diff != "" { - t.Fatalf("got incorrect PRs for users, diff: %s", diff) - } - if diff := cmp.Diff(tc.expectedChannels, unassigned); diff != "" { - t.Fatalf("got incorrect unassigned PRs, diff: %s", diff) - } - }) - } -} - -type fakeSlackClient struct { - userIdsByEmail map[string]string -} - -func (c fakeSlackClient) GetUserByEmail(email string) (*slack.User, error) { - userId, exists := c.userIdsByEmail[email] - if exists { - return &slack.User{ID: userId}, nil - } - - return nil, fmt.Errorf("no userId found for email: %s", email) -} - -func (c fakeSlackClient) PostMessage(channelID string, options ...slack.MsgOption) (string, string, error) { - //No-op - return "", "", nil -} - -func Test_config_CreateUsers(t *testing.T) { - client := fakeSlackClient{userIdsByEmail: map[string]string{"user1@redhat.com": "U1000000", "user2@redhat.com": "U222222", "user3@redhat.com": "U333333"}} - testCases := []struct { - name string - config config - gtk githubToKerberos - expected map[string]user - expectedErr error - }{ - { - name: "valid inputs", - config: config{ - Teams: []team{ - { - TeamMembers: []string{"user1", "user2"}, - TeamNames: []string{"some-team", "other-team"}, - Repos: []string{"org/repo"}, - }, - { - TeamMembers: []string{"user3"}, - TeamNames: []string{"some-team"}, - Repos: []string{"other-org/repo"}, - }, - }, - }, - gtk: githubToKerberos{"user-1": "user1", "user-2": "user2", "user-3": "user3"}, - expected: map[string]user{ - "user1": { - KerberosId: "user1", - GithubId: "user-1", - SlackId: "U1000000", - TeamNames: sets.New[string]("some-team", "other-team"), - Repos: sets.New[string]("org/repo"), - }, - "user2": { - KerberosId: "user2", - GithubId: "user-2", - SlackId: "U222222", - TeamNames: sets.New[string]("some-team", "other-team"), - Repos: sets.New[string]("org/repo"), - }, - "user3": { - KerberosId: "user3", - GithubId: "user-3", - SlackId: "U333333", - TeamNames: sets.New[string]("some-team"), - Repos: sets.New[string]("other-org/repo"), - }, - }, - }, - { - name: "user on multiple teams", - config: config{ - Teams: []team{ - { - TeamMembers: []string{"user1"}, - TeamNames: []string{"some-team", "other-team"}, - Repos: []string{"org/repo"}, - }, - { - TeamMembers: []string{"user1"}, - TeamNames: []string{"some-team", "additional-team"}, - Repos: []string{"other-org/repo"}, - }, - }, - }, - gtk: githubToKerberos{"user-1": "user1"}, - expected: map[string]user{ - "user1": { - KerberosId: "user1", - GithubId: "user-1", - SlackId: "U1000000", - TeamNames: sets.New[string]("some-team", "other-team", "additional-team"), - Repos: sets.New[string]("org/repo", "other-org/repo"), - }, - }, - }, - { - name: "no slack user found for a configured user", - config: config{ - Teams: []team{{ - TeamMembers: []string{"user1", "user4"}, - TeamNames: []string{"some-team"}, - }}, - }, - gtk: githubToKerberos{"user-1": "user1", "user-2": "user2"}, - expected: map[string]user{ - "user1": { - KerberosId: "user1", - GithubId: "user-1", - SlackId: "U1000000", - TeamNames: sets.New[string]("some-team"), - }, - }, - expectedErr: errors.New("[could not get slack id for: user4: no userId found for email: user4@redhat.com, no githubId found for: user4]"), - }, - { - name: "no github user found for a configured user", - config: config{ - Teams: []team{{ - TeamMembers: []string{"user1", "user2"}, - TeamNames: []string{"some-team"}, - }}, - }, - gtk: githubToKerberos{"user-1": "user1"}, - expected: map[string]user{ - "user1": { - KerberosId: "user1", - GithubId: "user-1", - SlackId: "U1000000", - TeamNames: sets.New[string]("some-team"), - }, - }, - expectedErr: errors.New("no githubId found for: user2"), - }, - { - name: "missing github id and missing slack id for different users", - config: config{ - Teams: []team{{ - TeamMembers: []string{"user1", "user2", "user4"}, - TeamNames: []string{"some-team"}, - }}, - }, - gtk: githubToKerberos{"user-1": "user1", "user-4": "user4"}, - expected: map[string]user{ - "user1": { - KerberosId: "user1", - GithubId: "user-1", - SlackId: "U1000000", - TeamNames: sets.New[string]("some-team"), - }, - }, - expectedErr: errors.New("[could not get slack id for: user4: no userId found for email: user4@redhat.com, no githubId found for: user2]"), - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - users, err := tc.config.createUsers(tc.gtk, client) - if diff := cmp.Diff(tc.expectedErr, err, testhelper.EquateErrorMessage); diff != "" { - t.Fatalf("returned error doesn't match expected, diff: %s", diff) - } - - if diff := cmp.Diff(tc.expected, users); diff != "" { - t.Fatalf("returned users don't match expected, diff: %s", diff) - } - }) - } -} - -func Test_config_validate(t *testing.T) { - client := fakeSlackClient{userIdsByEmail: map[string]string{"user1@redhat.com": "U1000000", "user2@redhat.com": "U222222", "no-gh@redhat.com": "U4444", "user5@redhat.com": "U55555555"}} - gtk := githubToKerberos{"user-1": "user1", "user-2": "user2", "noslack": "no-slack", "user-5": "user5"} - - testCases := []struct { - name string - config config - expected error - }{ - { - name: "valid", - config: config{ - Teams: []team{ - { - TeamMembers: []string{"user1", "user2"}, - TeamNames: []string{"some-team"}, - Repos: []string{"org/repo", "org/repo2", "org2/repo"}, - }, - { - TeamMembers: []string{"user5"}, - TeamNames: []string{"some-other-team"}, - Repos: []string{"org2/repo"}, - }, - }, - }, - }, - { - name: "no teamMembers", - config: config{ - Teams: []team{ - { - TeamNames: []string{"some-team"}, - Repos: []string{"org/repo", "org/repo2", "org2/repo"}, - }, - }, - }, - expected: errors.New("teams[0] doesn't contain any teamMembers"), - }, - { - name: "invalid repo", - config: config{ - Teams: []team{ - { - TeamMembers: []string{"user1", "user2"}, - TeamNames: []string{"some-team"}, - Repos: []string{"repo", "org/repo2", "org2/repo"}, - }, - }, - }, - expected: errors.New("teams[0] has improperly formatted org/repo: repo"), - }, - { - name: "invalid teamMembers", - config: config{ - Teams: []team{ - { - TeamMembers: []string{"no-slack", "no-gh"}, - TeamNames: []string{"some-team"}, - Repos: []string{"org/repo"}, - }, - }, - }, - expected: errors.New("[could not get slack id for: no-slack: no userId found for email: no-slack@redhat.com, no githubId found for: no-gh]"), - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := tc.config.validate(gtk, client) - if diff := cmp.Diff(tc.expected, err, testhelper.EquateErrorMessage); diff != "" { - t.Fatalf("returned error doesn't match expected, diff: %s", diff) - } - }) - } -} - -func Test_filterLabels(t *testing.T) { - holdLabel := github.Label{Name: "do-not-merge/hold"} - acceptedLabel := github.Label{Name: "accepted"} - unwantedLabel := github.Label{Name: "not interesting label"} - - interestingLabels := sets.Set[string]{} - interestingLabels.Insert(holdLabel.Name, acceptedLabel.Name) - - testCases := []struct { - name string - prLabels []github.Label - expected []string - }{ - { - name: "pr with no labels", - prLabels: []github.Label{}, - expected: nil, - }, - { - name: "pr with one label we are interested in", - prLabels: []github.Label{holdLabel}, - expected: []string{holdLabel.Name}, - }, - { - name: "returned labels are in correct order", - prLabels: []github.Label{holdLabel, acceptedLabel}, - expected: []string{acceptedLabel.Name, holdLabel.Name}, - }, - { - name: "pr with only uninteresting labels", - prLabels: []github.Label{unwantedLabel}, - expected: nil, - }, - { - name: "pr has one label we are not interested in", - prLabels: []github.Label{holdLabel, unwantedLabel}, - expected: []string{holdLabel.Name}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual := filterLabels(tc.prLabels, interestingLabels) - if diff := cmp.Diff(actual, tc.expected); diff != "" { - t.Fatalf("returned labels do not match expected labels, diff:%s", diff) - } - }) - } -} - -func Test_hasUnactionableLabels(t *testing.T) { - holdLabel := github.Label{Name: labels.Hold} - approvedLabel := github.Label{Name: labels.Approved} - wipLabel := github.Label{Name: labels.WorkInProgress} - needsRebaseLabel := github.Label{Name: labels.NeedsRebase} - - var testCases = []struct { - name string - labels []github.Label - expected bool - }{ - { - name: "no labels", - labels: []github.Label{}, - expected: false, - }, - { - name: "no unwanted labels", - labels: []github.Label{approvedLabel}, - expected: false, - }, - { - name: "only one label and it is unwanted", - labels: []github.Label{wipLabel}, - expected: true, - }, - { - name: "one unwanted label among ok labels", - labels: []github.Label{approvedLabel, needsRebaseLabel, holdLabel}, - expected: true, - }, - { - name: "only unwanted labels", - labels: []github.Label{wipLabel, needsRebaseLabel}, - expected: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if diff := cmp.Diff(hasUnactionableLabels(tc.labels), tc.expected); diff != "" { - t.Fatalf("actual result desn't match expected, diff: %s", diff) - } - }) - } -} - -func Test_splitPRs(t *testing.T) { - testCases := []struct { - name string - prs []prRequest - chunkSize int - expected [][]prRequest - }{ - { - name: "empty PR list", - prs: []prRequest{}, - chunkSize: 40, - expected: nil, - }, - { - name: "only one PR", - prs: []prRequest{ - { - Repo: "org/repo", - Number: 1, - Url: "github.com/org/repo/1", - Title: "Test PR 1", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - }, - chunkSize: 40, - expected: [][]prRequest{ - { - { - Repo: "org/repo", - Number: 1, - Url: "github.com/org/repo/1", - Title: "Test PR 1", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - }, - }, - }, - { - name: "split PRs into chunks", - prs: []prRequest{ - { - Repo: "org/repo", - Number: 1, - Url: "github.com/org/repo/1", - Title: "Test PR 1", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - { - Repo: "org/repo", - Number: 2, - Url: "github.com/org/repo/2", - Title: "Test PR 2", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - { - Repo: "org/repo", - Number: 3, - Url: "github.com/org/repo/3", - Title: "Test PR 3", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - }, - chunkSize: 2, - expected: [][]prRequest{ - { - { - Repo: "org/repo", - Number: 1, - Url: "github.com/org/repo/1", - Title: "Test PR 1", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - { - Repo: "org/repo", - Number: 2, - Url: "github.com/org/repo/2", - Title: "Test PR 2", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - }, - { - { - Repo: "org/repo", - Number: 3, - Url: "github.com/org/repo/3", - Title: "Test PR 3", - Author: "user", - Created: time.Time{}, - LastUpdated: time.Time{}, - }, - }, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual := splitPRs(tc.prs, tc.chunkSize) - if diff := cmp.Diff(actual, tc.expected); diff != "" { - t.Fatalf("split PRs returned unexpected result, diff: %s", diff) - } - }) - } -} diff --git a/cmd/publicize/README.md b/cmd/publicize/README.md deleted file mode 100644 index 7aa87260545..00000000000 --- a/cmd/publicize/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Publicize - -This tool is an external prow plugin. Its purpose is to merge the commit histories of two repositories. - -Configuration file example: -```yaml -repositories: - openshift/private-repo: openshift/public-repo - openshift/another-private-repo: openshift/another-public-repo -``` - - -# Requirements - -- Responds only in `/publicize` comments -- The plugin runs for only merged pull requests -- User must be an organization member -- The destination repository must be defined in the configuration file - - -# Workflow - -- User comments `/publicize` to a merged pull request -- The plugin clones the destination repository -- The pull request's repository and the branch is being fetched -- Histories are being merged with a new commit message -- The branch is being pushed to the destination repository - - diff --git a/cmd/publicize/main.go b/cmd/publicize/main.go deleted file mode 100644 index 773e19d386d..00000000000 --- a/cmd/publicize/main.go +++ /dev/null @@ -1,221 +0,0 @@ -package main - -import ( - "context" - "errors" - "flag" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/ghodss/yaml" - "github.com/sirupsen/logrus" - - utilerrors "k8s.io/apimachinery/pkg/util/errors" - prowconfig "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/config/secret" - prowflagutil "sigs.k8s.io/prow/pkg/flagutil" - "sigs.k8s.io/prow/pkg/githubeventserver" - "sigs.k8s.io/prow/pkg/interrupts" - "sigs.k8s.io/prow/pkg/logrusutil" - "sigs.k8s.io/prow/pkg/pjutil" - - "github.com/openshift/ci-tools/pkg/util/gzip" -) - -type Config struct { - Repositories map[string]string `json:"repositories,omitempty"` -} - -func (c *Config) validate() error { - var errs []error - for downstreamRepo, upstreamRepo := range c.Repositories { - if len(strings.Split(downstreamRepo, "/")) != 2 { - return fmt.Errorf("%s should be in org/repo format", downstreamRepo) - } - - if len(strings.Split(upstreamRepo, "/")) != 2 { - return fmt.Errorf("%s should be in org/repo format", upstreamRepo) - } - } - - return utilerrors.NewAggregate(errs) -} - -type options struct { - mut *sync.RWMutex - - configPath string - gitName string - gitEmail string - githubLogin string - webhookSecretFile string - - config *Config - - githubEventServerOptions githubeventserver.Options - github prowflagutil.GitHubOptions - - dryRun bool -} - -func gatherOptions() options { - o := options{} - fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - fs.BoolVar(&o.dryRun, "dry-run", true, "Dry run for testing. Uses API tokens but does not mutate.") - - fs.StringVar(&o.configPath, "config-path", "/etc/config/config.yaml", "Path to publicize configuration.") - fs.StringVar(&o.webhookSecretFile, "hmac-secret-file", "/etc/webhook/hmac", "Path to the file containing the GitHub HMAC secret.") - - fs.StringVar(&o.githubLogin, "github-login", "", "The GitHub username to use.") - fs.StringVar(&o.gitName, "git-name", "", "The name to use on the git commit. Requires --git-email. If not specified, uses the system default.") - fs.StringVar(&o.gitEmail, "git-email", "", "The email to use on the git commit. Requires --git-name. If not specified, uses the system default.") - - o.github.AddFlags(fs) - o.githubEventServerOptions.Bind(fs) - - if err := fs.Parse(os.Args[1:]); err != nil { - logrus.WithError(err).Fatalf("cannot parse args: '%s'", os.Args[1:]) - } - return o -} - -func (o *options) Validate() error { - if o.githubLogin == "" { - return errors.New("--github-login must be specified") - } - - if (o.gitEmail == "") != (o.gitName == "") { - return errors.New("--git-name and --git-email must be specified") - } - - if err := o.github.Validate(o.dryRun); err != nil { - return err - } - - bytes, err := gzip.ReadFileMaybeGZIP(o.configPath) - if err != nil { - return fmt.Errorf("Couldn't read publicize configuration file: %v", o.configPath) - } - - if err := yaml.Unmarshal(bytes, &o.config); err != nil { - return fmt.Errorf("Couldn't unmarshal publicize configuration: %w", err) - } - - if err := o.config.validate(); err != nil { - return err - } - - if err := o.githubEventServerOptions.DefaultAndValidate(); err != nil { - return err - } - - m := sync.RWMutex{} - o.mut = &m - - return nil -} - -func (o *options) getConfigWatchAndUpdate() (func(ctx context.Context), error) { - errFunc := func(err error, msg string) { - logrus.WithError(err).Error(msg) - } - - eventFunc := func() error { - bytes, err := gzip.ReadFileMaybeGZIP(o.configPath) - if err != nil { - return fmt.Errorf("Couldn't read publicize configuration file %s: %w", o.configPath, err) - } - - var c *Config - if err := yaml.Unmarshal(bytes, &c); err != nil { - return fmt.Errorf("Couldn't unmarshal publicize configuration: %w", err) - } - - if err := c.validate(); err != nil { - return err - } - - o.mut.Lock() - defer o.mut.Unlock() - o.config = c - logrus.Info("Configuration updated") - - return nil - } - watcher, err := prowconfig.GetCMMountWatcher(eventFunc, errFunc, filepath.Dir(o.configPath)) - if err != nil { - return nil, fmt.Errorf("couldn't get the file watcher: %w", err) - } - - return watcher, nil -} - -func main() { - logrusutil.ComponentInit() - logger := logrus.WithField("plugin", "publicize") - - o := gatherOptions() - if err := o.Validate(); err != nil { - logger.Fatalf("Invalid options: %v", err) - } - - configWatchAndUpdate, err := o.getConfigWatchAndUpdate() - if err != nil { - logger.WithError(err).Fatal("couldn't get config file watch and update function") - } - interrupts.Run(configWatchAndUpdate) - - if err := secret.Add(o.github.TokenPath, o.webhookSecretFile); err != nil { - logger.WithError(err).Fatal("Error starting secrets agent.") - } - - webhookTokenGenerator := secret.GetTokenGenerator(o.webhookSecretFile) - githubTokenGenerator := secret.GetTokenGenerator(o.github.TokenPath) - - githubClient, err := o.github.GitHubClient(o.dryRun) - if err != nil { - logger.WithError(err).Fatal("Error getting GitHub client.") - } - - gitClient, err := o.github.GitClientFactory("", nil, o.dryRun, false) - if err != nil { - logger.WithError(err).Fatal("Error getting Git client.") - } - - serv := &server{ - githubTokenGenerator: githubTokenGenerator, - config: func() *Config { - o.mut.Lock() - defer o.mut.Unlock() - return o.config - }, - ghc: githubClient, - gc: gitClient, - gitName: o.gitName, - gitEmail: o.gitEmail, - githubLogin: o.githubLogin, - githubHost: o.github.Host, - dry: o.dryRun, - } - - eventServer := githubeventserver.New(o.githubEventServerOptions, webhookTokenGenerator, logger) - eventServer.RegisterHandleIssueCommentEvent(serv.handleIssueComment) - eventServer.RegisterHelpProvider(helpProvider, logger) - - interrupts.OnInterrupt(func() { - eventServer.GracefulShutdown() - if err := gitClient.Clean(); err != nil { - logrus.WithError(err).Error("Could not clean up git client cache.") - } - }) - - health := pjutil.NewHealth() - health.ServeReady() - - interrupts.ListenAndServe(eventServer, time.Second*30) - interrupts.WaitForGracefulShutdown() -} diff --git a/cmd/publicize/server.go b/cmd/publicize/server.go deleted file mode 100644 index 959aa56b77d..00000000000 --- a/cmd/publicize/server.go +++ /dev/null @@ -1,199 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "net/url" - "regexp" - "strings" - - "github.com/sirupsen/logrus" - - prowconfig "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/git/v2" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/pluginhelp" -) - -type githubClient interface { - IsMember(org, user string) (bool, error) - CreateComment(owner, repo string, number int, comment string) error - GetPullRequest(org, repo string, number int) (*github.PullRequest, error) -} - -var publicizeRe = regexp.MustCompile(`(?mi)^/publicize\s*$`) - -func helpProvider(_ []prowconfig.OrgRepo) (*pluginhelp.PluginHelp, error) { - pluginHelp := &pluginhelp.PluginHelp{ - Description: `The publicize plugin is used for merging and push the commit history to a configured upstream repository.`, - } - pluginHelp.AddCommand(pluginhelp.Command{ - Usage: "/publicize", - Description: "Merge the commit histories into the configured upstream repository", - WhoCanUse: "Members of the trusted organization for the repo.", - Examples: []string{"/publicize"}, - }) - return pluginHelp, nil -} - -type server struct { - githubTokenGenerator func() []byte - - gitName string - gitEmail string - githubLogin string - githubHost string - - config func() *Config - - ghc githubClient - gc git.ClientFactory - - dry bool -} - -func (s *server) handleIssueComment(l *logrus.Entry, ic github.IssueCommentEvent) { - if !publicizeRe.MatchString(ic.Comment.Body) { - return - } - - org := ic.Repo.Owner.Login - repo := ic.Repo.Name - num := ic.Issue.Number - - logger := logrus.WithFields(logrus.Fields{ - github.OrgLogField: org, - github.RepoLogField: repo, - github.PrLogField: num, - }) - - logger.Info("Publicize of PR has been requested.") - - pr, err := s.ghc.GetPullRequest(org, repo, num) - if err != nil { - logger.WithError(err).Warn("couldn't get pull request") - s.createComment(ic, fmt.Sprintf("couldn't get pull request: %v", err), logger) - return - } - baseBranch := pr.Base.Ref - - if err := s.checkPrerequisites(logger, pr, ic); err != nil { - logger.WithError(err).Warn("error occurred while checking for prerequisites") - s.createComment(ic, fmt.Sprintf("%v", err), logger) - return - } - - destOrgRepo := s.config().Repositories[fmt.Sprintf("%s/%s", org, repo)] - destOrg := strings.Split(destOrgRepo, "/")[0] - destRepo := strings.Split(destOrgRepo, "/")[1] - - sourceRemoteResolver := func() (string, error) { - remote := &url.URL{Scheme: "https", Host: s.githubHost, Path: fmt.Sprintf("%s/%s", org, repo)} - remote.User = url.UserPassword(s.githubLogin, string(s.githubTokenGenerator())) - return remote.String(), nil - } - - logger.Infof("Trying to merge the PR to destination: %s/%s@%s", destOrg, destRepo, baseBranch) - headCommitRef, err := s.mergeAndPushToRemote(org, repo, destOrg, destRepo, sourceRemoteResolver, baseBranch, s.dry) - if err != nil { - logger.WithError(err).Warnf("couldn't merge the pull request and push to the destination: %v", err) - s.createComment(ic, fmt.Sprintf("Publicize failed with error: %v", err), logger) - return - } - - destOrgRepoLink := fmt.Sprintf("https://%s/%s/commit/%s", s.githubHost, destOrgRepo, headCommitRef) - s.createComment(ic, fmt.Sprintf("A merge commit [%s/%s@%s](%s) was created in the upstream repository to publish this work.", - destOrg, destRepo, baseBranch, destOrgRepoLink), logger) -} - -func (s *server) checkPrerequisites(logger *logrus.Entry, pr *github.PullRequest, ic github.IssueCommentEvent) error { - if !ic.Issue.IsPullRequest() { - return errors.New("Publicize plugin can only be used in pull requests") - } - - org := ic.Repo.Owner.Login - commentAuthor := ic.Comment.User.Login - - // Only org members should be able to publicize a pull request. - ok, err := s.ghc.IsMember(org, commentAuthor) - if err != nil { - return fmt.Errorf("couldn't check members: %w", err) - } - if !ok { - return fmt.Errorf("only [%s](https://github.com/orgs/%s/people) org members may request publication of a private pull request", org, org) - } - - if !pr.Merged { - return errors.New("cannot publicize an unmerged pull request") - } - - repo := ic.Repo.Name - logger.Info("Searching for upstream repository") - if _, ok := s.config().Repositories[fmt.Sprintf("%s/%s", org, repo)]; !ok { - logger.Warn("There is no upstream repository configured for the current repository.") - return fmt.Errorf("cannot publicize because there is no upstream repository configured for %s/%s", org, repo) - } - - return nil -} - -func (s *server) mergeAndPushToRemote(sourceOrg, sourceRepo, destOrg, destRepo string, sourceRemoteResolver func() (string, error), branch string, dry bool) (string, error) { - repoClient, err := s.gc.ClientFor(destOrg, destRepo) - if err != nil { - return "", fmt.Errorf("couldn't create repoclient for repository %s/%s: %w", destOrg, destRepo, err) - } - - defer func() { - if err := repoClient.Clean(); err != nil { - logrus.WithError(err).Error("couldn't clean temporary repo folder") - } - }() - - if err := repoClient.Checkout(branch); err != nil { - return "", fmt.Errorf("couldn't checkout to branch %s: %w", branch, err) - } - - if err := repoClient.FetchFromRemote(sourceRemoteResolver, branch); err != nil { - return "", fmt.Errorf("couldn't fetch from the downstream repository: %w", err) - } - - if err := repoClient.Config("user.name", s.gitName); err != nil { - return "", fmt.Errorf("couldn't set config user.name=%s: %w", s.gitName, err) - } - - if err := repoClient.Config("user.email", s.gitEmail); err != nil { - return "", fmt.Errorf("couldn't set config user.name=%s: %w", s.gitEmail, err) - } - - if err := repoClient.Config("commit.gpgsign", "false"); err != nil { - return "", fmt.Errorf("failed to disable gpg signing: %w", err) - } - - merged, err := repoClient.MergeWithStrategy("FETCH_HEAD", "merge", git.MergeOpt{CommitMessage: "DPTP reconciliation from downstream"}) - if err != nil { - return "", fmt.Errorf("couldn't merge %s/%s, merge --abort failed with reason: %w", destOrg, destRepo, err) - } - - if !merged { - return "", fmt.Errorf("couldn't merge %s/%s, due to merge conflict. You will need to create a new PR in %s/%s which merges/resolves from %s/%s. Once this PR merges, you can then use /publicize there to merge all changes into the the public repository.", destOrg, destRepo, sourceOrg, sourceRepo, destOrg, destRepo) - } - - if !dry { - if err := repoClient.PushToCentral(branch, false); err != nil { - return "", fmt.Errorf("couldn't push to upstream repository: %w", err) - } - } - - refs, err := repoClient.ShowRef(branch) - if err != nil { - return "", fmt.Errorf("couldn't get ref for branch %s: %w", branch, err) - } - - return strings.Split(refs, "\n")[0], nil -} - -func (s *server) createComment(ic github.IssueCommentEvent, message string, logger *logrus.Entry) { - if err := s.ghc.CreateComment(ic.Repo.Owner.Login, ic.Repo.Name, ic.Issue.Number, fmt.Sprintf("@%s: %s", ic.Comment.User.Login, message)); err != nil { - logger.WithError(err).Warn("coulnd't create comment") - } -} diff --git a/cmd/publicize/server_test.go b/cmd/publicize/server_test.go deleted file mode 100644 index f879eb77627..00000000000 --- a/cmd/publicize/server_test.go +++ /dev/null @@ -1,412 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "os" - "os/exec" - "path" - "path/filepath" - "reflect" - "strings" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/git/localgit" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/github/fakegithub" -) - -func TestCheckPrerequisites(t *testing.T) { - logrus.SetLevel(logrus.DebugLevel) - - testCases := []struct { - id string - commentBody string - - isMember bool - isMerged bool - isPullRequest bool - - repositories map[string]string - expectedComments []github.IssueComment - expectedError error - }{ - - { - id: "issue is not a pull request", - commentBody: "/publicize", - isMember: true, - repositories: map[string]string{"org-priv/repo": "org/repo"}, - expectedError: errors.New("Publicize plugin can only be used in pull requests"), - }, - { - id: "user is no org member", - commentBody: "/publicize", - isMember: false, - isPullRequest: true, - repositories: map[string]string{"org-priv/repo": "org/repo"}, - expectedError: errors.New("only [org-priv](https://github.com/orgs/org-priv/people) org members may request publication of a private pull request"), - }, - { - id: "pull request is not merged", - commentBody: "/publicize", - isMember: true, - isMerged: false, - isPullRequest: true, - repositories: map[string]string{"org-priv/repo": "org/repo"}, - expectedError: errors.New("cannot publicize an unmerged pull request"), - }, - { - id: "repository has no upstream repository configured", - commentBody: "/publicize", - isMember: true, - isMerged: true, - isPullRequest: true, - repositories: map[string]string{"org-priv/anotherRepo": "org/anotherRepo"}, - expectedError: errors.New("cannot publicize because there is no upstream repository configured for org-priv/repo"), - }, - { - id: "a hapy publicize", - commentBody: "/publicize", - isMember: true, - isMerged: true, - isPullRequest: true, - repositories: map[string]string{"org-priv/repo": "org/repo"}, - expectedError: nil, - }, - } - - for _, tc := range testCases { - t.Run(tc.id, func(t *testing.T) { - issueState := "open" - if tc.isMerged { - issueState = "closed" - } - - prNumber := 1111 - fc := &fakegithub.FakeClient{ - IssueComments: make(map[int][]github.IssueComment), - OrgMembers: map[string][]string{"org-priv": {"k8s-ci-robot"}}, - PullRequests: map[int]*github.PullRequest{ - prNumber: { - ID: 1, - Number: prNumber, - User: github.User{Login: "pr-user"}, - Title: tc.id, - Body: tc.id, - Merged: tc.isMerged, - Base: github.PullRequestBranch{Ref: "master"}, - }, - }, - } - - localGit, gcf, err := localgit.NewV2() - defer func() { - if err := localGit.Clean(); err != nil { - t.Errorf("couldn't clean localgit temp folders: %v", err) - } - - if err := gcf.Clean(); err != nil { - t.Errorf("coulnd't clean git client's temp folders: %v", err) - } - }() - - if err != nil { - t.Fatal(err) - } - - if err := localGit.MakeFakeRepo("org", "repo"); err != nil { - t.Fatal(err) - } - - if err := localGit.MakeFakeRepo("org-priv", "repo"); err != nil { - t.Fatal(err) - } - - ice := github.IssueCommentEvent{ - Action: github.IssueCommentActionCreated, - Comment: github.IssueComment{ - Body: tc.commentBody, - }, - Issue: github.Issue{ - User: github.User{Login: "k8s-ci-robot"}, - Number: prNumber, - State: issueState, - Assignees: []github.User{{Login: "dptp-assignee"}}, - }, - - Repo: github.Repo{ - Owner: github.User{Login: "org-priv"}, - Name: "repo", - }, - } - - if tc.isPullRequest { - ice.Issue.PullRequest = &struct{}{} - } - - if tc.isMember { - ice.Comment.User.Login = "k8s-ci-robot" - } - - serv := &server{ - gitName: "test", - gitEmail: "test@test.com", - ghc: fc, - gc: gcf, - config: func() *Config { - c := &Config{} - c.Repositories = tc.repositories - return c - }, - dry: true, - } - - actualErr := serv.checkPrerequisites(logrus.WithField("id", tc.id), fc.PullRequests[1111], ice) - - if !reflect.DeepEqual(actualErr, tc.expectedError) { - t.Fatalf("%s", cmp.Diff(actualErr.Error(), tc.expectedError.Error())) - } - }) - } -} - -func TestMergeAndPushToRemote(t *testing.T) { - publicOrg, publicRepo := "openshift", "test" - privateOrg, privateRepo := "openshift-priv", "test" - fixedTime := time.Now() - fixedTimeUnix := fixedTime.Unix() - makeRepo := func(localgit *localgit.LocalGit, org, repo string, init func() error) error { - if err := localgit.MakeFakeRepo(org, repo); err != nil { - return fmt.Errorf("couldn't create fake repo for %s/%s: %w", org, repo, err) - } - // The test relies on the repository created by MakeFakeRepo generating - // the same history across calls, which can only happen if time remains - // constant, as it is part of the Git commit hash. We amend the initial - // file and commit with fixed dates to guarantee identical commits. - path := filepath.Join(localgit.Dir, org, repo) - initial := filepath.Join(path, "initial") - if err := os.Chtimes(initial, fixedTime, fixedTime); err != nil { - return err - } - cmd := exec.Command("git", "add", initial) - cmd.Dir = path - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to amend fake repository file at %q: %w, output:\n%s", path, err, out) - } - cmd = exec.Command("git", "commit", "--quiet", "--amend", "--reset-author", "--no-edit") - cmd.Dir = path - cmd.Env = append( - os.Environ(), - fmt.Sprintf("GIT_AUTHOR_DATE=%d", fixedTimeUnix), - fmt.Sprintf("GIT_COMMITTER_DATE=%d", fixedTimeUnix)) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to amend fake repository at %q: %w, output:\n%s", path, err, out) - } - if init != nil { - if err := init(); err != nil { - return err - } - } - return nil - } - localgit, gc, err := localgit.NewV2() - if err != nil { - t.Fatalf("couldn't create localgit: %v", err) - } - localgit.InitialBranch = "master" - - defer func() { - if err := gc.Clean(); err != nil { - t.Fatalf("couldn't clean git cache: %v", err) - } - }() - - testCases := []struct { - id string - branch string - remoteResolver func() (string, error) - privateGitRepo func() error - publicGitRepo func() error - errExpectedMsg string - }{ - { - id: "wrong branch, error expected", - branch: "whatever", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, privateOrg, privateRepo), nil - }, - errExpectedMsg: "couldn't checkout to branch whatever: error checking out \"whatever\": exit status 1 error: pathspec 'whatever' did not match any file(s) known to git", - }, - { - id: "wrong remote resolver, error expected", - branch: "refs/heads/master", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, "wrongOrg", "wrongRepo"), nil - }, - errExpectedMsg: fmt.Sprintf(`couldn't fetch from the downstream repository: error fetching refs/heads/master from %s/wrongOrg/wrongRepo: exit status 128 fatal: '%s/wrongOrg/wrongRepo' does not appear to be a git repository -fatal: Could not read from remote repository. - -Please make sure you have the correct access rights -and the repository exists. -`, localgit.Dir, localgit.Dir), - }, - { - id: "nothing to merge, no error expected", - branch: "refs/heads/master", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, privateOrg, privateRepo), nil - }, - }, - { - id: "one commit to publicize, no error expected", - branch: "refs/heads/master", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, privateOrg, privateRepo), nil - }, - privateGitRepo: func() error { - filesToCommit := map[string][]byte{"test-file": []byte("TEST")} - if err := localgit.AddCommit(privateOrg, privateRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - }, - { - id: "multiple commits to publicize, no error expected", - branch: "refs/heads/master", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, privateOrg, privateRepo), nil - }, - privateGitRepo: func() error { - filesToCommit := map[string][]byte{ - "test-file": []byte("TEST"), - "test-file2": []byte("TEST"), - "test-file3": []byte("TEST"), - } - if err := localgit.AddCommit(privateOrg, privateRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - }, - { - id: "different histories without conflict, no error expected", - branch: "refs/heads/master", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, privateOrg, privateRepo), nil - }, - privateGitRepo: func() error { - filesToCommit := map[string][]byte{ - "test-file": []byte("TEST"), - "test-file2": []byte("TEST"), - "test-file3": []byte("TEST"), - } - if err := localgit.AddCommit(privateOrg, privateRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - publicGitRepo: func() error { - filesToCommit := map[string][]byte{ - "test-file4": []byte("TEST"), - "test-file5": []byte("TEST"), - "test-file6": []byte("TEST"), - } - if err := localgit.AddCommit(publicOrg, publicRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - }, - { - id: "one commit to publicize with conflict, error expected", - branch: "refs/heads/master", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, privateOrg, privateRepo), nil - }, - privateGitRepo: func() error { - filesToCommit := map[string][]byte{"test-file": []byte("CONFLICT")} - if err := localgit.AddCommit(privateOrg, privateRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - publicGitRepo: func() error { - filesToCommit := map[string][]byte{"test-file": []byte("TEST")} - if err := localgit.AddCommit(publicOrg, publicRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - errExpectedMsg: "couldn't merge openshift/test, due to merge conflict. You will need to create a new PR in openshift-priv/test which merges/resolves from openshift/test. Once this PR merges, you can then use /publicize there to merge all changes into the the public repository.", - }, - { - id: "multiple commits with one conflict, error expected", - branch: "refs/heads/master", - remoteResolver: func() (string, error) { - return path.Join(localgit.Dir, privateOrg, privateRepo), nil - }, - privateGitRepo: func() error { - filesToCommit := map[string][]byte{ - "test-file": []byte("CONFLICT"), - "test-file2": []byte("TEST"), - "test-file3": []byte("TEST"), - } - if err := localgit.AddCommit(privateOrg, privateRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - publicGitRepo: func() error { - filesToCommit := map[string][]byte{ - "test-file": []byte("TEST"), - "test-file5": []byte("TEST"), - "test-file6": []byte("TEST"), - } - if err := localgit.AddCommit(publicOrg, publicRepo, filesToCommit); err != nil { - return fmt.Errorf("couldn't add commit: %w", err) - } - return nil - }, - errExpectedMsg: "couldn't merge openshift/test, due to merge conflict. You will need to create a new PR in openshift-priv/test which merges/resolves from openshift/test. Once this PR merges, you can then use /publicize there to merge all changes into the the public repository.", - }, - } - - s := &server{ - gc: gc, - gitName: "Foo Bar", - gitEmail: "foobar@redhat.com", - } - - for _, tc := range testCases { - t.Run(tc.id, func(t *testing.T) { - if err := makeRepo(localgit, privateOrg, privateRepo, tc.privateGitRepo); err != nil { - t.Fatalf("couldn't create private fake repo: %v", err) - } - if err := makeRepo(localgit, publicOrg, publicRepo, tc.publicGitRepo); err != nil { - t.Fatalf("couldn't create public fake repo: %v", err) - } - headCommitRef, err := s.mergeAndPushToRemote(privateOrg, privateRepo, publicOrg, publicRepo, tc.remoteResolver, tc.branch, false) - if err != nil && tc.errExpectedMsg == "" { - t.Fatalf("error not expected: %v", err) - } - - if err != nil && !strings.HasPrefix(err.Error(), tc.errExpectedMsg) { - t.Fatal(cmp.Diff(err.Error(), tc.errExpectedMsg)) - } - - if err == nil && len(headCommitRef) != 40 { - t.Fatalf("expected a head commit ref to be 40 chars long: %s", headCommitRef) - } - - if err := localgit.Clean(); err != nil { - t.Fatalf("couldn't clean temporary folders: %v", err) - } - }) - } -} diff --git a/cmd/retester/main.go b/cmd/retester/main.go deleted file mode 100644 index cf4491fa1bf..00000000000 --- a/cmd/retester/main.go +++ /dev/null @@ -1,167 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "os" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" - "github.com/sirupsen/logrus" - - prowConfig "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/flagutil" - prowflagutil "sigs.k8s.io/prow/pkg/flagutil" - configflagutil "sigs.k8s.io/prow/pkg/flagutil/config" - "sigs.k8s.io/prow/pkg/interrupts" - "sigs.k8s.io/prow/pkg/metrics" - - "github.com/openshift/ci-tools/pkg/retester" -) - -type options struct { - config configflagutil.ConfigOptions - github prowflagutil.GitHubOptions - - runOnce bool - dryRun bool - - intervalRaw string - cacheRecordAgeRaw string - - interval time.Duration - - cacheFile string - cacheFileOnS3 bool - cacheRecordAge time.Duration - - configFile string -} - -func (o *options) Validate() error { - for _, group := range []flagutil.OptionGroup{&o.github, &o.config} { - if err := group.Validate(o.dryRun); err != nil { - return err - } - } - if o.configFile == "" { - return fmt.Errorf("--config-file is required") - } - if o.cacheFileOnS3 && o.cacheFile == "" { - return fmt.Errorf("--cache-file is required if --cache-file-on-s3 is set to true") - } - return nil -} - -func (o *options) complete() error { - var err error - o.interval, err = time.ParseDuration(o.intervalRaw) - if err != nil { - return fmt.Errorf("invalid --interval: %w", err) - } - o.cacheRecordAge, err = time.ParseDuration(o.cacheRecordAgeRaw) - if err != nil { - return fmt.Errorf("invalid --cache-record-age: %w", err) - } - return nil -} - -func gatherOptions() options { - o := options{} - fs := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - - fs.BoolVar(&o.dryRun, "dry-run", true, "Dry run for testing. Uses API tokens but does not mutate.") - fs.BoolVar(&o.runOnce, "run-once", false, "If true, run only once then quit.") - fs.BoolVar(&o.cacheFileOnS3, "cache-file-on-s3", false, "If true, use aws s3 bucket to store the cache file.") - fs.StringVar(&o.intervalRaw, "interval", "1h", "Parseable duration string that specifies the sync period") - fs.StringVar(&o.cacheFile, "cache-file", "", "File to persist cache. No persistence of cache if not set") - fs.StringVar(&o.cacheRecordAgeRaw, "cache-record-age", "168h", "Parseable duration string that specifies how long a cache record lives in cache after the last time it was considered") - fs.StringVar(&o.configFile, "config-file", "", "Path to the configure file of the retest.") - - for _, group := range []flagutil.OptionGroup{&o.github, &o.config} { - group.AddFlags(fs) - } - - if err := fs.Parse(os.Args[1:]); err != nil { - logrus.WithError(err).Fatal("could not parse input") - } - - return o -} - -func main() { - o := gatherOptions() - if err := o.complete(); err != nil { - logrus.WithError(err).Fatal("failed to complete options") - } - if err := o.Validate(); err != nil { - logrus.WithError(err).Fatal("failed to validate options") - } - - gc, err := o.github.GitHubClient(o.dryRun) - if err != nil { - logrus.WithError(err).Fatal("Error creating github client") - } - - gitClient, err := o.github.GitClientFactory("", nil, o.dryRun, false) - if err != nil { - logrus.WithError(err).Fatal("Error getting Git client.") - } - - configAgent, err := o.config.ConfigAgent() - if err != nil { - logrus.WithError(err).Fatal("Error starting config agent.") - } - - config, err := retester.LoadConfig(o.configFile) - if err != nil { - logrus.WithError(err).Fatal("Failed to load config from file") - } - - ctx := interrupts.Context() - - var awsConfig aws.Config - if o.cacheFileOnS3 { - awsConfig, err = awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion("us-east-1")) - if err != nil { - logrus.WithError(err).Fatal("Failed to create AWS config.") - } - _, err = awsConfig.Credentials.Retrieve(ctx) - if err != nil { - logrus.WithError(err).Fatal("Error getting AWS credentials.") - } - } - - c := retester.NewController(ctx, gc, configAgent.Config, gitClient, o.github.AppPrivateKeyPath != "", o.cacheFile, o.cacheRecordAge, config, &awsConfig) - - metrics.ExposeMetrics("retester", prowConfig.PushGateway{}, prowflagutil.DefaultMetricsPort) - - interrupts.OnInterrupt(func() { - if err := gitClient.Clean(); err != nil { - logrus.WithError(err).Error("Could not clean up git client cache.") - } - }) - - execute(ctx, c) - if o.runOnce { - return - } - - // This a sleep that can be interrupted :) - select { - case <-interrupts.Context().Done(): - return - case <-time.After(o.interval): - } - - interrupts.Tick(func() { execute(ctx, c) }, func() time.Duration { return o.interval }) - interrupts.WaitForGracefulShutdown() -} - -func execute(ctx context.Context, c *retester.RetestController) { - if err := c.Run(ctx); err != nil { - logrus.WithError(err).Error("Error running") - } -} diff --git a/cmd/retester/main_test.go b/cmd/retester/main_test.go deleted file mode 100644 index 22bc1f6f569..00000000000 --- a/cmd/retester/main_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package main - -import ( - "errors" - "os" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - - flagutil "sigs.k8s.io/prow/pkg/flagutil/config" - - "github.com/openshift/ci-tools/pkg/testhelper" -) - -var ( - sevenDays = 7 * 24 * time.Hour -) - -func TestGatherOptions(t *testing.T) { - testCases := []struct { - name string - args []string - expected options - }{ - { - name: "default", - args: []string{"cmd"}, - expected: options{ - dryRun: true, - intervalRaw: "1h", - cacheRecordAgeRaw: "168h", - }, - }, - { - name: "basic case", - args: []string{"cmd", "--run-once=true", "--interval=2h", "--cache-file=cache.yaml", "--cache-record-age=100h", "--config-file=config.yaml"}, - expected: options{ - runOnce: true, - dryRun: true, - intervalRaw: "2h", - cacheFile: "cache.yaml", - cacheRecordAgeRaw: "100h", - configFile: "config.yaml", - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - os.Args = tc.args - actual := gatherOptions() - if diff := cmp.Diff(tc.expected.runOnce, actual.runOnce); diff != "" { - t.Errorf("%s run once differs from expected:\n%s", tc.name, diff) - } - if diff := cmp.Diff(tc.expected.dryRun, actual.dryRun); diff != "" { - t.Errorf("%s dry run differs from expected:\n%s", tc.name, diff) - } - if diff := cmp.Diff(tc.expected.intervalRaw, actual.intervalRaw); diff != "" { - t.Errorf("%s interval raw differs from expected:\n%s", tc.name, diff) - } - if diff := cmp.Diff(tc.expected.cacheFile, actual.cacheFile); diff != "" { - t.Errorf("%s cache file differs from expected:\n%s", tc.name, diff) - } - if diff := cmp.Diff(tc.expected.cacheRecordAgeRaw, actual.cacheRecordAgeRaw); diff != "" { - t.Errorf("%s cache record age raw differs from expected:\n%s", tc.name, diff) - } - if diff := cmp.Diff(tc.expected.configFile, actual.configFile); diff != "" { - t.Errorf("%s config file differs from expected:\n%s", tc.name, diff) - } - }) - } -} - -func TestValidate(t *testing.T) { - testCases := []struct { - name string - o options - expected error - }{ - { - name: "basic", - o: options{ - config: flagutil.ConfigOptions{ConfigPath: "/etc/config/config.yaml"}, - dryRun: true, - interval: time.Hour, - cacheRecordAge: sevenDays, - configFile: "/etc/retester/config.yaml", - }, - }, - { - name: "no-config-file", - o: options{ - config: flagutil.ConfigOptions{ConfigPath: "/etc/config/config.yaml"}, - dryRun: true, - interval: time.Hour, - cacheRecordAge: sevenDays, - }, - expected: errors.New("--config-file is required"), - }, - { - name: "no-config-path", - o: options{ - //not set config path results: error(*errors.errorString) *{s: "-- is mandatory"} - config: flagutil.ConfigOptions{ConfigPathFlagName: "config-path"}, - }, - expected: errors.New("--config-path is mandatory"), - }, - { - name: "cache-file not set when using aws", - o: options{ - config: flagutil.ConfigOptions{ConfigPath: "/etc/config/config.yaml"}, - configFile: "/etc/retester/config.yaml", - dryRun: true, - interval: time.Hour, - cacheRecordAge: sevenDays, - cacheFileOnS3: true, - }, - expected: errors.New("--cache-file is required if --cache-file-on-s3 is set to true"), - }, - { - name: "cache-file not set when using local file cache", - o: options{ - config: flagutil.ConfigOptions{ConfigPath: "/etc/config/config.yaml"}, - configFile: "/etc/retester/config.yaml", - dryRun: true, - interval: time.Hour, - cacheRecordAge: sevenDays, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := tc.o.Validate() - if diff := cmp.Diff(tc.expected, err, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - }) - } -} - -func TestComplete(t *testing.T) { - testCases := []struct { - name string - o options - expected error - expectedInterval time.Duration - expectedCacheRecordAge time.Duration - }{ - { - name: "basic", - o: options{ - intervalRaw: "1h", - cacheRecordAgeRaw: "168h", - }, - expectedInterval: time.Hour, - expectedCacheRecordAge: sevenDays, - }, - { - name: "wrong format", - o: options{ - intervalRaw: "wrong format", - cacheRecordAgeRaw: "168h", - }, - expected: errors.New("invalid --interval: time: invalid duration \"wrong format\""), - }, { - name: "empty", - o: options{ - intervalRaw: "1h", - }, - expected: errors.New("invalid --cache-record-age: time: invalid duration \"\""), - expectedInterval: time.Hour, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := tc.o.complete() - if diff := cmp.Diff(tc.expected, err, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - if diff := cmp.Diff(tc.expectedInterval, tc.o.interval); diff != "" { - t.Errorf("%s interval differs from expected:\n%s", tc.name, diff) - } - if diff := cmp.Diff(tc.expectedCacheRecordAge, tc.o.cacheRecordAge); diff != "" { - t.Errorf("%s cache record age differs from expected:\n%s", tc.name, diff) - } - }) - } -} diff --git a/images/backport-verifier/Dockerfile b/images/backport-verifier/Dockerfile deleted file mode 100644 index 9157466990c..00000000000 --- a/images/backport-verifier/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest - -ADD backport-verifier /usr/bin/backport-verifier - -ENTRYPOINT ["/usr/bin/backport-verifier"] diff --git a/images/ci-scheduling-webhook/Dockerfile b/images/ci-scheduling-webhook/Dockerfile deleted file mode 100644 index 45cbd53dfc2..00000000000 --- a/images/ci-scheduling-webhook/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest -LABEL maintainer="jupierce@redhat.com" -ADD ci-scheduling-webhook /usr/bin/ci-scheduling-webhook -ENTRYPOINT ["/usr/bin/ci-scheduling-webhook", "--port", "8443"] \ No newline at end of file diff --git a/images/determinize-peribolos/Dockerfile b/images/determinize-peribolos/Dockerfile deleted file mode 100644 index b87fdef233d..00000000000 --- a/images/determinize-peribolos/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest - -ADD determinize-peribolos /usr/bin/determinize-peribolos -ENTRYPOINT ["/usr/bin/determinize-peribolos"] diff --git a/images/gpu-scheduling-webhook/Dockerfile b/images/gpu-scheduling-webhook/Dockerfile deleted file mode 100644 index f684b2b862e..00000000000 --- a/images/gpu-scheduling-webhook/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest -LABEL maintainer="dgemoli@redhat.com" - -ADD gpu-scheduling-webhook /usr/bin/gpu-scheduling-webhook -ENTRYPOINT ["/usr/bin/gpu-scheduling-webhook"] \ No newline at end of file diff --git a/images/helpdesk-faq/Dockerfile b/images/helpdesk-faq/Dockerfile deleted file mode 100644 index d6dc3e2d331..00000000000 --- a/images/helpdesk-faq/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest -LABEL maintainer="sgoeddel@redhat.com" - -ADD helpdesk-faq /usr/bin/helpdesk-faq -ENTRYPOINT ["/usr/bin/helpdesk-faq"] diff --git a/images/pipeline-controller/Dockerfile b/images/pipeline-controller/Dockerfile deleted file mode 100644 index 5a112bd4e7f..00000000000 --- a/images/pipeline-controller/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest -LABEL maintainer="jguzik@redhat.com" - -ADD pipeline-controller /usr/bin/pipeline-controller -ENTRYPOINT ["/usr/bin/pipeline-controller"] diff --git a/images/pr-reminder/Dockerfile b/images/pr-reminder/Dockerfile deleted file mode 100644 index 3bf0e9b148d..00000000000 --- a/images/pr-reminder/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest -LABEL maintainer="sgoeddel@redhat.com" - -RUN microdnf install -y tar gzip - -ADD pr-reminder /usr/bin/pr-reminder -ENTRYPOINT ["/usr/bin/pr-reminder"] diff --git a/images/publicize/Dockerfile b/images/publicize/Dockerfile deleted file mode 100644 index 164bf59c328..00000000000 --- a/images/publicize/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest - -LABEL maintainer="nmoraiti@redhat.com" - -RUN microdnf install -y git -ADD publicize /usr/bin/publicize - -ENTRYPOINT ["/usr/bin/publicize"] diff --git a/pkg/retester/cache.go b/pkg/retester/cache.go deleted file mode 100644 index d1851f9cda0..00000000000 --- a/pkg/retester/cache.go +++ /dev/null @@ -1,21 +0,0 @@ -package retester - -import ( - "context" - - "sigs.k8s.io/prow/pkg/tide" -) - -type retestBackoffAction int - -const ( - retestBackoffHold = iota - retestBackoffPause - retestBackoffRetest -) - -type backoffCache interface { - check(pr tide.PullRequest, baseSha string, policy RetesterPolicy) (retestBackoffAction, string) - load(ctx context.Context) error - save(ctx context.Context) error -} diff --git a/pkg/retester/fileBackOffCache.go b/pkg/retester/fileBackOffCache.go deleted file mode 100644 index c289c6af30c..00000000000 --- a/pkg/retester/fileBackOffCache.go +++ /dev/null @@ -1,136 +0,0 @@ -package retester - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/sirupsen/logrus" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/prow/pkg/tide" - "sigs.k8s.io/yaml" -) - -type fileBackoffCache struct { - cache map[string]*pullRequest - file string - cacheRecordAge time.Duration - logger *logrus.Entry -} - -func (b *fileBackoffCache) load(_ context.Context) error { - b.logger.WithField("backOffCache", "fileBackoffCache").Info("Loading the cache file ...") - return b.loadFromDiskNow(time.Now()) -} - -func (b *fileBackoffCache) loadFromDiskNow(now time.Time) error { - if b.file == "" { - return nil - } - if _, err := os.Stat(b.file); errors.Is(err, os.ErrNotExist) { - b.logger.WithField("file", b.file).Info("cache file does not exit") - return nil - } - bytes, err := os.ReadFile(b.file) - if err != nil { - return fmt.Errorf("failed to read file %s: %w", b.file, err) - } - cache, err := loadAndDelete(bytes, b.logger, now, b.cacheRecordAge) - if err != nil { - return err - } - b.cache = cache - return nil -} - -// loadAndDelete loads content into cache and deletes old records from cache -func loadAndDelete(content []byte, logger *logrus.Entry, now time.Time, cacheRecordAge time.Duration) (map[string]*pullRequest, error) { - cache := map[string]*pullRequest{} - if err := yaml.Unmarshal(content, &cache); err != nil { - return nil, fmt.Errorf("failed to unmarshal: %w", err) - } - for key, pr := range cache { - if age := now.Sub(pr.LastConsideredTime.Time); age > cacheRecordAge { - logger.WithField("key", key).WithField("LastConsideredTime", pr.LastConsideredTime). - WithField("age", age).Info("deleting old record from cache") - delete(cache, key) - } - } - return cache, nil -} - -func (b *fileBackoffCache) save(_ context.Context) (ret error) { - if b.file == "" { - return nil - } - bytes, err := yaml.Marshal(b.cache) - if err != nil { - return fmt.Errorf("failed to marshal: %w", err) - } - // write to a temp file and rename it to the cache file to ensure "atomic write": - // either it is complete or nothing - tmpFile, err := os.CreateTemp(filepath.Dir(b.file), "tmp-backoff-cache") - if err != nil { - return fmt.Errorf("failed to create a temp file: %w", err) - } - tmp := tmpFile.Name() - defer func() { - // do nothing when the file does not exist, e.g., write failed, or it has been renamed. - if _, err := os.Stat(tmp); errors.Is(err, os.ErrNotExist) { - return - } - if err := os.Remove(tmp); err != nil { - ret = fmt.Errorf("failed to delete file %s: %w", tmp, err) - } - }() - - if err := os.WriteFile(tmp, bytes, 0644); err != nil { - return fmt.Errorf("failed to write file %s: %w", tmp, err) - } - if err := os.Rename(tmp, b.file); err != nil { - return fmt.Errorf("failed to rename file from %s to %s: %w", tmp, b.file, err) - } - return ret -} - -func (b *fileBackoffCache) check(pr tide.PullRequest, baseSha string, policy RetesterPolicy) (retestBackoffAction, string) { - return check(&b.cache, pr, baseSha, policy) -} - -// check updates the cache and returns a retestBackoffAction according to baseSha, policy, and number of retests performed for the PR. -func check(cache *map[string]*pullRequest, pr tide.PullRequest, baseSha string, policy RetesterPolicy) (retestBackoffAction, string) { - key := prKey(&pr) - if _, has := (*cache)[key]; !has { - (*cache)[key] = &pullRequest{} - } - record := (*cache)[key] - record.LastConsideredTime = metav1.Now() - if currentPRSha := string(pr.HeadRefOID); record.PRSha != currentPRSha { - record.PRSha = currentPRSha - record.RetestsForPrSha = 0 - record.RetestsForBaseSha = 0 - } - if record.BaseSha != baseSha { - record.BaseSha = baseSha - record.RetestsForBaseSha = 0 - } - - if record.RetestsForPrSha == policy.MaxRetestsForSha { - record.RetestsForPrSha = 0 - record.RetestsForBaseSha = 0 - return retestBackoffHold, fmt.Sprintf("Revision %s was retested %d times: holding", record.PRSha, policy.MaxRetestsForSha) - } - - if record.RetestsForBaseSha == policy.MaxRetestsForShaAndBase { - return retestBackoffPause, fmt.Sprintf("Revision %s was retested %d times against base HEAD %s: pausing", record.PRSha, policy.MaxRetestsForShaAndBase, record.BaseSha) - } - - record.RetestsForBaseSha++ - record.RetestsForPrSha++ - - return retestBackoffRetest, fmt.Sprintf("Remaining retests: %d against base HEAD %s and %d for PR HEAD %s in total", policy.MaxRetestsForShaAndBase-record.RetestsForBaseSha, record.BaseSha, policy.MaxRetestsForSha-record.RetestsForPrSha, record.PRSha) -} diff --git a/pkg/retester/retester.go b/pkg/retester/retester.go deleted file mode 100644 index de3c3280082..00000000000 --- a/pkg/retester/retester.go +++ /dev/null @@ -1,548 +0,0 @@ -package retester - -import ( - "context" - "fmt" - "os" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/prometheus/client_golang/prometheus" - githubql "github.com/shurcooL/githubv4" - "github.com/sirupsen/logrus" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "sigs.k8s.io/prow/pkg/config" - "sigs.k8s.io/prow/pkg/git/v2" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/tide" - "sigs.k8s.io/yaml" -) - -type githubClient interface { - GetCombinedStatus(org, repo, ref string) (*github.CombinedStatus, error) - GetRef(string, string, string) (string, error) - QueryWithGitHubAppsSupport(ctx context.Context, q interface{}, vars map[string]interface{}, org string) error - CreateComment(owner, repo string, number int, comment string) error -} - -// pullRequest represents GitHub PR and number of retests. -type pullRequest struct { - PRSha string `json:"pr_sha,omitempty"` - BaseSha string `json:"base_sha,omitempty"` - RetestsForPrSha int `json:"retests_for_pr_sha,omitempty"` - RetestsForBaseSha int `json:"retests_for_base_sha,omitempty"` - LastConsideredTime metav1.Time `json:"last_considered_time,omitempty"` -} - -var ( - retestTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "retest_total", - Help: "Number of retest commands in total issued by the tool.", - }, - []string{"org", "repo"}, - ) -) - -func init() { - // Metrics have to be registered to be exposed: - prometheus.MustRegister(retestTotal) -} - -// Config is retester configuration for all configured repos and orgs. -// It has three levels: global, org, and repo. A specific level overrides general ones. -type Config struct { - Retester Retester `json:"retester"` -} - -// Retester is global level configuration for retester configuration. -// Policy is overridden by specific levels when they are enabled. -type Retester struct { - RetesterPolicy `json:",inline"` - Oranizations map[string]Oranization `json:"orgs,omitempty"` -} - -// Oranization is org level configuration for retester configuration. -// Policy is overridden by repo level when it is enabled. -type Oranization struct { - RetesterPolicy `json:",inline"` - Repos map[string]Repo `json:"repos"` -} - -// Repo is repo level configuration for retester configuration. -// Policy overridden all general levels. -type Repo struct { - RetesterPolicy `json:",inline"` -} - -// RetesterPolicy for the retester/org/repo. -// When merging policies, a 0 value results in inheriting the parent policy. -// False in level repo means disabled repo. Nothing can change that. -// True/False in level org means enabled/disabled org. But repo can be disabled/enabled. -type RetesterPolicy struct { - MaxRetestsForShaAndBase int `json:"max_retests_for_sha_and_base,omitempty"` - MaxRetestsForSha int `json:"max_retests_for_sha,omitempty"` - Enabled *bool `json:"enabled,omitempty"` -} - -// LoadConfig loads retester configuration via file. -func LoadConfig(configFilePath string) (*Config, error) { - data, err := os.ReadFile(configFilePath) - if err != nil { - return nil, fmt.Errorf("failed to read config %w", err) - } - - var config Config - if err := yaml.Unmarshal(data, &config); err != nil { - return nil, fmt.Errorf("failed to unmarshal config %w", err) - } - - return &config, nil -} - -// RetestController represents a retest controller which controls what the retester does. -type RetestController struct { - ghClient githubClient - gitClient git.ClientFactory - configGetter config.Getter - - logger *logrus.Entry - - usesGitHubApp bool - backoff backoffCache - - config *Config -} - -func (c *Config) GetRetesterPolicy(org, repo string) (RetesterPolicy, error) { - policy := RetesterPolicy{} - if c.Retester.RetesterPolicy == policy && len(c.Retester.Oranizations) == 0 { - return policy, nil - } - if orgStruct, ok := c.Retester.Oranizations[org]; ok && orgStruct.Enabled != nil { - policy.Enabled = orgStruct.Enabled - var repoStruct Repo - if repoStruct, ok = orgStruct.Repos[repo]; ok && repoStruct.Enabled != nil { - policy.Enabled = repoStruct.Enabled - if *repoStruct.Enabled { - // set max retests repo level value - if repoStruct.MaxRetestsForSha != 0 { - policy.MaxRetestsForSha = repoStruct.MaxRetestsForSha - } - if repoStruct.MaxRetestsForShaAndBase != 0 { - policy.MaxRetestsForShaAndBase = repoStruct.MaxRetestsForShaAndBase - } - } else { - return RetesterPolicy{}, nil - } - } - if *orgStruct.Enabled { - // set max retests org level value - if orgStruct.MaxRetestsForSha != 0 && policy.MaxRetestsForSha == 0 { - policy.MaxRetestsForSha = orgStruct.MaxRetestsForSha - } - if orgStruct.MaxRetestsForShaAndBase != 0 && policy.MaxRetestsForShaAndBase == 0 { - policy.MaxRetestsForShaAndBase = orgStruct.MaxRetestsForShaAndBase - } - } - if !*policy.Enabled && (c.Retester.Enabled == nil || !*c.Retester.Enabled) { - return RetesterPolicy{}, nil - } - } else if c.Retester.Enabled == nil || !*c.Retester.Enabled { - return RetesterPolicy{}, nil - } - // set max retests default value - if policy.MaxRetestsForSha == 0 { - policy.MaxRetestsForSha = c.Retester.MaxRetestsForSha - } - if policy.MaxRetestsForShaAndBase == 0 { - policy.MaxRetestsForShaAndBase = c.Retester.MaxRetestsForShaAndBase - } - return policy, nil -} - -func validatePolicies(policy RetesterPolicy) []error { - var errs []error - if policy.Enabled != nil { - if *policy.Enabled { - if policy.MaxRetestsForSha < 0 { - errs = append(errs, fmt.Errorf("max_retest_for_sha has invalid value: %d", policy.MaxRetestsForSha)) - } - if policy.MaxRetestsForShaAndBase < 0 { - errs = append(errs, fmt.Errorf("max_retests_for_sha_and_base has invalid value: %d", policy.MaxRetestsForShaAndBase)) - } - if policy.MaxRetestsForSha < policy.MaxRetestsForShaAndBase { - errs = append(errs, fmt.Errorf("max_retest_for_sha value can't be lower than max_retests_for_sha_and_base value: %d < %d", policy.MaxRetestsForSha, policy.MaxRetestsForShaAndBase)) - } - } else { - return nil - } - } - return errs -} - -// NewController generates a retest controller. -func NewController(ctx context.Context, ghClient githubClient, cfg config.Getter, gitClient git.ClientFactory, usesApp bool, cacheFile string, cacheRecordAge time.Duration, config *Config, awsConfig *aws.Config) *RetestController { - logger := logrus.NewEntry(logrus.StandardLogger()) - var backoff backoffCache - if awsConfig != nil { - backoff = &s3BackOffCache{cache: map[string]*pullRequest{}, file: cacheFile, cacheRecordAge: cacheRecordAge, logger: logger, awsClient: s3.NewFromConfig(*awsConfig)} - } else { - backoff = &fileBackoffCache{cache: map[string]*pullRequest{}, file: cacheFile, cacheRecordAge: cacheRecordAge, logger: logger} - } - - ret := &RetestController{ - ghClient: ghClient, - gitClient: gitClient, - configGetter: cfg, - logger: logger, - usesGitHubApp: usesApp, - backoff: backoff, - config: config, - } - if err := ret.backoff.load(ctx); err != nil { - logger.WithError(err).Warn("Failed to load backoff cache from disk") - } - return ret -} - -func prUrl(pr tide.PullRequest) string { - return fmt.Sprintf("https://github.com/%s/%s/pull/%d", pr.Repository.Owner.Login, pr.Repository.Name, pr.Number) -} - -// Run implements the business of the controller: filters out the pull requests satisfying the Tide's merge criteria except some required job failed and issue "/retest required" command on them. -func (c *RetestController) Run(ctx context.Context) error { - // Input: Tide Config - // Output: A list of PRs that are filter out by the queries in Tide's config - candidates, err := findCandidates(c.configGetter, c.ghClient, c.usesGitHubApp, c.logger) - if err != nil { - return fmt.Errorf("failed to find retestable candidates: %w", err) - } - return c.runWithCandidates(ctx, candidates) -} - -func (c *RetestController) runWithCandidates(ctx context.Context, candidates map[string]tide.PullRequest) error { - logrus.Infof("Found %d candidates for retest (pass label criteria, fail some tests)", len(candidates)) - for _, pr := range candidates { - logrus.Infof("Candidate PR: %s", prUrl(pr)) - } - - candidates = c.enabledPRs(candidates) - logrus.Infof("Remaining %d candidates for retest (from an enabled org or repo)", len(candidates)) - - candidates, err := c.atLeastOneRequiredJob(candidates) - if err != nil { - return fmt.Errorf("failed to filter candidate PRs that have at least one required job: %w", err) - } - - logrus.Infof("Remaining %d candidates for retest (fail at least one required prowjob)", len(candidates)) - for _, pr := range candidates { - logrus.Infof("Candidate PR: %s", prUrl(pr)) - } - - var errs []error - for _, pr := range candidates { - errs = append(errs, c.retestOrBackoff(pr)) - } - - if err := c.backoff.save(ctx); err != nil { - errs = append(errs, fmt.Errorf("failed to save cache to disk: %w", err)) - } - logrus.Info("Sync finished") - return utilerrors.NewAggregate(errs) -} - -func (c *RetestController) createComment(pr tide.PullRequest, cmd, message string) { - comment := fmt.Sprintf("%s\n\n%s\n", cmd, message) - if err := c.ghClient.CreateComment(string(pr.Repository.Owner.Login), string(pr.Repository.Name), int(pr.Number), comment); err != nil { - c.logger.WithField("comment", comment).WithError(err).Error("failed to create a comment") - } else if cmd == "/retest-required" { - retestTotal.With(prometheus.Labels{"org": string(pr.Repository.Owner.Login), "repo": string(pr.Repository.Name)}).Inc() - } -} - -func (c *RetestController) retestOrBackoff(pr tide.PullRequest) error { - branchRef := string(pr.BaseRef.Prefix) + string(pr.BaseRef.Name) - baseSha, err := c.ghClient.GetRef(string(pr.Repository.Owner.Login), string(pr.Repository.Name), strings.TrimPrefix(branchRef, "refs/")) - if err != nil { - return err - } - - var policy RetesterPolicy - org := string(pr.Repository.Owner.Login) - repo := string(pr.Repository.Name) - if policy, err = c.config.GetRetesterPolicy(org, repo); err != nil { - return fmt.Errorf("failed to get the max retests: %w", err) - } - if validationErrors := validatePolicies(policy); len(validationErrors) != 0 { - return fmt.Errorf("failed to validate retester policy: %v", validationErrors) - } - - action, message := c.backoff.check(pr, baseSha, policy) - switch action { - case retestBackoffHold: - c.createComment(pr, "/hold", message) - case retestBackoffPause: - c.logger.Infof("%s: %s (%s)", prUrl(pr), "no comment", message) - case retestBackoffRetest: - c.createComment(pr, "/retest-required", message) - } - return nil -} - -func findCandidates(config config.Getter, gc githubClient, usesGitHubAppsAuth bool, logger *logrus.Entry) (map[string]tide.PullRequest, error) { - prs, err := query(config, gc, usesGitHubAppsAuth, logger) - if err != nil { - return nil, fmt.Errorf("failed to query GitHub for prs: %w", err) - } - - return prs, nil -} - -func (c *RetestController) atLeastOneRequiredJob(candidates map[string]tide.PullRequest) (map[string]tide.PullRequest, error) { - output := map[string]tide.PullRequest{} - for key, pr := range candidates { - // Get all non-optional Prowjobs configured for this org/repo/branch that could run on this PR - presubmits := c.presubmitsForPRByContext(pr) - - c.logger.Infof("Pull request %s has %d relevant presubmits configured", key, len(presubmits)) - - // If this PR cannot ever trigger any required Prowjob, `/retest-required` will never do anything useful for it - if len(presubmits) == 0 { - continue - } - - // Get all contexts on the HEAD of the PR - contexts, err := headContexts(c.ghClient, pr) - if err != nil { - c.logger.WithError(err).Errorf("Failed to get contexts for %s", key) - return nil, err - } - c.logger.Infof("HEAD commit of PR %s has %d contexts", key, len(contexts)) - - for _, ctx := range contexts { - if ctx.State != githubql.StatusStateFailure { - continue - } - // It is enough to find a single failed context that corresponds to a required Prowjob - if ps, has := presubmits[string(ctx.Context)]; has { - c.logger.Infof("PR %s fails required job %s (context=%s)", key, ps.Name, ctx.Context) - output[key] = pr - break - } - } - if _, ok := output[key]; !ok { - c.logger.Infof("PR %s has no failing context of a required Prowjob", key) - } - } - return output, nil -} - -// refactor out the query function from the tide's controller -// https://github.com/kubernetes/test-infra/blob/0d18a317a517e1bdb9a2c728a46fbeb3642445dd/prow/tide/tide.go#L450 -func query(config config.Getter, gc githubClient, usesGitHubAppsAuth bool, logger *logrus.Entry) (map[string]tide.PullRequest, error) { - prs := make(map[string]tide.PullRequest) - var errs []error - for i, query := range config().Tide.Queries { - - // Use org-sharded queries only when GitHub apps auth is in use - var queries map[string]string - if usesGitHubAppsAuth { - queries = query.OrgQueries() - } else { - queries = map[string]string{"": query.Query()} - } - - for org, q := range queries { - org, i := org, i - q := "status:failure " + q - - results, err := search(gc.QueryWithGitHubAppsSupport, logger, q, time.Time{}, time.Now(), org) - - if err != nil && len(results) == 0 { - logger.WithField("query", q).WithError(err).Warn("Failed to execute query.") - errs = append(errs, fmt.Errorf("query %d, err: %w", i, err)) - continue - } - if err != nil { - logger.WithError(err).WithField("query", q).Warning("found partial results") - } - - for _, pr := range results { - prs[prKey(&pr)] = pr - } - } - logrus.Infof("Finished query: %d", i) - } - - return prs, utilerrors.NewAggregate(errs) -} - -func prKey(pr *tide.PullRequest) string { - return fmt.Sprintf("%s#%d", string(pr.Repository.NameWithOwner), int(pr.Number)) -} - -type querier func(ctx context.Context, q interface{}, vars map[string]interface{}, org string) error - -func floor(t time.Time) time.Time { - if t.Before(github.FoundingYear) { - return github.FoundingYear - } - return t -} -func datedQuery(q string, start, end time.Time) string { - return fmt.Sprintf("%s %s", q, dateToken(start, end)) -} - -// dateToken generates a GitHub search query token for the specified date range. -// See: https://help.github.com/articles/understanding-the-search-syntax/#query-for-dates -func dateToken(start, end time.Time) string { - // GitHub's GraphQL API silently fails if you provide it with an invalid time - // string. - // Dates before 1970 (unix epoch) are considered invalid. - startString, endString := "*", "*" - if start.Year() >= 1970 { - startString = start.Format(github.SearchTimeFormat) - } - if end.Year() >= 1970 { - endString = end.Format(github.SearchTimeFormat) - } - return fmt.Sprintf("updated:%s..%s", startString, endString) -} - -type searchQuery struct { - RateLimit struct { - Cost githubql.Int - Remaining githubql.Int - } - Search struct { - PageInfo struct { - HasNextPage githubql.Boolean - EndCursor githubql.String - } - Nodes []PRNode - } `graphql:"search(type: ISSUE, first: 37, after: $searchCursor, query: $query)"` -} -type PRNode struct { - PullRequest tide.PullRequest `graphql:"... on PullRequest"` -} - -func search(query querier, log *logrus.Entry, q string, start, end time.Time, org string) ([]tide.PullRequest, error) { - start = floor(start) - end = floor(end) - log = log.WithFields(logrus.Fields{ - "query": q, - "start": start.String(), - "end": end.String(), - }) - requestStart := time.Now() - var cursor *githubql.String - vars := map[string]interface{}{ - "query": githubql.String(datedQuery(q, start, end)), - "searchCursor": cursor, - } - - var totalCost, remaining int - var ret []tide.PullRequest - var sq searchQuery - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - for { - log.Debug("Sending query") - if err := query(ctx, &sq, vars, org); err != nil { - if cursor != nil { - err = fmt.Errorf("cursor: %q, err: %w", *cursor, err) - } - return ret, err - } - totalCost += int(sq.RateLimit.Cost) - remaining = int(sq.RateLimit.Remaining) - for _, n := range sq.Search.Nodes { - ret = append(ret, n.PullRequest) - } - if !sq.Search.PageInfo.HasNextPage { - break - } - cursor = &sq.Search.PageInfo.EndCursor - vars["searchCursor"] = cursor - log = log.WithField("searchCursor", *cursor) - } - log.WithFields(logrus.Fields{ - "duration": time.Since(requestStart).String(), - "pr_found_count": len(ret), - "cost": totalCost, - "remaining": remaining, - }).Debug("Finished query") - return ret, nil -} - -func (c *RetestController) presubmitsForPRByContext(pr tide.PullRequest) map[string]config.Presubmit { - presubmits := map[string]config.Presubmit{} - - presubmitsForRepo := c.configGetter().GetPresubmitsStatic(string(pr.Repository.Owner.Login) + "/" + string(pr.Repository.Name)) - - for _, ps := range presubmitsForRepo { - if ps.ContextRequired() && ps.CouldRun(string(pr.BaseRef.Name)) { - presubmits[ps.Context] = ps - } - } - - return presubmits -} - -func (c *RetestController) enabledPRs(candidates map[string]tide.PullRequest) map[string]tide.PullRequest { - output := map[string]tide.PullRequest{} - for key, pr := range candidates { - org := string(pr.Repository.Owner.Login) - repo := string(pr.Repository.Name) - policy, err := c.config.GetRetesterPolicy(org, repo) - if err != nil { - c.logger.WithError(err).Warn("Failed to get retester policy") - } - if validationErrors := validatePolicies(policy); len(validationErrors) != 0 { - c.logger.Warnf("Failed to validate retester policy: %v", validationErrors) - } - if policy.Enabled != nil && *policy.Enabled { - output[key] = pr - } else { - c.logger.Infof("PR %s is not from an enabled org or repo", key) - } - } - return output -} - -// headContexts gets the status contexts for the commit with OID == pr.HeadRefOID -// -// First, we try to get this value from the commits we got with the PR query. -// Unfortunately the 'last' commit ordering is determined by author date -// not commit date so if commits are reordered non-chronologically on the PR -// branch the 'last' commit isn't necessarily the logically last commit. -// We list multiple commits with the query to increase our chance of success, -// but if we don't find the head commit we have to ask GitHub for it -// specifically (this costs an API token). -func headContexts(ghc githubClient, pr tide.PullRequest) ([]tide.Context, error) { - // We didn't get the head commit from the query (the commits must not be - // logically ordered) so we need to specifically ask GitHub for the status - // and coerce it to a graphql type. - org := string(pr.Repository.Owner.Login) - repo := string(pr.Repository.Name) - combined, err := ghc.GetCombinedStatus(org, repo, string(pr.HeadRefOID)) - if err != nil { - return nil, fmt.Errorf("failed to get the combined status: %w", err) - } - - contexts := make([]tide.Context, 0, len(combined.Statuses)) - for _, status := range combined.Statuses { - contexts = append(contexts, tide.Context{ - Context: githubql.String(status.Context), - Description: githubql.String(status.Description), - State: githubql.StatusState(strings.ToUpper(status.State)), - }) - } - - return contexts, nil -} diff --git a/pkg/retester/retester_test.go b/pkg/retester/retester_test.go deleted file mode 100644 index b695468169d..00000000000 --- a/pkg/retester/retester_test.go +++ /dev/null @@ -1,989 +0,0 @@ -package retester - -import ( - "context" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go-v2/service/s3" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/google/go-cmp/cmp" - "github.com/shurcooL/githubv4" - "github.com/sirupsen/logrus" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - configflagutil "sigs.k8s.io/prow/pkg/flagutil/config" - "sigs.k8s.io/prow/pkg/github" - "sigs.k8s.io/prow/pkg/github/fakegithub" - "sigs.k8s.io/prow/pkg/tide" - - "github.com/openshift/ci-tools/pkg/testhelper" -) - -type MyFakeClient struct { - *fakegithub.FakeClient -} - -func (f *MyFakeClient) QueryWithGitHubAppsSupport(ctx context.Context, q interface{}, vars map[string]interface{}, org string) error { - return nil -} - -func (f *MyFakeClient) GetRef(owner, repo, ref string) (string, error) { - if owner == "failed test" { - return "", fmt.Errorf("failed") - } - return "abcde", nil -} - -var ( - True = true - False = false -) - -func TestLoadConfig(t *testing.T) { - c := &Config{ - Retester: Retester{ - RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 1, MaxRetestsForShaAndBase: 1, Enabled: &True, - }, - Oranizations: map[string]Oranization{"openshift": { - RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 2, MaxRetestsForShaAndBase: 2, Enabled: &True, - }, - Repos: map[string]Repo{ - "ci-docs": {RetesterPolicy: RetesterPolicy{Enabled: &True}}, - "ci-tools": {RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 3, MaxRetestsForShaAndBase: 3, Enabled: &True, - }}, - }}, - }, - }} - - configOpenShift := &Config{ - Retester: Retester{ - RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 9, MaxRetestsForShaAndBase: 3, - }, - Oranizations: map[string]Oranization{"openshift": { - RetesterPolicy: RetesterPolicy{ - Enabled: &True, - }, - }, - - "openshift-knative": { - RetesterPolicy: RetesterPolicy{ - Enabled: &True, - }, - }, - }, - }} - - testCases := []struct { - name string - file string - expected *Config - expectedError error - }{ - { - name: "config", - file: "testdata/testconfig/config.yaml", - expected: c, - }, - { - name: "config", - file: "testdata/testconfig/openshift-config.yaml", - expected: configOpenShift, - }, - { - name: "default", - file: "testdata/testconfig/default.yaml", - expected: &Config{Retester: Retester{RetesterPolicy: RetesterPolicy{MaxRetestsForSha: 9, MaxRetestsForShaAndBase: 3}}}, - }, - { - name: "empty", - file: "testdata/testconfig/empty.yaml", - expected: &Config{Retester: Retester{}}, - }, - { - name: "no-config", - file: "testdata/testconfig/no-config.yaml", - expected: &Config{Retester: Retester{}}, - }, - { - name: "no such file", - file: "testdata/testconfig/not_found", - expectedError: fmt.Errorf("failed to read config open testdata/testconfig/not_found: no such file or directory"), - }, - { - name: "unmarshal config error", - file: "testdata/testconfig/wrong_format.yaml", - expectedError: fmt.Errorf("failed to unmarshal config error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type retester.Config"), - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual, err := LoadConfig(tc.file) - if diff := cmp.Diff(tc.expectedError, err, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - if tc.expectedError == nil { - if diff := cmp.Diff(tc.expected, actual); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - } - }) - } -} - -func TestGetRetesterPolicy(t *testing.T) { - c := &Config{ - Retester: Retester{ - RetesterPolicy: RetesterPolicy{MaxRetestsForShaAndBase: 3, MaxRetestsForSha: 9}, - Oranizations: map[string]Oranization{ - "openshift": { - RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 2, MaxRetestsForShaAndBase: 2, Enabled: &True, - }, - Repos: map[string]Repo{ - "ci-tools": {RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 3, MaxRetestsForShaAndBase: 3, Enabled: &True, - }}, - "repo-max": {RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 6, Enabled: &True, - }}, - "repo": {RetesterPolicy: RetesterPolicy{Enabled: &False}}, - }}, - "no-openshift": { - RetesterPolicy: RetesterPolicy{Enabled: &False}, - Repos: map[string]Repo{ - "true": {RetesterPolicy: RetesterPolicy{Enabled: &True}}, - "ci-tools": {RetesterPolicy: RetesterPolicy{ - MaxRetestsForSha: 4, MaxRetestsForShaAndBase: 4, Enabled: &True, - }}, - "repo": {RetesterPolicy: RetesterPolicy{Enabled: &False}}, - }}, - }, - }} - testCases := []struct { - name string - org string - repo string - config *Config - expected RetesterPolicy - expectedError error - }{ - { - name: "enabled repo and enabled org", - org: "openshift", - repo: "ci-tools", - config: c, - expected: RetesterPolicy{3, 3, &True}, - }, - { - name: "enabled repo with one max retest value and enabled org", - org: "openshift", - repo: "repo-max", - config: c, - expected: RetesterPolicy{2, 6, &True}, - }, - { - name: "enabled repo and disabled org", - org: "no-openshift", - repo: "ci-tools", - config: c, - expected: RetesterPolicy{4, 4, &True}, - }, - { - name: "disabled repo and enabled org", - org: "openshift", - repo: "repo", - config: c, - }, - { - name: "not configured repo and enabled org", - org: "openshift", - repo: "ci-docs", - config: c, - expected: RetesterPolicy{2, 2, &True}, - }, - { - name: "not configured repo and disabled org", - org: "no-openshift", - repo: "ci-docs", - config: c, - }, - { - name: "configured repo and disabled org", - org: "no-openshift", - repo: "true", - config: c, - expected: RetesterPolicy{3, 9, &True}, - }, - { - name: "not configured repo and not configured org", - org: "org", - repo: "ci-docs", - config: c, - }, - { - name: "Empty config", - org: "openshift", - repo: "ci-tools", - config: &Config{Retester{}}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual, err := tc.config.GetRetesterPolicy(tc.org, tc.repo) - if diff := cmp.Diff(tc.expectedError, err, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - if tc.expectedError == nil { - if diff := cmp.Diff(tc.expected, actual); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - } - }) - } -} - -func TestValidatePolicies(t *testing.T) { - - testCases := []struct { - name string - policy RetesterPolicy - expected []error - }{ - { - name: "basic case", - policy: RetesterPolicy{3, 9, &True}, - }, - { - name: "empty policy is valid", - }, - { - name: "disable", - policy: RetesterPolicy{-1, -1, &False}, - }, - { - name: "negative", - policy: RetesterPolicy{-1, -1, &True}, - expected: []error{ - errors.New("max_retest_for_sha has invalid value: -1"), - errors.New("max_retests_for_sha_and_base has invalid value: -1")}, - }, - { - name: "lower", - policy: RetesterPolicy{9, 3, &True}, - expected: []error{errors.New("max_retest_for_sha value can't be lower than max_retests_for_sha_and_base value: 3 < 9")}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual := validatePolicies(tc.policy) - if diff := cmp.Diff(tc.expected, actual, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - }) - } -} - -func TestRetestOrBackoff(t *testing.T) { - True := true - config := &Config{Retester: Retester{ - RetesterPolicy: RetesterPolicy{MaxRetestsForShaAndBase: 3, MaxRetestsForSha: 9}, Oranizations: map[string]Oranization{ - "org": {RetesterPolicy: RetesterPolicy{Enabled: &True}}, - }, - }} - ghc := &MyFakeClient{fakegithub.NewFakeClient()} - var num githubv4.Int = 123 - var num2 githubv4.Int = 321 - pr123 := github.PullRequest{} - pr321 := github.PullRequest{} - ghc.PullRequests = map[int]*github.PullRequest{123: &pr123, 321: &pr321} - logger := logrus.NewEntry( - logrus.StandardLogger()) - - testCases := []struct { - name string - pr tide.PullRequest - c *RetestController - expected string - expectedError error - }{ - { - name: "basic case", - pr: tide.PullRequest{ - Number: num, - Author: struct{ Login githubv4.String }{Login: "org"}, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "repo", Owner: struct{ Login githubv4.String }{Login: "org"}}, - }, - c: &RetestController{ - ghClient: ghc, - logger: logger, - backoff: &fileBackoffCache{cache: map[string]*pullRequest{}, logger: logger}, - config: config, - }, - expected: "/retest-required\n\nRemaining retests: 2 against base HEAD abcde and 8 for PR HEAD in total\n", - }, - { - name: "failed test", - pr: tide.PullRequest{ - Number: num2, - Author: struct{ Login githubv4.String }{Login: "failed test"}, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "repo", Owner: struct{ Login githubv4.String }{Login: "failed test"}}, - }, - c: &RetestController{ - ghClient: ghc, - logger: logger, - backoff: &fileBackoffCache{cache: map[string]*pullRequest{}, logger: logger}, - config: config, - }, - expected: "", - expectedError: fmt.Errorf("failed"), - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := tc.c.retestOrBackoff(tc.pr) - if diff := cmp.Diff(tc.expectedError, err, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - if tc.expectedError == nil { - actual := "" - if len(ghc.IssueComments[int(tc.pr.Number)]) != 0 { - actual = ghc.IssueComments[int(tc.pr.Number)][0].Body - } - if diff := cmp.Diff(tc.expected, actual); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - } - }) - } -} - -func TestEnabledPRs(t *testing.T) { - True := true - False := false - logger := logrus.NewEntry(logrus.StandardLogger()) - testCases := []struct { - name string - c *RetestController - candidates map[string]tide.PullRequest - expected map[string]tide.PullRequest - }{ - { - name: "basic case", - c: &RetestController{ - config: &Config{Retester: Retester{ - RetesterPolicy: RetesterPolicy{MaxRetestsForShaAndBase: 1, MaxRetestsForSha: 1, Enabled: &True}, Oranizations: map[string]Oranization{ - "openshift": {RetesterPolicy: RetesterPolicy{Enabled: &False}, - Repos: map[string]Repo{"ci-tools": {RetesterPolicy: RetesterPolicy{Enabled: &True}}}, - }, - "org-a": {RetesterPolicy: RetesterPolicy{Enabled: &True}}, - }, - }}, - logger: logger, - }, - candidates: map[string]tide.PullRequest{ - "a": { - Number: 1, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "ci-tools", Owner: struct{ Login githubv4.String }{Login: "openshift"}}, - }, - "b": { - Number: 1, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "some-tools", Owner: struct{ Login githubv4.String }{Login: "openshift"}}, - }, - "c": { - Number: 1, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "some-tools", Owner: struct{ Login githubv4.String }{Login: "org-a"}}, - }, - "d": { - Number: 1, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "some-tools", Owner: struct{ Login githubv4.String }{Login: "org-b"}}, - }, - }, - expected: map[string]tide.PullRequest{ - "a": { - Number: 1, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "ci-tools", Owner: struct{ Login githubv4.String }{Login: "openshift"}}, - }, - "c": { - Number: 1, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "some-tools", Owner: struct{ Login githubv4.String }{Login: "org-a"}}, - }, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual := tc.c.enabledPRs(tc.candidates) - if diff := cmp.Diff(tc.expected, actual); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - }) - } -} - -var ( - now = metav1.NewTime(time.Date(2022, 8, 18, 0, 0, 0, 0, time.UTC)) - justNow = metav1.NewTime(now.Add(-time.Minute)) -) - -func TestLoadFromDiskNow(t *testing.T) { - logger := logrus.NewEntry(logrus.StandardLogger()) - testCases := []struct { - name string - cache fileBackoffCache - now time.Time - file string - expectedMap map[string]*pullRequest - expected error - }{ - { - name: "basic case", - file: "basic_case.yaml", - cache: fileBackoffCache{ - cacheRecordAge: time.Hour, - logger: logger, - }, - expectedMap: map[string]*pullRequest{"pr1": {PRSha: "sha1", RetestsForBaseSha: 2, RetestsForPrSha: 3, LastConsideredTime: now}, - "pr3": {PRSha: "sha2", RetestsForBaseSha: 1, RetestsForPrSha: 3, LastConsideredTime: justNow}}, - now: time.Date(2022, 8, 18, 0, 0, 0, 0, time.UTC), - }, - { - name: "empty file name", - file: "", - }, - { - name: "file no exist", - file: "no-exist.cache", - cache: fileBackoffCache{ - logger: logger, - }, - }, - { - name: "wrong format", - file: "wrong_format.yaml", - cache: fileBackoffCache{ - logger: logger, - }, - expected: errors.New("failed to unmarshal: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type map[string]*retester.pullRequest"), - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.file != "" { - tc.cache.file = filepath.Join("testdata", "loadFromDiskNow", tc.file) - } - actual := tc.cache.loadFromDiskNow(tc.now) - if diff := cmp.Diff(tc.expected, actual, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - if tc.expected == nil && actual == nil { - if diff := cmp.Diff(tc.expectedMap, tc.cache.cache); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - } - }) - } -} - -func TestSaveFileBackoffCache(t *testing.T) { - dir, err := os.MkdirTemp("", "saveToDisk") - if err != nil { - t.Errorf("failed to create temporary directory %s: %s", dir, err.Error()) - } - defer os.RemoveAll(dir) - testCases := []struct { - name string - cache fileBackoffCache - expected error - expectedContent string - }{ - { - name: "basic case", - cache: fileBackoffCache{cache: map[string]*pullRequest{"pr1": {PRSha: "sha1", RetestsForBaseSha: 2, RetestsForPrSha: 3, LastConsideredTime: now}, - "pr3": {PRSha: "sha2", RetestsForBaseSha: 1, RetestsForPrSha: 3, LastConsideredTime: justNow}}}, - expectedContent: `pr1: - last_considered_time: "2022-08-18T00:00:00Z" - pr_sha: sha1 - retests_for_base_sha: 2 - retests_for_pr_sha: 3 -pr3: - last_considered_time: "2022-08-17T23:59:00Z" - pr_sha: sha2 - retests_for_base_sha: 1 - retests_for_pr_sha: 3 -`, - }, - { - name: "empty file name", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if tc.name != "empty file name" { - tc.cache.file = filepath.Join(dir, tc.name) - } - actual := tc.cache.save(context.TODO()) - if diff := cmp.Diff(tc.expected, actual, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - if tc.expected == nil && tc.cache.file != "" { - actualBytes, err := os.ReadFile(tc.cache.file) - if err != nil { - t.Errorf("failed to read file %s: %s", tc.cache.file, err.Error()) - } - actualContent := string(actualBytes) - if diff := cmp.Diff(tc.expectedContent, actualContent); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - } - }) - } -} - -type mockS3Client struct { - PutObjectFunc func(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) - GetObjectFunc func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) -} - -func (m *mockS3Client) PutObject(ctx context.Context, input *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { - return m.PutObjectFunc(input) -} - -func (m *mockS3Client) GetObject(ctx context.Context, input *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { - return m.GetObjectFunc(input) -} - -func TestSaveS3BackoffCache(t *testing.T) { - svc := &mockS3Client{} - svc.PutObjectFunc = func(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { - return &s3.PutObjectOutput{}, nil - } - - sampleErrorMsg := "some AWS error" - svcFaulty := &mockS3Client{} - svcFaulty.PutObjectFunc = func(input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { - return &s3.PutObjectOutput{}, fmt.Errorf("%s", sampleErrorMsg) - } - - testCases := []struct { - name string - s3cache s3BackOffCache - expectedError error - }{ - { - name: "successful case", - s3cache: s3BackOffCache{ - cache: map[string]*pullRequest{ - "pr1": {PRSha: "sha1", RetestsForBaseSha: 2, RetestsForPrSha: 3, LastConsideredTime: now}, - "pr2": {PRSha: "sha2", RetestsForBaseSha: 1, RetestsForPrSha: 3, LastConsideredTime: justNow}, - }, - awsClient: svc, - }, - expectedError: nil, - }, - { - name: "unsuccessful case", - s3cache: s3BackOffCache{ - file: "file-name", - awsClient: svcFaulty, - }, - expectedError: fmt.Errorf("failed to upload file file-name into prow-retester bucket: %s", sampleErrorMsg), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actualErr := tc.s3cache.save(context.TODO()) - if diff := cmp.Diff(tc.expectedError, actualErr, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("error differs from expected:\n%s", diff) - } - }) - } -} - -func TestLoadAndDelete(t *testing.T) { - logger := logrus.NewEntry(logrus.StandardLogger()) - - testCases := []struct { - name string - contentFile string - cacheRecordAge time.Duration - now time.Time - expectedError error - expectedMap map[string]*pullRequest - }{ - { - name: "basic case", - contentFile: "basic_case.yaml", - cacheRecordAge: time.Hour, - now: time.Date(2022, 8, 18, 0, 0, 0, 0, time.UTC), - expectedError: nil, - expectedMap: map[string]*pullRequest{ - "pr1": {PRSha: "sha1", RetestsForBaseSha: 2, RetestsForPrSha: 3, LastConsideredTime: now}, - "pr3": {PRSha: "sha2", RetestsForBaseSha: 1, RetestsForPrSha: 3, LastConsideredTime: justNow}, - }, - }, - { - name: "wrong format", - contentFile: "wrong_format.yaml", - expectedError: errors.New("failed to unmarshal: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type map[string]*retester.pullRequest"), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - content, _ := os.ReadFile(filepath.Join("testdata", "loadFromDiskNow", tc.contentFile)) - actualMap, actualErr := loadAndDelete(content, logger, tc.now, tc.cacheRecordAge) - - if diff := cmp.Diff(tc.expectedError, actualErr, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("error differs from expected:\n%s", diff) - } - if tc.expectedError == nil && actualErr == nil { - if diff := cmp.Diff(tc.expectedMap, actualMap); diff != "" { - t.Errorf("map differs from expected:\n%s", diff) - } - } - }) - } -} - -func TestLoadFromAwsNow(t *testing.T) { - now := time.Date(2023, 5, 23, 0, 0, 0, 0, time.UTC) - logger := logrus.NewEntry(logrus.StandardLogger()) - mockClient := &mockS3Client{} - mockClient.GetObjectFunc = func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { - client := &s3.GetObjectOutput{} - client.Body = io.NopCloser(strings.NewReader(`pr1: - last_considered_time: "2023-05-23T00:00:00Z" - pr_sha: sha1 - retests_for_base_sha: 2 - retests_for_pr_sha: 3 -`)) - return client, nil - } - - faultyMockClient := &mockS3Client{} - sampleErrorMsg := "some AWS error" - faultyMockClient.GetObjectFunc = func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { - return nil, fmt.Errorf("%s", sampleErrorMsg) - } - - mockClientForNoFile := &mockS3Client{} - mockClientForNoFile.GetObjectFunc = func(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { - return nil, &s3types.NoSuchKey{} - } - - testCases := []struct { - name string - s3cache s3BackOffCache - expectedError error - }{ - { - name: "successful case", - s3cache: s3BackOffCache{ - file: "file-name", - logger: logger, - awsClient: mockClient, - }, - expectedError: nil, - }, - { - name: "unsuccessful case", - s3cache: s3BackOffCache{ - file: "file-name", - logger: logger, - awsClient: faultyMockClient, - }, - expectedError: fmt.Errorf("error getting file-name file from aws s3 bucket prow-retester: %s", sampleErrorMsg), - }, - { - name: "file not yet in the bucket", - s3cache: s3BackOffCache{ - file: "file-name", - logger: logger, - awsClient: mockClientForNoFile, - }, - expectedError: nil, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actualErr := tc.s3cache.loadFromAwsNow(context.TODO(), now) - if diff := cmp.Diff(tc.expectedError, actualErr, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("error differs from expected:\n%s", diff) - } - }) - } -} - -func TestPrUrl(t *testing.T) { - pr := tide.PullRequest{ - Number: githubv4.Int(1234), - Author: struct{ Login githubv4.String }{Login: "org"}, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "repo", Owner: struct{ Login githubv4.String }{Login: "org"}}, - } - expected := "https://github.com/org/repo/pull/1234" - actual := prUrl(pr) - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("basic case differs from expected:\n%s", diff) - } -} - -func TestPrKey(t *testing.T) { - pr := tide.PullRequest{ - Number: githubv4.Int(1234), - Author: struct{ Login githubv4.String }{Login: "org"}, - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "repo", NameWithOwner: "org/repo", Owner: struct{ Login githubv4.String }{Login: "org"}}, - } - expected := "org/repo#1234" - actual := prKey(&pr) - if diff := cmp.Diff(expected, actual); diff != "" { - t.Errorf("basic case differs from expected:\n%s", diff) - } -} - -func TestCheck(t *testing.T) { - True := true - logger := logrus.NewEntry(logrus.StandardLogger()) - - testCases := []struct { - name string - cache fileBackoffCache - pr tide.PullRequest - baseSha string - policy RetesterPolicy - expected retestBackoffAction - expectedString string - }{ - { - name: "hold PR", - cache: fileBackoffCache{cache: map[string]*pullRequest{"org/repo#123": {PRSha: "holdPR", RetestsForBaseSha: 3, RetestsForPrSha: 9}}, logger: logger}, - pr: tide.PullRequest{Number: githubv4.Int(123), - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "repo", NameWithOwner: "org/repo", Owner: struct{ Login githubv4.String }{Login: "org"}}, - HeadRefOID: "holdPR"}, - policy: RetesterPolicy{3, 9, &True}, - expected: 0, - expectedString: "Revision holdPR was retested 9 times: holding", - }, - { - name: "pause PR", - cache: fileBackoffCache{cache: map[string]*pullRequest{"org/repo#123": {PRSha: "pausePR", RetestsForBaseSha: 3, RetestsForPrSha: 3}}, logger: logger}, - pr: tide.PullRequest{Number: githubv4.Int(123), - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "repo", NameWithOwner: "org/repo", Owner: struct{ Login githubv4.String }{Login: "org"}}, - HeadRefOID: "pausePR"}, - policy: RetesterPolicy{3, 9, &True}, - expected: 1, - expectedString: "Revision pausePR was retested 3 times against base HEAD : pausing", - }, - { - name: "retest PR", - cache: fileBackoffCache{cache: map[string]*pullRequest{}, logger: logger}, - pr: tide.PullRequest{HeadRefOID: "retestPR"}, - policy: RetesterPolicy{3, 9, &True}, - expected: 2, - expectedString: "Remaining retests: 2 against base HEAD and 8 for PR HEAD retestPR in total", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - actual, actualString := tc.cache.check(tc.pr, tc.baseSha, tc.policy) - if diff := cmp.Diff(tc.expected, actual); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - if diff := cmp.Diff(tc.expectedString, actualString); diff != "" { - t.Errorf("%s differs from expected:\n%s", tc.name, diff) - } - }) - } -} - -func TestRunWithCandidates(t *testing.T) { - config := &Config{Retester: Retester{ - RetesterPolicy: RetesterPolicy{MaxRetestsForShaAndBase: 3, MaxRetestsForSha: 9}, Oranizations: map[string]Oranization{ - "openshift": {RetesterPolicy: RetesterPolicy{Enabled: &True}}, - }, - }} - ghc := &MyFakeClient{fakegithub.NewFakeClient()} - - logger := logrus.NewEntry(logrus.StandardLogger()) - - testCases := []struct { - name string - prowconfig string - jobconfig string - candidates map[string]tide.PullRequest - expected error - }{ - { - name: "one candidate", - prowconfig: "simple.yaml", - jobconfig: "simple.yaml", - candidates: map[string]tide.PullRequest{ - "a": { - Number: 1, - HeadRefOID: "a", - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "ci-tools", Owner: struct{ Login githubv4.String }{Login: "openshift"}}, - }, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - tc.prowconfig = filepath.Join("testdata", "prowconfig", tc.prowconfig) - tc.jobconfig = filepath.Join("testdata", "jobconfig", tc.jobconfig) - configOpts := configflagutil.ConfigOptions{ConfigPath: tc.prowconfig, JobConfigPath: tc.jobconfig} - configAgent, err := configOpts.ConfigAgent() - if err != nil { - t.Errorf("Error starting config agent.") - } - fakeStatus := map[string]*github.CombinedStatus{ - "a": { - Statuses: []github.Status{ - { - State: "failure", - Context: "test-presubmit", - Description: "Job failed", - }, - }, - }, - } - ghc.CombinedStatuses = fakeStatus - c := &RetestController{ - ghClient: ghc, - configGetter: configAgent.Config, - logger: logger, - usesGitHubApp: true, - backoff: &fileBackoffCache{cache: map[string]*pullRequest{}, logger: logger}, - config: config, - } - actual := c.runWithCandidates(context.TODO(), tc.candidates) - if diff := cmp.Diff(tc.expected, actual, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - - }) - } -} - -func TestFindCandidates(t *testing.T) { - config := &Config{Retester: Retester{ - RetesterPolicy: RetesterPolicy{MaxRetestsForShaAndBase: 3, MaxRetestsForSha: 9}, Oranizations: map[string]Oranization{ - "openshift": {RetesterPolicy: RetesterPolicy{Enabled: &True}}, - }, - }} - ghc := &MyFakeClient{fakegithub.NewFakeClient()} - - logger := logrus.NewEntry(logrus.StandardLogger()) - - testCases := []struct { - name string - prowconfig string - candidates map[string]tide.PullRequest - expected map[string]tide.PullRequest - }{ - { - name: "no candidates", - prowconfig: "simple.yaml", - candidates: map[string]tide.PullRequest{ - "a": { - Number: 1, - HeadRefOID: "a", - Repository: struct { - Name githubv4.String - NameWithOwner githubv4.String - Owner struct{ Login githubv4.String } - }{Name: "ci-tools", Owner: struct{ Login githubv4.String }{Login: "openshift"}}, - }, - }, - expected: map[string]tide.PullRequest{}, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - tc.prowconfig = filepath.Join("testdata", "prowconfig", tc.prowconfig) - configOpts := configflagutil.ConfigOptions{ConfigPath: tc.prowconfig} - configAgent, err := configOpts.ConfigAgent() - if err != nil { - t.Errorf("Error starting config agent.") - } - c := &RetestController{ - ghClient: ghc, - configGetter: configAgent.Config, - logger: logger, - usesGitHubApp: true, - backoff: &fileBackoffCache{cache: map[string]*pullRequest{}, logger: logger}, - config: config, - } - actual, err := findCandidates(c.configGetter, c.ghClient, c.usesGitHubApp, c.logger) - if err != nil { - t.Errorf("Error finding candidates: %v", err) - } - if diff := cmp.Diff(tc.expected, actual, testhelper.EquateErrorMessage); diff != "" { - t.Errorf("Error differs from expected:\n%s", diff) - } - }) - } -} diff --git a/pkg/retester/s3BackOffCache.go b/pkg/retester/s3BackOffCache.go deleted file mode 100644 index 2ed9e60bdc0..00000000000 --- a/pkg/retester/s3BackOffCache.go +++ /dev/null @@ -1,96 +0,0 @@ -package retester - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/sirupsen/logrus" - - "sigs.k8s.io/prow/pkg/tide" - "sigs.k8s.io/yaml" -) - -const ( - retesterBucket = "prow-retester" -) - -type s3Client interface { - PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) - GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) -} - -type s3BackOffCache struct { - cache map[string]*pullRequest - file string - cacheRecordAge time.Duration - logger *logrus.Entry - - awsClient s3Client -} - -func (b *s3BackOffCache) load(ctx context.Context) error { - b.logger.WithField("backOffCache", "s3BackOffCache").Info("Loading the cache file ...") - return b.loadFromAwsNow(ctx, time.Now()) -} - -// loadFromAwsNow gets the backoff cache file from AWS S3 bucket and marshals its content into the s3BackOffCache -func (b *s3BackOffCache) loadFromAwsNow(ctx context.Context, now time.Time) error { - if b.file == "" { - return nil - } - - result, err := b.awsClient.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(retesterBucket), - Key: aws.String(b.file), - }) - if err != nil { - nsk := &s3types.NoSuchKey{} - if errors.As(err, &nsk) { - b.logger.WithField("file", b.file).Info("file doesn't exist in the s3 bucket") - return nil - } - return fmt.Errorf("error getting %s file from aws s3 bucket %s: %w", b.file, retesterBucket, err) - } - - content, err := io.ReadAll(result.Body) - if err != nil { - return fmt.Errorf("failed to read file %s: %w", b.file, err) - } - - cache, err := loadAndDelete(content, b.logger, now, b.cacheRecordAge) - if err != nil { - return err - } - b.cache = cache - return nil -} - -// save uploads the contents of s3BackOffCache to the retester AWS S3 bucket -func (b *s3BackOffCache) save(ctx context.Context) error { - content, err := yaml.Marshal(b.cache) - if err != nil { - return fmt.Errorf("failed to marshal: %w", err) - } - - _, err = b.awsClient.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(retesterBucket), - Key: aws.String(b.file), - Body: bytes.NewReader(content), - }) - if err != nil { - return fmt.Errorf("failed to upload file %s into %s bucket: %w", b.file, retesterBucket, err) - } - - return nil -} - -func (b *s3BackOffCache) check(pr tide.PullRequest, baseSha string, policy RetesterPolicy) (retestBackoffAction, string) { - return check(&b.cache, pr, baseSha, policy) -} diff --git a/pkg/retester/testdata/NewController/basic_case.yaml b/pkg/retester/testdata/NewController/basic_case.yaml deleted file mode 100644 index 910f26dae29..00000000000 --- a/pkg/retester/testdata/NewController/basic_case.yaml +++ /dev/null @@ -1,15 +0,0 @@ -pr1: - last_considered_time: "2022-08-18T00:00:00Z" - pr_sha: sha1 - retests_for_base_sha: 2 - retests_for_pr_sha: 3 -pr3: - last_considered_time: "2022-08-17T23:59:00Z" - pr_sha: sha2 - retests_for_base_sha: 1 - retests_for_pr_sha: 3 -prOld: - last_considered_time: "2021-08-17T23:59:00Z" - pr_sha: sha2 - retests_for_base_sha: 1 - retests_for_pr_sha: 3 diff --git a/pkg/retester/testdata/jobconfig/simple.yaml b/pkg/retester/testdata/jobconfig/simple.yaml deleted file mode 100644 index c88c24d5c78..00000000000 --- a/pkg/retester/testdata/jobconfig/simple.yaml +++ /dev/null @@ -1,6 +0,0 @@ -presubmits: - openshift/ci-tools: - - name: test-presubmit - spec: - containers: - - serviceAccountName: test \ No newline at end of file diff --git a/pkg/retester/testdata/loadFromDiskNow/basic_case.yaml b/pkg/retester/testdata/loadFromDiskNow/basic_case.yaml deleted file mode 100644 index 910f26dae29..00000000000 --- a/pkg/retester/testdata/loadFromDiskNow/basic_case.yaml +++ /dev/null @@ -1,15 +0,0 @@ -pr1: - last_considered_time: "2022-08-18T00:00:00Z" - pr_sha: sha1 - retests_for_base_sha: 2 - retests_for_pr_sha: 3 -pr3: - last_considered_time: "2022-08-17T23:59:00Z" - pr_sha: sha2 - retests_for_base_sha: 1 - retests_for_pr_sha: 3 -prOld: - last_considered_time: "2021-08-17T23:59:00Z" - pr_sha: sha2 - retests_for_base_sha: 1 - retests_for_pr_sha: 3 diff --git a/pkg/retester/testdata/loadFromDiskNow/wrong_format.yaml b/pkg/retester/testdata/loadFromDiskNow/wrong_format.yaml deleted file mode 100644 index 286888a0ab5..00000000000 --- a/pkg/retester/testdata/loadFromDiskNow/wrong_format.yaml +++ /dev/null @@ -1 +0,0 @@ -not yaml diff --git a/pkg/retester/testdata/prowconfig/simple.yaml b/pkg/retester/testdata/prowconfig/simple.yaml deleted file mode 100644 index a7c588a1097..00000000000 --- a/pkg/retester/testdata/prowconfig/simple.yaml +++ /dev/null @@ -1,4 +0,0 @@ -tide: - queries: - - orgs: - - "openshift" \ No newline at end of file diff --git a/pkg/retester/testdata/testconfig/config.yaml b/pkg/retester/testdata/testconfig/config.yaml deleted file mode 100644 index a3da5046077..00000000000 --- a/pkg/retester/testdata/testconfig/config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -retester: - enabled: true - max_retests_for_sha_and_base: 1 - max_retests_for_sha: 1 - orgs: - openshift: - enabled: true - max_retests_for_sha_and_base: 2 - max_retests_for_sha: 2 - repos: - ci-tools: - enabled: true - max_retests_for_sha_and_base: 3 - max_retests_for_sha: 3 - ci-docs: - enabled: true \ No newline at end of file diff --git a/pkg/retester/testdata/testconfig/default.yaml b/pkg/retester/testdata/testconfig/default.yaml deleted file mode 100644 index b66f11b8d17..00000000000 --- a/pkg/retester/testdata/testconfig/default.yaml +++ /dev/null @@ -1,3 +0,0 @@ -retester: - max_retests_for_sha_and_base: 3 - max_retests_for_sha: 9 \ No newline at end of file diff --git a/pkg/retester/testdata/testconfig/empty.yaml b/pkg/retester/testdata/testconfig/empty.yaml deleted file mode 100644 index 3625cbe1772..00000000000 --- a/pkg/retester/testdata/testconfig/empty.yaml +++ /dev/null @@ -1 +0,0 @@ -retester: {} \ No newline at end of file diff --git a/pkg/retester/testdata/testconfig/no-config.yaml b/pkg/retester/testdata/testconfig/no-config.yaml deleted file mode 100644 index 0ddc723f804..00000000000 --- a/pkg/retester/testdata/testconfig/no-config.yaml +++ /dev/null @@ -1 +0,0 @@ -something: a \ No newline at end of file diff --git a/pkg/retester/testdata/testconfig/openshift-config.yaml b/pkg/retester/testdata/testconfig/openshift-config.yaml deleted file mode 100644 index 8c443feb622..00000000000 --- a/pkg/retester/testdata/testconfig/openshift-config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -retester: - max_retests_for_sha_and_base: 3 - max_retests_for_sha: 9 - orgs: - openshift: - enabled: true - openshift-knative: - enabled: true diff --git a/pkg/retester/testdata/testconfig/wrong_format.yaml b/pkg/retester/testdata/testconfig/wrong_format.yaml deleted file mode 100644 index 286888a0ab5..00000000000 --- a/pkg/retester/testdata/testconfig/wrong_format.yaml +++ /dev/null @@ -1 +0,0 @@ -not yaml