From 5d3ceaccfb55be5b029c30b2c44593c4618d388e Mon Sep 17 00:00:00 2001 From: Samuel Suter Date: Wed, 11 Nov 2020 01:52:39 -0700 Subject: [PATCH] (feat) Plugin version constraints Implements #1335 --- agent/plugin/plugin.go | 77 +++++++++++++++++++++++++++++++++++++ agent/plugin/plugin_test.go | 68 ++++++++++++++++++++++++++++++++ bootstrap/bootstrap.go | 24 ++++++++++++ go.mod | 1 + go.sum | 2 + 5 files changed, 172 insertions(+) diff --git a/agent/plugin/plugin.go b/agent/plugin/plugin.go index f2556a3565..5b29427295 100644 --- a/agent/plugin/plugin.go +++ b/agent/plugin/plugin.go @@ -9,8 +9,15 @@ import ( "strings" "github.com/buildkite/agent/v3/env" + + "github.com/Masterminds/semver" ) +// https://github.com/Masterminds/semver/blob/0ce76fe59f74ed481275b478858c859ba90eb668/version.go#L41 +const semVerRegex string = `v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?` + + `(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + + `(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + type Plugin struct { // Where the plugin can be found (can either be a file system path, or // a git repository) @@ -19,6 +26,9 @@ type Plugin struct { // The version of the plugin that should be running Version string + // Constraint describes the plugin semver version constraint, it is mutually exclusive with `Version` + Constraint *semver.Constraints + // The clone method Scheme string @@ -48,6 +58,17 @@ func CreatePlugin(location string, config map[string]interface{}) (*Plugin, erro plugin.Scheme = u.Scheme plugin.Location = u.Host + u.Path plugin.Version = u.Fragment + + // If a constraint has been specified + if constraint := u.Query().Get("constraint"); constraint != "" { + plugin.Constraint, err = semver.NewConstraint(constraint) + if err != nil { + return nil, fmt.Errorf("Cannot parse plugin version constraint %q: %s", constraint, err) + } + if plugin.Version != "" { + return nil, fmt.Errorf("Cannot specify both version and constraint") + } + } plugin.Vendored = vendoredRegex.MatchString(plugin.Location) if plugin.Version != "" && strings.Count(plugin.Version, "#") > 0 { @@ -202,6 +223,7 @@ var ( toDashRegex = regexp.MustCompile(`-|\s+`) removeWhitespaceRegex = regexp.MustCompile(`\s+`) removeDoubleUnderscore = regexp.MustCompile(`_+`) + versionRegex = regexp.MustCompile("refs/tags/(?P" + semVerRegex + ")") ) // formatEnvKey converts strings into an ENV key friendly format @@ -304,3 +326,58 @@ func (p *Plugin) constructRepositoryHost() (string, error) { return s, nil } + +// ResolveConstraint processes `ls-remote --tags` output and sets the plugin Version to the highest matching semver based on the constraint +// The ls-remote operation is intentionally not in this package to avoid importing the shell package into this context +func (p *Plugin) ResolveConstraint(remoteTags string) error { + if p.Constraint == nil { + return fmt.Errorf("Plugin has no constraint specified") + } + + versions, err := findSemverTags(remoteTags) + if err != nil { + return err + } + + // Reverse sort them (newest first) + sort.Sort(sort.Reverse(semver.Collection(versions))) + + for _, tag := range versions { + if p.Constraint.Check(tag) { + // Set the plugin version to this git tag + p.Version = tag.Original() + return nil + } + } + return fmt.Errorf("No version found matching constraint") +} + +// findSemverTags inspects a `ls-remote --tags` output and extracts all refs that are a valid semver +func findSemverTags(remoteTags string) ([]*semver.Version, error) { + // Find index position of the named `semver` submatch in our regular expression + semverSubexpressionIndex := 0 + for i, name := range versionRegex.SubexpNames() { + if name == "semver" { + semverSubexpressionIndex = i + break + } + } + + matches := versionRegex.FindAllStringSubmatch(remoteTags, -1) + vs := make([]*semver.Version, 0) // Since some matches might be invalid we cannot predict the length + for _, match := range matches { + // Only extract our specific `semver` named submatch + if len(match) < semverSubexpressionIndex+1 { + // Not enough matches found in this capture group - ignore it and move onto the next + continue + } + v, err := semver.NewVersion(match[semverSubexpressionIndex]) + if err == nil { + // We ignore errors here - there might be tags that are not a valid semver at this point + // though unlikely, because of the regex match we do + vs = append(vs, v) + } + } + + return vs, nil +} diff --git a/agent/plugin/plugin_test.go b/agent/plugin/plugin_test.go index cf4dd036a5..76488bf828 100644 --- a/agent/plugin/plugin_test.go +++ b/agent/plugin/plugin_test.go @@ -5,10 +5,70 @@ import ( "fmt" "testing" + "github.com/Masterminds/semver" "github.com/buildkite/agent/v3/env" "github.com/stretchr/testify/assert" ) +func TestFindSemverTags(t *testing.T) { + lsRemoteOut := `48187550589c805195e8c3f15d4839783e5e869e refs/tags/v1.0 +1e90ce3fbd7d8dca0c5009e20b9598cb42872306 refs/tags/v1.1 +c41f62fba4e66c930c258cf5ee163c311083d8a3 refs/tags/v1.2.0 +1ee4e5c7b0de9670d0b10265e116ab4b809fda36 refs/tags/v1.2.1 +4e2a1593653460071e1f276dd3a8f030d1b469c3 refs/tags/v1.3.0 +2cef046fd56c76ad01580018866877a8a0ad0827 refs/tags/v1.3.1` + + expected := []*semver.Version{ + semver.MustParse("v1.0"), + semver.MustParse("v1.1"), + semver.MustParse("v1.2.0"), + semver.MustParse("v1.2.1"), + semver.MustParse("v1.3.0"), + semver.MustParse("v1.3.1"), + } + versions, err := findSemverTags(lsRemoteOut) + if err != nil { + t.Error(err) + } + + assert.Equal(t, expected, versions) +} + +func TestResolveConstraint(t *testing.T) { + t.Parallel() + lsRemoteOut := `48187550589c805195e8c3f15d4839783e5e869e refs/tags/v1.0 + 1e90ce3fbd7d8dca0c5009e20b9598cb42872306 refs/tags/v1.1 + c41f62fba4e66c930c258cf5ee163c311083d8a3 refs/tags/v1.2.0 + 1ee4e5c7b0de9670d0b10265e116ab4b809fda36 refs/tags/v1.2.1 + 4e2a1593653460071e1f276dd3a8f030d1b469c3 refs/tags/v1.3.0 + 2cef046fd56c76ad01580018866877a8a0ad0827 refs/tags/v1.3.1` + + t.Run("version-match", func(tt *testing.T) { + tt.Parallel() + jsonText := `[{"https://github.com/buildkite-plugins/docker-compose?constraint=~1.2.x":{"container":"app"}}]` + + plugins, err := CreateFromJSON(jsonText) + if err != nil { + t.Error(err) + } + err = plugins[0].ResolveConstraint(lsRemoteOut) + //assert.NoError(tt, err) + assert.Equal(t, "v1.2.1", plugins[0].Version, "correct version is matched") + }) + + t.Run("missing-constraint", func(tt *testing.T) { + tt.Parallel() + jsonText := `[{"https://github.com/buildkite-plugins/docker-compose":{"container":"app"}}]` + + plugins, err := CreateFromJSON(jsonText) + if err != nil { + t.Error(err) + } + err = plugins[0].ResolveConstraint(lsRemoteOut) + assert.Error(tt, err) + }) + +} func TestCreateFromJSON(t *testing.T) { t.Parallel() @@ -114,6 +174,14 @@ func TestCreateFromJSONFailsOnParseErrors(t *testing.T) { `["github.com/buildkite-plugins/ping#master#lololo"]`, "Too many #'s in \"github.com/buildkite-plugins/ping#master#lololo\"", }, + { + `[{"https://github.com/buildkite-plugins/docker-compose?constraint=^2.x#a34fa34":{"container":"app"}}]`, + "Cannot specify both version and constraint", + }, + { + `[{"https://github.com/buildkite-plugins/docker-compose?constraint=something bad":{"container":"app"}}]`, + "improper constraint", + }, } { tc := tc t.Run("", func(tt *testing.T) { diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index cd919d7242..16ecb608ab 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -669,6 +669,20 @@ func (b *Bootstrap) checkoutPlugin(p *plugin.Plugin) (*pluginCheckout, error) { return nil, fmt.Errorf("Can't checkout plugin without a `plugins-path`") } + if p.Constraint != nil { + // A version constraint has been specified - attempt to resolve it to set a proper Version + // This must occur before we access .Identifier() or .Label() of this plugin + repo, err := p.Repository() + if err != nil { + return nil, err + } + remoteTags, err := b.getRemoteTags(repo) + err = p.ResolveConstraint(remoteTags) + if err != nil { + return nil, err + } + } + // Get the identifer for the plugin id, err := p.Identifier() if err != nil { @@ -1251,6 +1265,16 @@ func (b *Bootstrap) defaultCheckoutPhase() error { return nil } +// getRemote tags returns an version sorted list of tags on a remote repository +func (b *Bootstrap) getRemoteTags(repo string) (string, error) { + output, err := b.shell.RunAndCapture("git", "ls-remote", "--sort", "version:refname", "--tags", repo) + if err != nil { + b.shell.Errorf("Error running git ls-remote --sort version:refname --tags %q: %v", repo, err) + return "", err + } + return output, nil +} + func (b *Bootstrap) resolveCommit() { commitRef, _ := b.shell.Env.Get("BUILDKITE_COMMIT") if commitRef == "" { diff --git a/go.mod b/go.mod index ae453d176f..8eb7143032 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.14 require ( cloud.google.com/go v0.0.0-20170217213217-65216237311a github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 + github.com/Masterminds/semver v1.5.0 github.com/aws/aws-sdk-go v1.32.10 github.com/buildkite/bintest/v3 v3.1.0 github.com/buildkite/interpolate v0.0.0-20200526001904-07f35b4ae251 diff --git a/go.sum b/go.sum index 3f714b13d1..29638660a7 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ cloud.google.com/go v0.0.0-20170217213217-65216237311a/go.mod h1:aQUYkXzVsufM+Dw github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/aws/aws-sdk-go v1.32.10 h1:cEJTxGcBGlsM2tN36MZQKhlK93O9HrnaRs+lq2f0zN8= github.com/aws/aws-sdk-go v1.32.10/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/buildkite/bintest v0.0.0-20190227031731-820c89d3b3a0 h1:UYcAVM2IdwHl3wTyyB2HfZc2MEVQrUqw/OkbwHlsy30=