(feat) Plugin version constraints

agent/plugin/plugin.go

@@ -9,8 +9,15 @@ import (
 	"strings"
 
 	"github.com/buildkite/agent/v3/env"
+
+	"github.com/Masterminds/semver"
 )
 
+// https://github.com/Masterminds/semver/blob/0ce76fe59f74ed481275b478858c859ba90eb668/version.go#L41
+const semVerRegex string = `v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?` +
+	`(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` +
+	`(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?`
+
 type Plugin struct {
 	// Where the plugin can be found (can either be a file system path, or
 	// a git repository)
@@ -19,6 +26,9 @@ type Plugin struct {
 	// The version of the plugin that should be running
 	Version string
 
+	// Constraint describes the plugin semver version constraint, it is mutually exclusive with `Version`
+	Constraint *semver.Constraints
+
 	// The clone method
 	Scheme string
 
@@ -48,6 +58,17 @@ func CreatePlugin(location string, config map[string]interface{}) (*Plugin, erro
 	plugin.Scheme = u.Scheme
 	plugin.Location = u.Host + u.Path
 	plugin.Version = u.Fragment
+
+	// If a constraint has been specified
+	if constraint := u.Query().Get("constraint"); constraint != "" {
+		plugin.Constraint, err = semver.NewConstraint(constraint)
+		if err != nil {
+			return nil, fmt.Errorf("Cannot parse plugin version constraint %q: %s", constraint, err)
+		}
+		if plugin.Version != "" {
+			return nil, fmt.Errorf("Cannot specify both version and constraint")
+		}
+	}
 	plugin.Vendored = vendoredRegex.MatchString(plugin.Location)
 
 	if plugin.Version != "" && strings.Count(plugin.Version, "#") > 0 {
@@ -202,6 +223,7 @@ var (
 	toDashRegex            = regexp.MustCompile(`-|\s+`)
 	removeWhitespaceRegex  = regexp.MustCompile(`\s+`)
 	removeDoubleUnderscore = regexp.MustCompile(`_+`)
+	versionRegex           = regexp.MustCompile("refs/tags/(?P<semver>" + semVerRegex + ")")
 )
 
 // formatEnvKey converts strings into an ENV key friendly format
@@ -304,3 +326,58 @@ func (p *Plugin) constructRepositoryHost() (string, error) {
 
 	return s, nil
 }
+
+// ResolveConstraint processes `ls-remote --tags` output and sets the plugin Version to the highest matching semver based on the constraint
+// The ls-remote operation is intentionally not in this package to avoid importing the shell package into this context
+func (p *Plugin) ResolveConstraint(remoteTags string) error {
+	if p.Constraint == nil {
+		return fmt.Errorf("Plugin has no constraint specified")
+	}
+
+	versions, err := findSemverTags(remoteTags)
+	if err != nil {
+		return err
+	}
+
+	// Reverse sort them (newest first)
+	sort.Sort(sort.Reverse(semver.Collection(versions)))
+
+	for _, tag := range versions {
+		if p.Constraint.Check(tag) {
+			// Set the plugin version to this git tag
+			p.Version = tag.Original()
+			return nil
+		}
+	}
+	return fmt.Errorf("No version found matching constraint")
+}
+
+// findSemverTags inspects a `ls-remote --tags` output and extracts all refs that are a valid semver
+func findSemverTags(remoteTags string) ([]*semver.Version, error) {
+	// Find index position of the named `semver` submatch in our regular expression
+	semverSubexpressionIndex := 0
+	for i, name := range versionRegex.SubexpNames() {
+		if name == "semver" {
+			semverSubexpressionIndex = i
+			break
+		}
+	}
+
+	matches := versionRegex.FindAllStringSubmatch(remoteTags, -1)
+	vs := make([]*semver.Version, 0) // Since some matches might be invalid we cannot predict the length
+	for _, match := range matches {
+		// Only extract our specific `semver` named submatch
+		if len(match) < semverSubexpressionIndex+1 {
+			// Not enough matches found in this capture group - ignore it and move onto the next
+			continue
+		}
+		v, err := semver.NewVersion(match[semverSubexpressionIndex])
+		if err == nil {
+			// We ignore errors here - there might be tags that are not a valid semver at this point
+			// though unlikely, because of the regex match we do
+			vs = append(vs, v)
+		}
+	}
+
+	return vs, nil
+}

agent/plugin/plugin_test.go

@@ -5,10 +5,70 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/Masterminds/semver"
 	"github.com/buildkite/agent/v3/env"
 	"github.com/stretchr/testify/assert"
 )
 
+func TestFindSemverTags(t *testing.T) {
+	lsRemoteOut := `48187550589c805195e8c3f15d4839783e5e869e	refs/tags/v1.0
+1e90ce3fbd7d8dca0c5009e20b9598cb42872306	refs/tags/v1.1
+c41f62fba4e66c930c258cf5ee163c311083d8a3	refs/tags/v1.2.0
+1ee4e5c7b0de9670d0b10265e116ab4b809fda36	refs/tags/v1.2.1
+4e2a1593653460071e1f276dd3a8f030d1b469c3	refs/tags/v1.3.0
+2cef046fd56c76ad01580018866877a8a0ad0827	refs/tags/v1.3.1`
+
+	expected := []*semver.Version{
+		semver.MustParse("v1.0"),
+		semver.MustParse("v1.1"),
+		semver.MustParse("v1.2.0"),
+		semver.MustParse("v1.2.1"),
+		semver.MustParse("v1.3.0"),
+		semver.MustParse("v1.3.1"),
+	}
+	versions, err := findSemverTags(lsRemoteOut)
+	if err != nil {
+		t.Error(err)
+	}
+
+	assert.Equal(t, expected, versions)
+}
+
+func TestResolveConstraint(t *testing.T) {
+	t.Parallel()
+	lsRemoteOut := `48187550589c805195e8c3f15d4839783e5e869e	refs/tags/v1.0
+	1e90ce3fbd7d8dca0c5009e20b9598cb42872306	refs/tags/v1.1
+	c41f62fba4e66c930c258cf5ee163c311083d8a3	refs/tags/v1.2.0
+	1ee4e5c7b0de9670d0b10265e116ab4b809fda36	refs/tags/v1.2.1
+	4e2a1593653460071e1f276dd3a8f030d1b469c3	refs/tags/v1.3.0
+	2cef046fd56c76ad01580018866877a8a0ad0827	refs/tags/v1.3.1`
+
+	t.Run("version-match", func(tt *testing.T) {
+		tt.Parallel()
+		jsonText := `[{"https://github.com/buildkite-plugins/docker-compose?constraint=~1.2.x":{"container":"app"}}]`
+
+		plugins, err := CreateFromJSON(jsonText)
+		if err != nil {
+			t.Error(err)
+		}
+		err = plugins[0].ResolveConstraint(lsRemoteOut)
+		//assert.NoError(tt, err)
+		assert.Equal(t, "v1.2.1", plugins[0].Version, "correct version is matched")
+	})
+
+	t.Run("missing-constraint", func(tt *testing.T) {
+		tt.Parallel()
+		jsonText := `[{"https://github.com/buildkite-plugins/docker-compose":{"container":"app"}}]`
+
+		plugins, err := CreateFromJSON(jsonText)
+		if err != nil {
+			t.Error(err)
+		}
+		err = plugins[0].ResolveConstraint(lsRemoteOut)
+		assert.Error(tt, err)
+	})
+
+}
 func TestCreateFromJSON(t *testing.T) {
 	t.Parallel()
 
@@ -114,6 +174,14 @@ func TestCreateFromJSONFailsOnParseErrors(t *testing.T) {
 			`["github.com/buildkite-plugins/ping#master#lololo"]`,
 			"Too many #'s in \"github.com/buildkite-plugins/ping#master#lololo\"",
 		},
+		{
+			`[{"https://github.com/buildkite-plugins/docker-compose?constraint=^2.x#a34fa34":{"container":"app"}}]`,
+			"Cannot specify both version and constraint",
+		},
+		{
+			`[{"https://github.com/buildkite-plugins/docker-compose?constraint=something bad":{"container":"app"}}]`,
+			"improper constraint",
+		},
 	} {
 		tc := tc
 		t.Run("", func(tt *testing.T) {

bootstrap/bootstrap.go

@@ -669,6 +669,20 @@ func (b *Bootstrap) checkoutPlugin(p *plugin.Plugin) (*pluginCheckout, error) {
 		return nil, fmt.Errorf("Can't checkout plugin without a `plugins-path`")
 	}
 
+	if p.Constraint != nil {
+		// A version constraint has been specified - attempt to resolve it to set a proper Version
+		// This must occur before we access .Identifier() or .Label() of this plugin
+		repo, err := p.Repository()
+		if err != nil {
+			return nil, err
+		}
+		remoteTags, err := b.getRemoteTags(repo)
+		err = p.ResolveConstraint(remoteTags)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Get the identifer for the plugin
 	id, err := p.Identifier()
 	if err != nil {
@@ -1251,6 +1265,16 @@ func (b *Bootstrap) defaultCheckoutPhase() error {
 	return nil
 }
 
+// getRemote tags returns an version sorted list of tags on a remote repository
+func (b *Bootstrap) getRemoteTags(repo string) (string, error) {
+	output, err := b.shell.RunAndCapture("git", "ls-remote", "--sort", "version:refname", "--tags", repo)
+	if err != nil {
+		b.shell.Errorf("Error running git ls-remote --sort version:refname --tags %q: %v", repo, err)
+		return "", err
+	}
+	return output, nil
+}
+
 func (b *Bootstrap) resolveCommit() {
 	commitRef, _ := b.shell.Env.Get("BUILDKITE_COMMIT")
 	if commitRef == "" {

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	cloud.google.com/go v0.0.0-20170217213217-65216237311a
 	github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
+	github.com/Masterminds/semver v1.5.0
 	github.com/aws/aws-sdk-go v1.32.10
 	github.com/buildkite/bintest/v3 v3.1.0
 	github.com/buildkite/interpolate v0.0.0-20200526001904-07f35b4ae251

go.sum

@@ -3,6 +3,8 @@ cloud.google.com/go v0.0.0-20170217213217-65216237311a/go.mod h1:aQUYkXzVsufM+Dw
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/aws/aws-sdk-go v1.32.10 h1:cEJTxGcBGlsM2tN36MZQKhlK93O9HrnaRs+lq2f0zN8=
 github.com/aws/aws-sdk-go v1.32.10/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/buildkite/bintest v0.0.0-20190227031731-820c89d3b3a0 h1:UYcAVM2IdwHl3wTyyB2HfZc2MEVQrUqw/OkbwHlsy30=