feat(templates): cache templates based on etag

api/admin/settings.go

@@ -11,6 +11,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/go-vela/server/api/types/settings"
+	"github.com/go-vela/server/cache"
 	"github.com/go-vela/server/compiler/native"
 	"github.com/go-vela/server/database"
 	"github.com/go-vela/server/internal/image"
@@ -337,7 +338,7 @@ func RestoreSettings(c *gin.Context) {
 		return
 	}
 
-	compiler, err := native.FromCLICommand(ctx, cliCmd)
+	compiler, err := native.FromCLICommand(ctx, cliCmd, cache.FromContext(c))
 	if err != nil {
 		retErr := fmt.Errorf("unable to restore platform settings: %w", err)
 

api/pipeline/expand_test.go

@@ -149,7 +149,7 @@ func newTestCompiler(t *testing.T) compiler.Engine {
 		},
 	}
 
-	engine, err := native.FromCLICommand(context.Background(), cmd)
+	engine, err := native.FromCLICommand(context.Background(), cmd, nil)
 	if err != nil {
 		t.Fatalf("unable to create compiler: %v", err)
 	}

api/pipeline/template.go

@@ -11,6 +11,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/go-vela/server/api/types"
+	"github.com/go-vela/server/cache"
 	"github.com/go-vela/server/compiler"
 	"github.com/go-vela/server/compiler/registry/github"
 	"github.com/go-vela/server/compiler/types/yaml"
@@ -115,7 +116,7 @@ func GetTemplates(c *gin.Context) {
 		templates[name] = template.ToAPI()
 
 		// create a compiler registry client for parsing (no address or token needed for Parse)
-		registry, err := github.New(ctx, "", "")
+		registry, err := github.New(ctx, "", "", cache.FromContext(c))
 		if err != nil {
 			util.HandleError(c, http.StatusBadRequest, fmt.Errorf("%s: unable to create compiler github client: %w", baseErr, err))
 

cache/cache.go

@@ -21,6 +21,7 @@ func FromCLICommand(ctx context.Context, c *cli.Command) (Service, error) {
 		Address:         c.String("cache.addr"),
 		Cluster:         c.Bool("cache.cluster"),
 		InstallTokenKey: c.String("cache.install-token-key"),
+		TemplateTTL:     c.Duration("cache.template-ttl"),
 	}
 
 	// setup the queue

cache/flags.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/urfave/cli/v3"
 )
@@ -68,4 +69,14 @@ var Flags = []cli.Flag{
 		),
 		Required: true,
 	},
+	&cli.DurationFlag{
+		Name:  "cache.template-ttl",
+		Usage: "duration for how long a cached template should be considered valid before it is refreshed",
+		Sources: cli.NewValueSourceChain(
+			cli.EnvVar("VELA_CACHE_TEMPLATE_TTL"),
+			cli.EnvVar("CACHE_TEMPLATE_TTL"),
+			cli.File("/vela/cache/template_ttl"),
+		),
+		Value: 48 * time.Hour, // 48 hours
+	},
 }

cache/models/template.go

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package models
+
+import (
+	"net/http"
+	"time"
+)
+
+type TemplateEntry struct {
+	ETag      string
+	Status    int
+	Header    http.Header
+	Body      []byte
+	UpdatedAt time.Time
+}

cache/redis/extend_template.go

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package redis
+
+import (
+	"context"
+)
+
+// ExtendTemplateExpiry extends the TTL for a template entry in Redis.
+func (c *Client) ExtendTemplateExpiry(ctx context.Context, key string) error {
+	// extend the TTL for the template entry
+	err := c.Redis.Expire(ctx, key, c.config.TemplateTTL).Err()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

cache/redis/extend_template_test.go

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package redis
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/go-vela/server/cache/models"
+)
+
+func TestRedis_ExtendTemplateExpiry(t *testing.T) {
+	// setup redis mock
+	_redis, err := NewTest("c94bc43c11613ceb6c9f6ac73451e41de90806b2ca6953010b547b20fde9ad90")
+	if err != nil {
+		t.Errorf("unable to create cache service: %v", err)
+	}
+
+	// store an entry to extend
+	entry := &models.TemplateEntry{
+		ETag:      `"etag-123"`,
+		Status:    http.StatusOK,
+		Header:    http.Header{"Content-Type": {"application/json"}},
+		Body:      []byte("template body"),
+		UpdatedAt: time.Now().UTC(),
+	}
+
+	err = _redis.StoreTemplateContents(t.Context(), "github:contents:abc123", entry)
+	if err != nil {
+		t.Errorf("unable to store template contents: %v", err)
+	}
+
+	// setup tests
+	tests := []struct {
+		name    string
+		key     string
+		wantErr bool
+	}{
+		{
+			name:    "existing entry",
+			key:     "github:contents:abc123",
+			wantErr: false,
+		},
+		{
+			name:    "non-existent entry",
+			key:     "github:contents:missing",
+			wantErr: false,
+		},
+	}
+
+	// run tests
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := _redis.ExtendTemplateExpiry(t.Context(), test.key)
+
+			if test.wantErr {
+				if err == nil {
+					t.Errorf("ExtendTemplateExpiry should have returned err")
+				}
+
+				return
+			}
+
+			if err != nil {
+				t.Errorf("ExtendTemplateExpiry returned err: %v", err)
+			}
+		})
+	}
+}

cache/redis/get_template.go

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package redis
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+
+	"github.com/redis/go-redis/v9"
+
+	"github.com/go-vela/server/cache/models"
+)
+
+func (c *Client) GetTemplateContents(ctx context.Context, key string) (*models.TemplateEntry, error) {
+	meta, err := c.Redis.Get(ctx, key).Bytes()
+	if err != nil {
+		if errors.Is(err, redis.Nil) {
+			return nil, nil
+		}
+
+		return nil, err
+	}
+
+	templateEntry := new(models.TemplateEntry)
+
+	err = json.Unmarshal(meta, templateEntry)
+	if err != nil {
+		return nil, err
+	}
+
+	return templateEntry, nil
+}

cache/redis/get_template_test.go

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package redis
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/go-vela/server/cache/models"
+)
+
+func TestRedis_GetTemplateContents(t *testing.T) {
+	// setup types
+	entry := &models.TemplateEntry{
+		ETag:      `"etag-123"`,
+		Status:    http.StatusOK,
+		Header:    http.Header{"Content-Type": {"application/json"}},
+		Body:      []byte("template body"),
+		UpdatedAt: time.Now().UTC(),
+	}
+
+	// setup redis mock
+	_redis, err := NewTest("c94bc43c11613ceb6c9f6ac73451e41de90806b2ca6953010b547b20fde9ad90")
+	if err != nil {
+		t.Errorf("unable to create cache service: %v", err)
+	}
+
+	err = _redis.StoreTemplateContents(t.Context(), "github:contents:abc123", entry)
+	if err != nil {
+		t.Errorf("unable to store template contents: %v", err)
+	}
+
+	// setup tests
+	tests := []struct {
+		name    string
+		key     string
+		want    *models.TemplateEntry
+		wantErr bool
+	}{
+		{
+			name:    "existing entry",
+			key:     "github:contents:abc123",
+			want:    entry,
+			wantErr: false,
+		},
+		{
+			name:    "non-existent entry",
+			key:     "github:contents:missing",
+			want:    nil,
+			wantErr: false,
+		},
+	}
+
+	// run tests
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := _redis.GetTemplateContents(t.Context(), test.key)
+
+			if test.wantErr {
+				if err == nil {
+					t.Errorf("GetTemplateContents should have returned err")
+				}
+
+				return
+			}
+
+			if err != nil {
+				t.Errorf("GetTemplateContents returned err: %v", err)
+			}
+
+			if diff := cmp.Diff(test.want, got); diff != "" {
+				t.Errorf("GetTemplateContents() mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

cache/redis/opts.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"time"
 )
 
 // ClientOpt represents a configuration option to initialize the queue client for Redis.
@@ -61,3 +62,15 @@ func WithInstallTokenKey(key string) ClientOpt {
 		return nil
 	}
 }
+
+// WithTemplateTTL sets the template TTL in the cache client for Redis.
+func WithTemplateTTL(ttl time.Duration) ClientOpt {
+	return func(c *Client) error {
+		c.Logger.Trace("configuring template TTL in redis cache client")
+
+		// set the template TTL in the redis cache client
+		c.config.TemplateTTL = ttl
+
+		return nil
+	}
+}

cache/redis/redis.go

@@ -22,6 +22,8 @@ type config struct {
 	Cluster bool
 
 	InstallTokenKey string
+
+	TemplateTTL time.Duration
 }
 
 type Client struct {

cache/redis/store_template.go

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package redis
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/go-vela/server/cache/models"
+)
+
+// StoreTemplateContents stores it in Redis with a TTL.
+func (c *Client) StoreTemplateContents(ctx context.Context, key string, t *models.TemplateEntry) error {
+	metaBytes, err := json.Marshal(t)
+	if err != nil {
+		return err
+	}
+
+	// store a small marker value (or metadata JSON if needed)
+	err = c.Redis.Set(ctx, key, metaBytes, c.config.TemplateTTL).Err()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}