feat: extract bins

internal/archive/archive_test.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 
 	"github.com/canonical/chisel/internal/archive"
+	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/archive/testarchive"
 	"github.com/canonical/chisel/internal/tarball"
 	"github.com/canonical/chisel/internal/testutil"
@@ -859,6 +860,7 @@ func (s *S) testOpenArchiveArch(c *C, test realArchiveTest, arch string) {
 	err = tarball.Extract(pkg, &tarball.ExtractOptions{
 		Package:   test.pkg,
 		TargetDir: extractDir,
+		OpenData:  deb.DataReader,
 		Extract: map[string][]tarball.ExtractInfo{
 			fmt.Sprintf("/usr/share/doc/%s/copyright", test.pkg): {
 				{Path: "/copyright"},

internal/slicer/slicer.go

@@ -16,6 +16,7 @@ import (
 	"github.com/klauspost/compress/zstd"
 
 	"github.com/canonical/chisel/internal/archive"
+	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/manifestutil"
 	"github.com/canonical/chisel/internal/scripts"
@@ -273,15 +274,22 @@ func Run(options *RunOptions) error {
 			continue
 		}
 		src := pkgSources[slice.Package]
-		// Store packages are distributed as plain tarballs, whose extraction
-		// is not yet implemented. Fail until the format support is in place.
-		if src.kind != sourceArchive {
-			return fmt.Errorf("cannot extract package %q from store: store packages are not yet supported", src.pkg.RealName)
+		var openData func(io.ReadSeeker) (io.ReadCloser, error)
+		if src.kind == sourceStore {
+			switch store.StoreKind(src.store.Options().Kind) {
+			case store.StoreKindBin:
+				openData = tarball.XZDataReader
+			default:
+				return fmt.Errorf("cannot extract from store of kind %q: unsupported artefact format", src.store.Options().Kind)
+			}
+		} else {
+			openData = deb.DataReader
 		}
 		err := tarball.Extract(reader, &tarball.ExtractOptions{
 			Package:   slice.Package,
 			Extract:   extract[slice.Package],
 			TargetDir: targetDir,
+			OpenData:  openData,
 			Create:    create,
 		})
 		reader.Close()

internal/slicer/slicer_test.go

@@ -1980,7 +1980,7 @@ var slicerTests = []slicerTest{{
 		"/dir/file": "file 0644 cc55e2ec {test-package_third}",
 	},
 }, {
-	summary: "Store package fails as it is not yet supported",
+	summary: "Store package extracts, manifest reports missing package",
 	slices:  []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}},
 	arch:    "amd64",
 	pkgs: []*testutil.TestPackage{{
@@ -1989,7 +1989,11 @@ var slicerTests = []slicerTest{{
 	}, {
 		Name:  "store-pkg",
 		Store: "bin",
-		Data:  testutil.PackageData["test-package"],
+		Data: testutil.MustMakeBin([]testutil.TarEntry{
+			testutil.Dir(0o755, "./"),
+			testutil.Dir(0o755, "./dir/"),
+			testutil.Reg(0o644, "./dir/store-file", "store content"),
+		}),
 	}},
 	release: map[string]string{
 		"chisel.yaml": testutil.DefaultChiselYamlWithStores,
@@ -2010,7 +2014,29 @@ var slicerTests = []slicerTest{{
 						/dir/store-file:
 		`,
 	},
-	error: `cannot extract package "store-pkg" from store: store packages are not yet supported`,
+	error: `internal error: invalid manifest: slice bin-store-pkg_myslice refers to missing package "bin-store-pkg"`,
+}, {
+	summary: "Store package with invalid (non-xz) data fails extraction",
+	slices:  []setup.SliceKey{{"bin-store-pkg", "myslice"}},
+	arch:    "amd64",
+	pkgs: []*testutil.TestPackage{{
+		Name:  "store-pkg",
+		Store: "bin",
+		Data:  []byte("this is not an xz archive"),
+	}},
+	release: map[string]string{
+		"chisel.yaml": testutil.DefaultChiselYamlWithStores,
+		"slices/mydir/store-pkg.yaml": `
+			package: store-pkg
+			store: bin
+			default-track: 3.1
+			slices:
+				myslice:
+					contents:
+						/dir/store-file:
+		`,
+	},
+	error: `cannot extract from package "bin-store-pkg": xz: invalid header magic bytes`,
 }}
 
 func (s *S) TestRun(c *C) {
@@ -2150,6 +2176,7 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) {
 				stores[name] = &testutil.TestStore{
 					Packages: pkgs,
 					Opts: store.Options{
+						Kind:    relStore.Kind,
 						Version: relStore.Version,
 					},
 				}

internal/store/store.go

@@ -37,9 +37,9 @@ type Options struct {
 	Version  string
 }
 
-type storeKind string
+type StoreKind string
 
-const storeKindBin storeKind = "bin"
+const StoreKindBin StoreKind = "bin"
 
 const defaultRisk = "stable"
 
@@ -95,8 +95,8 @@ func Open(options *Options) (Store, error) {
 		return nil, err
 	}
 
-	switch storeKind(options.Kind) {
-	case storeKindBin:
+	switch StoreKind(options.Kind) {
+	case StoreKindBin:
 		apiURL := binAPIBase
 		downloadHost := binDownloadHost
 		if os.Getenv(binStagingEnvVar) != "" {
@@ -177,7 +177,7 @@ func (s *binStore) resolveRevision(name, track, risk string) (*binRevision, erro
 	reqBody, err := json.Marshal(resolveRequest{
 		Packages: []resolvePackage{{
 			InstanceKey: name,
-			Namespace:   string(storeKindBin),
+			Namespace:   string(StoreKindBin),
 			Name:        name,
 			Channel:     track + "/" + risk,
 			Platform:    binPlatform{Architecture: s.options.Arch},

internal/tarball/extract.go

@@ -12,7 +12,6 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/strdist"
 )
@@ -21,6 +20,11 @@ type ExtractOptions struct {
 	Package   string
 	TargetDir string
 	Extract   map[string][]ExtractInfo
+	// OpenData opens the uncompressed tar data stream of the package. It
+	// abstracts over the package format (e.g. a deb archive opener or a plain
+	// tarball opener), allowing Extract to operate on any package whose data
+	// payload is a tar stream.
+	OpenData func(io.ReadSeeker) (io.ReadCloser, error)
 	// Create can optionally be set to control the creation of extracted entries.
 	// extractInfos is set to the matching entries in Extract, and is nil in cases where
 	// the created entry is implicit and unlisted (for example, parent directories).
@@ -35,6 +39,9 @@ type ExtractInfo struct {
 }
 
 func getValidOptions(options *ExtractOptions) (*ExtractOptions, error) {
+	if options.OpenData == nil {
+		return nil, fmt.Errorf("internal error: ExtractOptions.OpenData is unset")
+	}
 	for extractPath, extractInfos := range options.Extract {
 		isGlob := strings.ContainsAny(extractPath, "*?")
 		if isGlob {
@@ -83,7 +90,7 @@ func Extract(pkgReader io.ReadSeeker, options *ExtractOptions) (err error) {
 }
 
 func extractData(pkgReader io.ReadSeeker, options *ExtractOptions) error {
-	dataReader, err := deb.DataReader(pkgReader)
+	dataReader, err := options.OpenData(pkgReader)
 	if err != nil {
 		return err
 	}
@@ -300,7 +307,7 @@ type extractHardLinkOptions struct {
 // extractHardLinks iterates through the tarball a second time to extract the
 // hard links that were not extracted in the first pass.
 func extractHardLinks(pkgReader io.ReadSeeker, opts *extractHardLinkOptions) error {
-	dataReader, err := deb.DataReader(pkgReader)
+	dataReader, err := opts.OpenData(pkgReader)
 	if err != nil {
 		return err
 	}

internal/tarball/extract_test.go

@@ -10,6 +10,7 @@ import (
 
 	. "gopkg.in/check.v1"
 
+	"github.com/canonical/chisel/internal/deb"
 	"github.com/canonical/chisel/internal/fsutil"
 	"github.com/canonical/chisel/internal/tarball"
 	"github.com/canonical/chisel/internal/testutil"
@@ -496,6 +497,8 @@ func (s *S) TestExtract(c *C) {
 		options := test.options
 		options.Package = "test-package"
 		options.TargetDir = dir
+		// The test fixtures are .deb archives, so use the deb data opener.
+		options.OpenData = deb.DataReader
 		createdPaths := make(map[string]bool)
 		options.Create = func(_ []tarball.ExtractInfo, o *fsutil.CreateOptions) error {
 			relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, dir))
@@ -605,6 +608,8 @@ func (s *S) TestExtractCreateCallback(c *C) {
 		options := test.options
 		options.Package = "test-package"
 		options.TargetDir = dir
+		// The test fixtures are .deb archives, so use the deb data opener.
+		options.OpenData = deb.DataReader
 		createExtractInfos := map[string][]tarball.ExtractInfo{}
 		options.Create = func(extractInfos []tarball.ExtractInfo, o *fsutil.CreateOptions) error {
 			if extractInfos == nil {
@@ -628,3 +633,15 @@ func (s *S) TestExtractCreateCallback(c *C) {
 		c.Assert(createExtractInfos, DeepEquals, test.calls)
 	}
 }
+
+func (s *S) TestExtractMissingOpenData(c *C) {
+	options := tarball.ExtractOptions{
+		Package:   "test-package",
+		TargetDir: c.MkDir(),
+		Extract: map[string][]tarball.ExtractInfo{
+			"/dir/file": {{Path: "/dir/file"}},
+		},
+	}
+	err := tarball.Extract(bytes.NewReader(testutil.PackageData["test-package"]), &options)
+	c.Assert(err, ErrorMatches, `cannot extract from package "test-package": internal error: ExtractOptions.OpenData is unset`)
+}

internal/tarball/xz.go

@@ -0,0 +1,15 @@
+package tarball
+
+import (
+	"io"
+
+	"github.com/ulikunitz/xz"
+)
+
+func XZDataReader(pkgReader io.ReadSeeker) (io.ReadCloser, error) {
+	xzReader, err := xz.NewReader(pkgReader)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(xzReader), nil
+}

internal/testutil/pkgdata.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/blakesmith/ar"
 	"github.com/klauspost/compress/zstd"
+	"github.com/ulikunitz/xz"
 )
 
 var PackageData = map[string][]byte{}
@@ -160,6 +161,42 @@ func MustMakeDeb(entries []TarEntry) []byte {
 	return data
 }
 
+// compressBytesXz compresses the input using XZ, the compression format used by
+// store (bin) packages.
+func compressBytesXz(input []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	writer, err := xz.NewWriter(&buf)
+	if err != nil {
+		return nil, err
+	}
+	if _, err = writer.Write(input); err != nil {
+		return nil, err
+	}
+	if err = writer.Close(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// MakeBin builds a store (bin) package from the given tar entries: an
+// XZ-compressed plain tarball. Unlike MakeDeb, there is no ar container.
+func MakeBin(entries []TarEntry) ([]byte, error) {
+	tarData, err := makeTar(entries)
+	if err != nil {
+		return nil, err
+	}
+	return compressBytesXz(tarData)
+}
+
+// MustMakeBin is the panicking variant of MakeBin.
+func MustMakeBin(entries []TarEntry) []byte {
+	data, err := MakeBin(entries)
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
 // Reg is a shortcut for creating a regular file TarEntry structure (with
 // tar.Typeflag set tar.TypeReg). Reg stands for "REGular file".
 func Reg(mode int64, path, content string) TarEntry {