feat: search, load and validates manifests

cmd/chisel/cmd_cut.go

@@ -2,13 +2,16 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"slices"
 	"time"
 
 	"github.com/jessevdk/go-flags"
 
 	"github.com/canonical/chisel/internal/archive"
 	"github.com/canonical/chisel/internal/cache"
+	"github.com/canonical/chisel/internal/manifestutil"
 	"github.com/canonical/chisel/internal/setup"
 	"github.com/canonical/chisel/internal/slicer"
 )
@@ -73,6 +76,29 @@ func (cmd *cmdCut) Execute(args []string) error {
 		}
 	}
 
+	rootfsPath, err := preExistingRootfs(cmd.RootDir)
+	if err != nil {
+		return err
+	}
+
+	if len(rootfsPath) > 0 {
+		manifest, err := manifestutil.RootFSManifest(release, rootfsPath)
+		if err != nil {
+			// TODO: When enabling the feature, error out.
+			logf("Warning: %v", err)
+		} else {
+			// Merge the slice keys used to build the existing rootfs with the ones
+			// explicitly requested slices.
+			// Previous slice keys contains both explicitly requested slices and slices
+			// resolved following essentials.
+			for _, s := range manifestutil.SliceKeys(manifest) {
+				if !slices.Contains(sliceKeys, s) {
+					sliceKeys = append(sliceKeys, s)
+				}
+			}
+		}
+	}
+
 	selection, err := setup.Select(release, sliceKeys, cmd.Arch)
 	if err != nil {
 		return err
@@ -128,3 +154,28 @@ func (cmd *cmdCut) Execute(args []string) error {
 	})
 	return err
 }
+
+// preExistingRootfs determines if the target directory was produced by chisel.
+func preExistingRootfs(rootdir string) (string, error) {
+	// Get targetDir path
+	// Note: This is already done in `slicer.Run`. Extract it and avoid doing it twice?
+	targetDir := filepath.Clean(rootdir)
+	if !filepath.IsAbs(targetDir) {
+		dir, err := os.Getwd()
+		if err != nil {
+			return "", fmt.Errorf("cannot obtain current directory: %w", err)
+		}
+		targetDir = filepath.Join(dir, targetDir)
+	}
+
+	entries, err := os.ReadDir(targetDir)
+	if err != nil {
+		return "", fmt.Errorf("cannot read root directory %q: %v", targetDir, err)
+	}
+
+	if len(entries) == 0 {
+		return "", nil
+	}
+
+	return targetDir, nil
+}

internal/manifestutil/manifestutil.go

@@ -1,19 +1,25 @@
 package manifestutil
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/fs"
+	"os"
+	"path"
 	"path/filepath"
 	"slices"
 	"sort"
 	"strings"
+	"syscall"
 
 	"github.com/canonical/chisel/internal/apacheutil"
 	"github.com/canonical/chisel/internal/archive"
 	"github.com/canonical/chisel/internal/setup"
 	"github.com/canonical/chisel/public/jsonwall"
 	"github.com/canonical/chisel/public/manifest"
+	"github.com/klauspost/compress/zstd"
 )
 
 const DefaultFilename = "manifest.wall"
@@ -34,6 +40,24 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice {
 	return manifestSlices
 }
 
+// FindPathsInRelease finds all the paths marked with "generate:manifest"
+// for the given release.
+func FindPathsInRelease(r *setup.Release) []string {
+	manifests := []string{}
+	for _, pkg := range r.Packages {
+		for _, slice := range pkg.Slices {
+			for path, info := range slice.Contents {
+				if info.Generate == setup.GenerateManifest {
+					dir := strings.TrimSuffix(path, "**")
+					path = filepath.Join(dir, DefaultFilename)
+					manifests = append(manifests, path)
+				}
+			}
+		}
+	}
+	return manifests
+}
+
 type WriteOptions struct {
 	PackageInfo []*archive.PackageInfo
 	Selection   []*setup.Slice
@@ -340,3 +364,210 @@ func Validate(mfest *manifest.Manifest) (err error) {
 	}
 	return nil
 }
+
+// RootFSManifest extracts the manifest from a targetDir
+func RootFSManifest(release *setup.Release, targetDir string) (*manifest.Manifest, error) {
+	manifestPaths := FindPathsInRelease(release)
+	if len(manifestPaths) == 0 {
+		// No manifest in the release means it cannot produce a rootfs that can
+		// be recut. Treat this case as cutting a new rootfs.
+		return nil, nil
+	}
+
+	// Select the first manifest of the list as the reference one for now.
+	// Another heuristic could be used (ex. select the one from base-files_chisel).
+	refManifestRelPath := manifestPaths[0]
+	refManifestPath := path.Join(targetDir, refManifestRelPath)
+	refManifest, err := load(refManifestPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read manifest %q from the root directory: %v", refManifestRelPath, err)
+	}
+
+	err = checkConsistency(refManifestRelPath, targetDir, manifestPaths[1:])
+	if err != nil {
+		return nil, err
+	}
+
+	err = validateRootfs(refManifest, targetDir)
+	if err != nil {
+		return nil, err
+	}
+	return refManifest, nil
+}
+
+// load reads, validates and returns a manifest.
+func load(manifestPath string) (*manifest.Manifest, error) {
+	f, err := os.Open(manifestPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	r, err := zstd.NewReader(f)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	mfest, err := manifest.Read(r)
+	if err != nil {
+		return nil, err
+	}
+
+	err = Validate(mfest)
+	if err != nil {
+		return nil, err
+	}
+
+	return mfest, nil
+}
+
+// checkConsistency checks consistency between a list of manifests and a
+// reference one.
+func checkConsistency(referenceRelPath string, targetDir string, manifests []string) error {
+	reference := path.Join(targetDir, referenceRelPath)
+	hashReference, err := hash(reference)
+	if err != nil {
+		return err
+	}
+
+	infoRef, err := os.Stat(reference)
+	if err != nil {
+		return err
+	}
+
+	modeRef := infoRef.Mode()
+
+	for _, manifestRelPath := range manifests {
+		manifestPath := path.Join(targetDir, manifestRelPath)
+		infoManifest, err := os.Stat(manifestPath)
+		if err != nil {
+			return err
+		}
+
+		modeManifest := infoManifest.Mode()
+		if modeManifest != modeRef {
+			return fmt.Errorf("invalid manifest: permissions on %s (%s) are different from the reference manifest %s (%s)", manifestRelPath, modeManifest, referenceRelPath, modeRef)
+		}
+
+		hashM, err := hash(manifestPath)
+		if err != nil {
+			return err
+		}
+		if !slices.Equal(hashM, hashReference) {
+			return fmt.Errorf("invalid manifest: %s is inconsistent with %s", manifestRelPath, referenceRelPath)
+		}
+	}
+
+	return nil
+}
+
+func hash(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return nil, err
+	}
+
+	return h.Sum(nil), nil
+}
+
+// SliceKeys returns setup.SliceKeys from a manifest.
+func SliceKeys(m *manifest.Manifest) []setup.SliceKey {
+	sliceKeys := []setup.SliceKey{}
+	m.IterateSlices("", func(slice *manifest.Slice) error {
+		sk, err := apacheutil.ParseSliceKey(slice.Name)
+		if err != nil {
+			return err
+		}
+		sliceKeys = append(sliceKeys, sk)
+		return nil
+	})
+
+	return sliceKeys
+}
+
+// reportFromRootfs builds a Report from root directory
+// Implementation heavily borrowed from testutil.TreeDump
+func reportFromRootfs(rootDir string) (*Report, error) {
+	report, err := NewReport(rootDir)
+	if err != nil {
+		return nil, fmt.Errorf("internal error: cannot create report: %w", err)
+	}
+
+	var inodes []uint64
+	pathsByInodes := make(map[uint64][]string)
+
+	dirfs := os.DirFS(report.Root)
+	err = fs.WalkDir(dirfs, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return fmt.Errorf("walk error: %w", err)
+		}
+		if path == "." {
+			return nil
+		}
+		fpath := filepath.Join(report.Root, path)
+		finfo, err := d.Info()
+		if err != nil {
+			return fmt.Errorf("cannot get stat info for %q: %w", fpath, err)
+		}
+
+		entry := ReportEntry{
+			Mode: finfo.Mode(),
+		}
+
+		var size int
+
+		ftype := finfo.Mode() & fs.ModeType
+		switch ftype {
+		case fs.ModeDir:
+			path = "/" + path + "/"
+		case fs.ModeSymlink:
+			lpath, err := os.Readlink(fpath)
+			if err != nil {
+				return err
+			}
+			path = "/" + path
+			entry.Link = lpath
+		case 0: // Regular
+			data, err := os.ReadFile(fpath)
+			if err != nil {
+				return fmt.Errorf("cannot read file: %w", err)
+			}
+			if len(data) >= 0 {
+				sum := sha256.Sum256(data)
+				entry.SHA256 = hex.EncodeToString(sum[:])
+			}
+			path = "/" + path
+			size = int(finfo.Size())
+		default:
+			return fmt.Errorf("unknown file type %d: %s", ftype, fpath)
+		}
+		entry.Path = path
+		entry.Size = size
+
+		if ftype != fs.ModeDir {
+			stat, ok := finfo.Sys().(*syscall.Stat_t)
+			if !ok {
+				return fmt.Errorf("cannot get syscall stat info for %q", fpath)
+			}
+			inode := stat.Ino
+			if len(pathsByInodes[inode]) == 1 {
+				inodes = append(inodes, inode)
+			}
+			entry.Inode = inode
+			pathsByInodes[inode] = append(pathsByInodes[inode], path)
+		}
+
+		report.Entries[path] = entry
+
+		return nil
+	})
+
+	return report, nil
+}