feat: search, load and validates manifests

cmd/chisel/cmd_cut.go

@@ -11,6 +11,7 @@ import (
 	"github.com/canonical/chisel/internal/cache"
 	"github.com/canonical/chisel/internal/setup"
 	"github.com/canonical/chisel/internal/slicer"
+	"github.com/canonical/chisel/public/manifest"
 )
 
 var shortCutHelp = "Cut a tree with selected slices"
@@ -73,6 +74,21 @@ func (cmd *cmdCut) Execute(args []string) error {
 		}
 	}
 
+	mfest, err := slicer.Inspect(cmd.RootDir, release)
+	if err != nil {
+		return err
+	}
+	if mfest != nil {
+		mfest.IterateSlices("", func(slice *manifest.Slice) error {
+			sk, err := setup.ParseSliceKey(slice.Name)
+			if err != nil {
+				return err
+			}
+			sliceKeys = append(sliceKeys, sk)
+			return nil
+		})
+	}
+
 	selection, err := setup.Select(release, sliceKeys, cmd.Arch)
 	if err != nil {
 		return err

internal/manifestutil/manifestutil.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"maps"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -34,6 +35,19 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice {
 	return manifestSlices
 }
 
+// FindPathsInRelease finds all the paths marked with "generate:manifest"
+// for the given release.
+func FindPathsInRelease(r *setup.Release) []string {
+	allSlices := []*setup.Slice{}
+	for _, pkg := range r.Packages {
+		for _, slice := range pkg.Slices {
+			allSlices = append(allSlices, slice)
+		}
+	}
+	manifestMap := FindPaths(allSlices)
+	return slices.Sorted(maps.Keys(manifestMap))
+}
+
 type WriteOptions struct {
 	PackageInfo []*archive.PackageInfo
 	Selection   []*setup.Slice
@@ -340,3 +354,12 @@ func Validate(mfest *manifest.Manifest) (err error) {
 	}
 	return nil
 }
+
+// CompareSchemas compare two manifest schema strings
+func CompareSchemas(va, vb string) int {
+	if va == manifest.Schema && va == vb {
+		return 0
+	}
+	return -1
+}
+

internal/slicer/check.go

@@ -0,0 +1,186 @@
+package slicer
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/klauspost/compress/zstd"
+
+	"github.com/canonical/chisel/internal/manifestutil"
+	"github.com/canonical/chisel/public/manifest"
+)
+
+type pathInfo struct {
+	mode string
+	size int64
+	link string
+	hash string
+}
+
+func unixPerm(mode fs.FileMode) (perm uint32) {
+	perm = uint32(mode.Perm())
+	if mode&fs.ModeSticky != 0 {
+		perm |= 0o1000
+	}
+	return perm
+}
+
+// checkDir checks the content of the target directory matches with
+// the manifest.
+// This function works under the assumption the manifest is valid.
+// Files not managed by chisel are ignored.
+func checkDir(mfest *manifest.Manifest, rootDir string) error {
+	singlePathsByFSInode := make(map[uint64]string)
+	fsInodeByManifestInode := make(map[uint64]uint64)
+	manifestInfos := make(map[string]*pathInfo)
+	err := mfest.IteratePaths("", func(path *manifest.Path) error {
+		pathHash := path.FinalSHA256
+		if pathHash == "" {
+			pathHash = path.SHA256
+		}
+		recordedPathInfo := &pathInfo{
+			mode: path.Mode,
+			size: int64(path.Size),
+			link: path.Link,
+			hash: pathHash,
+		}
+
+		fsInfo := &pathInfo{}
+		fullPath := filepath.Join(rootDir, path.Path)
+		info, err := os.Lstat(fullPath)
+		if err != nil {
+			return err
+		}
+		mode := info.Mode()
+		fsInfo.mode = fmt.Sprintf("0%o", unixPerm(mode))
+		ftype := mode & fs.ModeType
+		switch ftype {
+		case fs.ModeDir:
+			// Nothing to do
+		case fs.ModeSymlink:
+			fsInfo.link, err = os.Readlink(fullPath)
+			if err != nil {
+				return fmt.Errorf("cannot read symlink %q: %w", fullPath, err)
+			}
+		case 0: // Regular
+			h, err := contentHash(fullPath)
+			if err != nil {
+				return fmt.Errorf("cannot compute hash for %q: %w", fullPath, err)
+			}
+			fsInfo.hash = hex.EncodeToString(h)
+			fsInfo.size = info.Size()
+		default:
+			return fmt.Errorf("inconsistent content: %q has unrecognized type %s", fullPath, mode.String())
+		}
+
+		// Collect manifests for tailored checking later.
+		// Adjust observed hash and size to still compare in a generic way.
+		if filepath.Base(path.Path) == manifestutil.DefaultFilename && recordedPathInfo.size == 0 && recordedPathInfo.hash == "" {
+			mfestInfo := *fsInfo
+			manifestInfos[path.Path] = &mfestInfo
+			fsInfo.size = 0
+			fsInfo.hash = ""
+		}
+
+		if recordedPathInfo.mode != fsInfo.mode {
+			return fmt.Errorf("inconsistent mode at %q: recorded %v, observed %v", path.Path, recordedPathInfo.mode, fsInfo.mode)
+		}
+		if recordedPathInfo.size != fsInfo.size {
+			return fmt.Errorf("inconsistent size at %q: recorded %v, observed %v", path.Path, recordedPathInfo.size, fsInfo.size)
+		}
+		if recordedPathInfo.link != fsInfo.link {
+			return fmt.Errorf("inconsistent link at %q: recorded %v, observed %v", path.Path, recordedPathInfo.link, fsInfo.link)
+		}
+		if recordedPathInfo.hash != fsInfo.hash {
+			return fmt.Errorf("inconsistent hash at %q: recorded %v, observed %v", path.Path, recordedPathInfo.hash, fsInfo.hash)
+		}
+		// Check hardlink
+		if ftype != fs.ModeDir {
+			stat, ok := info.Sys().(*syscall.Stat_t)
+			if !ok {
+				return fmt.Errorf("cannot get syscall stat info for %q", info.Name())
+			}
+			inode := stat.Ino
+
+			if path.Inode == 0 {
+				// This path must not be linked to any other
+				singlePath, ok := singlePathsByFSInode[inode]
+				if ok {
+					return fmt.Errorf("inconsistent content at %q: file hardlinked to %q", path.Path, singlePath)
+				}
+				singlePathsByFSInode[inode] = path.Path
+			} else {
+				recordedInode, ok := fsInodeByManifestInode[path.Inode]
+				if !ok {
+					fsInodeByManifestInode[path.Inode] = inode
+				} else if recordedInode != inode {
+					return fmt.Errorf("inconsistent content at %q: file hardlinked to a different inode", path.Path)
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Check manifests
+	// They must all be valid manifests.
+	// They must be consistent per schema version.
+	schemaManifestInfos := make(map[string]*pathInfo)
+	for path, info := range manifestInfos {
+		fullPath := filepath.Join(rootDir, path)
+		f, err := os.Open(fullPath)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		r, err := zstd.NewReader(f)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+		mfest, err = manifest.Read(r)
+		if err != nil {
+			return err
+		}
+		err = manifestutil.Validate(mfest)
+		if err != nil {
+			return err
+		}
+		schema := mfest.Schema()
+		refInfo, ok := schemaManifestInfos[schema]
+		if !ok {
+			schemaManifestInfos[schema] = info
+			continue
+		}
+
+		if refInfo.size != info.size {
+			return fmt.Errorf("inconsistent manifest size for version %s at %q: recorded %v, observed %v", schema, path, refInfo.size, info.size)
+		}
+		if refInfo.hash != info.hash {
+			return fmt.Errorf("inconsistent manifest hash for version %s at %q: recorded %v, observed %v", schema, path, refInfo.hash, info.hash)
+		}
+	}
+	return nil
+}
+
+func contentHash(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return nil, err
+	}
+	return h.Sum(nil), nil
+}

internal/slicer/slicer.go

@@ -3,10 +3,12 @@ package slicer
 import (
 	"archive/tar"
 	"bytes"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -21,6 +23,7 @@ import (
 	"github.com/canonical/chisel/internal/manifestutil"
 	"github.com/canonical/chisel/internal/scripts"
 	"github.com/canonical/chisel/internal/setup"
+	"github.com/canonical/chisel/public/manifest"
 )
 
 const manifestMode fs.FileMode = 0644
@@ -537,3 +540,93 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel
 	}
 	return pkgArchive, nil
 }
+
+// Inspect examines and validates the targetDir.
+// Return the list of SliceKeys used to build the targetDir.
+func Inspect(targetDir string, release *setup.Release) (*manifest.Manifest, error) {
+	var mfest *manifest.Manifest
+	manifestPaths := manifestutil.FindPathsInRelease(release)
+	if len(manifestPaths) > 0 {
+		logf("Inspecting root directory...")
+		var err error
+		mfest, err = selectValidManifest(targetDir, manifestPaths)
+		if err != nil {
+			return nil, err
+		}
+		if mfest != nil {
+			err = checkDir(mfest, targetDir)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return mfest, nil
+}
+
+// selectValidManifest the first valid manifest found in a directory with the
+// latest schema version.
+// A manifest is considered valid if it is consistent with others present and
+// of the same schema.
+func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Manifest, error) {
+	targetDir = filepath.Clean(targetDir)
+	if !filepath.IsAbs(targetDir) {
+		dir, err := os.Getwd()
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain current directory: %w", err)
+		}
+		targetDir = filepath.Join(dir, targetDir)
+	}
+
+	type manifestHash struct {
+		path string
+		hash string
+	}
+	var selected *manifest.Manifest
+	schemaManifest := make(map[string]manifestHash)
+	for _, mfestPath := range manifestPaths {
+		err := func() error {
+			mfestFullPath := path.Join(targetDir, mfestPath)
+			f, err := os.Open(mfestFullPath)
+			if err != nil {
+				if os.IsNotExist(err) {
+					return nil
+				}
+				return err
+			}
+			defer f.Close()
+			r, err := zstd.NewReader(f)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+			mfest, err := manifest.Read(r)
+			if err != nil {
+				return nil
+			}
+			err = manifestutil.Validate(mfest)
+			if err != nil {
+				return nil
+			}
+
+			if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 {
+				h, err := contentHash(mfestFullPath)
+				if err != nil {
+					return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err)
+				}
+				mfestHash := hex.EncodeToString(h)
+				refMfest, ok := schemaManifest[mfest.Schema()]
+				if !ok {
+					schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash}
+				} else if refMfest.hash != mfestHash {
+					return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath)
+				}
+				selected = mfest
+			}
+			return nil
+		}()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return selected, nil
+}

public/manifest/manifest.go

@@ -106,3 +106,7 @@ func iteratePrefix[T prefixable](manifest *Manifest, prefix *T, onMatch func(*T)
 	}
 	return nil
 }
+
+func (manifest *Manifest) Schema() string {
+	return manifest.db.Schema()
+}