diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79..4e9356c0 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "slices" "time" @@ -11,6 +12,7 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/public/manifest" ) var shortCutHelp = "Cut a tree with selected slices" @@ -73,6 +75,27 @@ func (cmd *cmdCut) Execute(args []string) error { } } + var mfest *manifest.Manifest + // TODO: Remove this gating once the final upgrading strategy is in place. + if os.Getenv("CHISEL_RECUT_EXPERIMENTAL") != "" { + mfest, err := slicer.SelectValidManifest(cmd.RootDir, release) + if err != nil { + return err + } + if mfest != nil { + err = mfest.IterateSlices("", func(slice *manifest.Slice) error { + sk, err := setup.ParseSliceKey(slice.Name) + if err != nil { + return err + } + sliceKeys = append(sliceKeys, sk) + return nil + }) + if err != nil { + return err + } + } + } selection, err := setup.Select(release, sliceKeys, cmd.Arch) if err != nil { return err @@ -125,6 +148,7 @@ func (cmd *cmdCut) Execute(args []string) error { Selection: selection, Archives: archives, TargetDir: cmd.RootDir, + Manifest: mfest, }) return err } diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 0503c96f..08e58828 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -168,6 +168,10 @@ func createDir(o *CreateOptions) error { } err = os.Mkdir(path, o.Mode) if os.IsExist(err) { + // TODO: Detect if existing content is a file. ErrExist is also returned + // if a file exists at this path, so returning nil here creates an + // inconsistency between our view of the content and the real content on + // disk which is a bug that must be fixed. return nil } return err @@ -179,6 +183,9 @@ func createFile(o *CreateOptions) error { if err != nil { return err } + // TODO: Detect if existing content is a symlink and remove it if so. The + // current implementation resolves the symlink and will override the target + // and not the symlink itself which is a bug. file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, o.Mode) if err != nil { return err diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 16b05402..071ed326 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "io/fs" + "maps" "path/filepath" "slices" "sort" @@ -34,6 +35,24 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { return manifestSlices } +// FindPathsInRelease finds all the paths marked with "generate:manifest" +// for the given release. +func FindPathsInRelease(r *setup.Release) []string { + manifestPaths := make(map[string]struct{}) + for _, pkg := range r.Packages { + for _, slice := range pkg.Slices { + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + manifestPaths[path] = struct{}{} + } + } + } + } + return slices.Sorted(maps.Keys(manifestPaths)) +} + type WriteOptions struct { PackageInfo []*archive.PackageInfo Selection []*setup.Slice diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 2bab0a68..f096e15b 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -110,10 +110,121 @@ func (s *S) TestFindPaths(c *C) { } } +var findPathsInReleaseTests = []struct { + summary string + release *setup.Release + expected []string +}{{ + summary: "Single package with single slice", + release: &setup.Release{ + Packages: map[string]*setup.Package{ + "package1": { + Name: "package1", + Slices: map[string]*setup.Slice{ + "slice1": { + Name: "slice1", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + }, + }, + }, + }, + expected: []string{"/folder/manifest.wall"}, +}, { + summary: "No slices with generate:manifest", + release: &setup.Release{ + Packages: map[string]*setup.Package{ + "package1": { + Name: "package1", + Slices: map[string]*setup.Slice{ + "slice1": { + Name: "slice1", + Contents: map[string]setup.PathInfo{}, + }, + }, + }, + }, + }, + expected: nil, +}, { + summary: "Multiple packages with multiple slices", + release: &setup.Release{ + Packages: map[string]*setup.Package{ + "package1": { + Name: "package1", + Slices: map[string]*setup.Slice{ + "slice1": { + Name: "slice1", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + "slice2": { + Name: "slice2", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + }, + }, + "package2": { + Name: "package2", + Slices: map[string]*setup.Slice{ + "slice3": { + Name: "slice3", + Contents: map[string]setup.PathInfo{}, + }, + "slice4": { + Name: "slice4", + Contents: map[string]setup.PathInfo{ + "/other-folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + }, + }, + }, + }, + expected: []string{"/folder/manifest.wall", "/other-folder/manifest.wall"}, +}, { + summary: "Empty release", + release: &setup.Release{ + Packages: map[string]*setup.Package{}, + }, + expected: nil, +}} + +func (s *S) TestFindPathsInRelease(c *C) { + for _, test := range findPathsInReleaseTests { + c.Logf("Summary: %s", test.summary) + + manifestPaths := manifestutil.FindPathsInRelease(test.release) + + c.Assert(manifestPaths, HasLen, len(test.expected)) + slices.Sort(manifestPaths) + slices.Sort(test.expected) + c.Assert(manifestPaths, DeepEquals, test.expected) + } +} + var slice1 = &setup.Slice{ Package: "package1", Name: "slice1", } + var slice2 = &setup.Slice{ Package: "package2", Name: "slice2", diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb..e9bc859e 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -3,10 +3,15 @@ package slicer import ( "archive/tar" "bytes" + "crypto/sha256" + "encoding/hex" + "errors" "fmt" "io" "io/fs" + "maps" "os" + "path" "path/filepath" "slices" "sort" @@ -21,6 +26,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/public/manifest" ) const manifestMode fs.FileMode = 0644 @@ -29,6 +35,7 @@ type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive TargetDir string + Manifest *manifest.Manifest } type pathData struct { @@ -90,14 +97,44 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + optsCopy := *options + installOpts := &optsCopy + installOpts.TargetDir = targetDir + if options.Manifest != nil { + tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-workdir-*") + if err != nil { + return fmt.Errorf("cannot create temporary working directory: %w", err) + } + installOpts.TargetDir = tmpWorkDir + defer func() { + os.RemoveAll(tmpWorkDir) + }() + } + + report, err := install(installOpts) if err != nil { return err } + if options.Manifest != nil { + err = upgrade(targetDir, installOpts.TargetDir, report, options.Manifest) + if err != nil { + return err + } + } + + return nil +} + +func install(options *RunOptions) (*manifestutil.Report, error) { + pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + if err != nil { + return nil, err + } + prefers, err := options.Selection.Prefers() if err != nil { - return err + return nil, err } // Build information to process the selection. @@ -154,7 +191,7 @@ func Run(options *RunOptions) error { } reader, info, err := pkgArchive[slice.Package].Fetch(slice.Package) if err != nil { - return err + return nil, err } defer reader.Close() packages[slice.Package] = reader @@ -165,9 +202,9 @@ func Run(options *RunOptions) error { // listed as until: mutate in all the slices that reference them. knownPaths := map[string]pathData{} addKnownPath(knownPaths, "/", pathData{}) - report, err := manifestutil.NewReport(targetDir) + report, err := manifestutil.NewReport(options.TargetDir) if err != nil { - return fmt.Errorf("internal error: cannot create report: %w", err) + return nil, fmt.Errorf("internal error: cannot create report: %w", err) } // Record directories which are created but where not listed in the slice @@ -183,7 +220,7 @@ func Run(options *RunOptions) error { return err } - relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, targetDir)) + relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, options.TargetDir)) if o.Mode.IsDir() { relPath = relPath + "/" } @@ -241,13 +278,13 @@ func Run(options *RunOptions) error { err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], - TargetDir: targetDir, + TargetDir: options.TargetDir, Create: create, }) reader.Close() packages[slice.Package] = nil if err != nil { - return err + return nil, err } } @@ -303,9 +340,9 @@ func Run(options *RunOptions) error { mutable: pathInfo.Mutable, } addKnownPath(knownPaths, relPath, data) - entry, err := createFile(targetDir, relPath, pathInfo) + entry, err := createFile(options.TargetDir, relPath, pathInfo) if err != nil { - return err + return nil, err } // Do not add paths with "until: mutate". @@ -313,7 +350,7 @@ func Run(options *RunOptions) error { for _, slice := range slices { err = report.Add(slice, entry) if err != nil { - return err + return nil, err } } } @@ -323,7 +360,7 @@ func Run(options *RunOptions) error { // dependencies must run before dependents. checker := contentChecker{knownPaths} content := &scripts.ContentValue{ - RootDir: targetDir, + RootDir: options.TargetDir, CheckWrite: checker.checkMutable, CheckRead: checker.checkKnown, OnWrite: report.Mutate, @@ -338,16 +375,141 @@ func Run(options *RunOptions) error { } err := scripts.Run(&opts) if err != nil { - return fmt.Errorf("slice %s: %w", slice, err) + return nil, fmt.Errorf("slice %s: %w", slice, err) + } + } + + err = removeAfterMutate(options.TargetDir, knownPaths) + if err != nil { + return nil, err + } + + err = generateManifests(options.TargetDir, options.Selection, report, pkgInfos) + if err != nil { + return nil, err + } + + return report, nil +} + +// upgrade upgrades content in targetDir with content in tempDir. +func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfest *manifest.Manifest) error { + logf("Upgrading content...") + paths := slices.Sorted(maps.Keys(report.Entries)) + for _, path := range paths { + entry := report.Entries[path] + srcPath := filepath.Clean(filepath.Join(tempDir, path)) + dstPath := filepath.Clean(filepath.Join(targetDir, path)) + + // Create parent directories, removing any file on the way up. + if err := mkParentAll(dstPath); err != nil { + return fmt.Errorf("cannot create parent directory for %q: %s", path, err) + } + + var err error + switch entry.Mode & fs.ModeType { + case 0, fs.ModeSymlink: + err = upgradeFile(srcPath, dstPath) + if err != nil { + err = fmt.Errorf("cannot upgrade file at %q: %s", path, err) + } + case fs.ModeDir: + err = upgradeDir(dstPath, entry) + if err != nil { + err = fmt.Errorf("cannot upgrade directory at %q: %s", path, err) + } + default: + err = fmt.Errorf("unsupported file type: %s", path) + } + if err != nil { + return err } } - err = removeAfterMutate(targetDir, knownPaths) + // Remove missing paths. + missingPaths := make([]string, 0) + err := mfest.IteratePaths("", func(path *manifest.Path) error { + _, ok := report.Entries[path.Path] + if !ok { + missingPaths = append(missingPaths, path.Path) + } + return nil + }) if err != nil { return err } + sort.Sort(sort.Reverse(sort.StringSlice(missingPaths))) + // Go through the list in reverse order to empty directories before removing + // them. Any ENOTEMPTY error encountered means user content is in the directory + // and Chisel does not manage it anymore. + for _, relPath := range missingPaths { + path := filepath.Clean(filepath.Join(targetDir, relPath)) + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, syscall.ENOTEMPTY) { + return err + } + } + return nil +} - return generateManifests(targetDir, options.Selection, report, pkgInfos) +func mkParentAll(path string) error { + parent := filepath.Dir(path) + err := os.MkdirAll(parent, 0o755) + if err == nil { + return nil + } + e, ok := err.(*os.PathError) + if !ok || !errors.Is(e.Unwrap(), syscall.ENOTDIR) { + return err + } + err = os.Remove(parent) + if err != nil { + return err + } + err = os.MkdirAll(parent, 0o755) + if err != nil { + return mkParentAll(parent) + } + return nil +} + +func upgradeDir(path string, entry manifestutil.ReportEntry) error { + err := os.Mkdir(path, entry.Mode) + if err != nil { + if !os.IsExist(err) { + return err + } + fileinfo, err := os.Lstat(path) + if err != nil { + return err + } + if fileinfo.IsDir() { + return os.Chmod(path, entry.Mode) + } + // Path is a regular file or symlink, remove it. + err = os.Remove(path) + if err != nil { + return err + } + + return os.Mkdir(path, entry.Mode) + } + return nil +} + +func upgradeFile(srcPath string, dstPath string) error { + err := os.Rename(srcPath, dstPath) + if err != nil { + if !os.IsExist(err) { + return err + } + err = os.RemoveAll(dstPath) + if err != nil { + return err + } + return os.Rename(srcPath, dstPath) + } + return nil } func generateManifests(targetDir string, selection *setup.Selection, @@ -537,3 +699,84 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel } return pkgArchive, nil } + +// SelectValidManifest returns, if found, a valid manifest. Consistency with +// other manifests is verified so the selection is deterministic. +func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Manifest, error) { + targetDir = filepath.Clean(targetDir) + if !filepath.IsAbs(targetDir) { + dir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("cannot obtain current directory: %w", err) + } + targetDir = filepath.Join(dir, targetDir) + } + manifestPaths := manifestutil.FindPathsInRelease(release) + if len(manifestPaths) == 0 { + return nil, nil + } + + var selected *manifest.Manifest + var selectedHash string + var selectedPath string + foundUnknownSchema := false + for _, mfestPath := range manifestPaths { + mfestFullPath := path.Join(targetDir, mfestPath) + f, err := os.Open(mfestFullPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return nil, err + } + defer r.Close() + mfest, err := manifest.Read(r) + if err != nil { + if errors.Is(err, manifest.ErrUnknownSchema) { + foundUnknownSchema = true + // Ignore manifests with unknown (potentially future) schema versions. + continue + } + return nil, err + } + err = manifestutil.Validate(mfest) + if err != nil { + return nil, err + } + h, err := contentHash(mfestFullPath) + if err != nil { + return nil, fmt.Errorf("cannot compute hash for %q: %s", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + if selected == nil { + selected = mfest + selectedHash = mfestHash + selectedPath = mfestPath + } else if selectedHash != mfestHash { + return nil, fmt.Errorf("cannot select a manifest: %q and %q are inconsistent", selectedPath, mfestPath) + } + } + if foundUnknownSchema && selected == nil { + return nil, fmt.Errorf("cannot select a manifest: manifest(s) found use unknown schema") + } + return selected, nil +} + +func contentHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index fe408e6d..d19441b7 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -19,6 +19,7 @@ import ( "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" "github.com/canonical/chisel/internal/testutil" + "github.com/canonical/chisel/public/jsonwall" "github.com/canonical/chisel/public/manifest" ) @@ -2165,6 +2166,597 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { } } +type slicerRecutTest struct { + summary string + arch string + release map[string]string + pkgs []*testutil.TestPackage + cutSlices []setup.SliceKey + recutSlices []setup.SliceKey + hackopt func(c *C, opts *slicer.RunOptions) + // Modifies the filesystem built after the first execution and before the + // second one. + alterFilesystem func(c *C, targetDir string) + filesystem map[string]string + manifestPaths map[string]string + manifestPkgs map[string]string + error string +} + +var slicerRecutTests = []slicerRecutTest{{ + summary: "Basic upgrade", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /dir/file: + /dir/file-copy: {copy: /dir/file} + /other-dir/file: {symlink: ../dir/file} + slice2: + contents: + /dir/other-file: + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/file-copy": "file 0644 cc55e2ec", + "/dir/other-file": "file 0644 63d5dd49", + "/other-dir/": "dir 0755", + "/other-dir/file": "symlink ../dir/file", + }, + manifestPaths: map[string]string{ + "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", + "/dir/file-copy": "file 0644 cc55e2ec {test-package_slice1}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", + "/other-dir/file": "symlink ../dir/file {test-package_slice1}", + }, +}, { + summary: "Upgrade removes obsolete paths when selection shrinks", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice2"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /dir/file: + slice2: + contents: + /dir/other-file: + `, + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/other-file": "file 0644 63d5dd49", + }, + manifestPaths: map[string]string{ + "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", + }, +}, { + summary: "Upgrade restores modified content and mode", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /dir/file: + `, + }, + alterFilesystem: func(c *C, targetDir string) { + modifiedPath := filepath.Join(targetDir, "dir/file") + err := os.WriteFile(modifiedPath, []byte("data2"), 0o700) + c.Assert(err, IsNil) + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + }, + manifestPaths: map[string]string{ + "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", + }, +}, { + summary: "Upgrade keeps untracked files", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /dir/file: + `, + }, + alterFilesystem: func(c *C, targetDir string) { + err := os.MkdirAll(filepath.Join(targetDir, "extra"), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(targetDir, "extra", "untracked"), []byte("data"), 0o644) + c.Assert(err, IsNil) + }, + filesystem: map[string]string{ + "/extra/": "dir 0755", + "/extra/untracked": "file 0644 3a6eb079", + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + }, + manifestPaths: map[string]string{ + "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", + }, +}, { + summary: "Upgrade overrides existing mode", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /dir/: + slice2: + contents: + /dir/file: + /other-dir/: + `, + }, + alterFilesystem: func(c *C, targetDir string) { + err := os.MkdirAll(filepath.Join(targetDir, "other-dir"), 0o775) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(targetDir, "dir", "file"), []byte("data"), 0o644) + c.Assert(err, IsNil) + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/other-dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + }, + manifestPaths: map[string]string{ + "/dir/": "dir 0755 {test-package_slice1}", + "/other-dir/": "dir 0755 {test-package_slice2}", + "/dir/file": "file 0644 cc55e2ec {test-package_slice2}", + }, +}, { + summary: "Upgrade overwrites existing symlink", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /dir/: + slice2: + contents: + /baz: {text: data} + /foo: {symlink: baz} + `, + }, + alterFilesystem: func(c *C, targetDir string) { + linkPath := filepath.Join(targetDir, "foo") + err := os.Symlink("bar", linkPath) + c.Assert(err, IsNil) + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/baz": "file 0644 3a6eb079", + "/foo": "symlink baz", + }, + manifestPaths: map[string]string{ + "/dir/": "dir 0755 {test-package_slice1}", + "/baz": "file 0644 3a6eb079 {test-package_slice2}", + "/foo": "symlink baz {test-package_slice2}", + }, +}, { + summary: "Upgrade removes content to create parent dirs", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /dir/file: {text: data} + slice2: + contents: + `, + }, + alterFilesystem: func(c *C, targetDir string) { + path := filepath.Join(targetDir, "dir") + err := os.RemoveAll(path) + c.Assert(err, IsNil) + err = os.WriteFile(path, []byte("data"), 0o644) + c.Assert(err, IsNil) + }, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/dir/file": "file 0644 3a6eb079 {test-package_slice1}", + }, +}, { + summary: "Upgrade removes content whith unmatching type", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /file: {text: data} + /a-dir/: {make: true} + `, + }, + alterFilesystem: func(c *C, targetDir string) { + filePath := filepath.Join(targetDir, "file") + err := os.Remove(filePath) + c.Assert(err, IsNil) + err = os.Mkdir(filePath, 0o755) + c.Assert(err, IsNil) + dirPath := filepath.Join(targetDir, "a-dir") + err = os.Remove(dirPath) + c.Assert(err, IsNil) + err = os.WriteFile(dirPath, []byte("data"), 0o644) + c.Assert(err, IsNil) + }, + filesystem: map[string]string{ + "/file": "file 0644 3a6eb079", + "/a-dir/": "dir 0755", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 3a6eb079 {test-package_slice1}", + "/a-dir/": "dir 0755 {test-package_slice1}", + }, +}, { + summary: "Upgrade removes obsolete content but keeps non-empty directories", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice2"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /old-dir/: {make: true} + /link: {symlink: target} + /baz: {text: data} + /foo: {symlink: baz} + slice2: + contents: + /new-file: {text: data1} + `, + }, + alterFilesystem: func(c *C, targetDir string) { + err := os.WriteFile(filepath.Join(targetDir, "old-dir", "file"), []byte("data"), 0o644) + c.Assert(err, IsNil) + }, + filesystem: map[string]string{ + "/new-file": "file 0644 5b41362b", + "/old-dir/": "dir 0755", + "/old-dir/file": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/new-file": "file 0644 5b41362b {test-package_slice2}", + }, +}} + +func (s *S) TestRunRecut(c *C) { + for _, test := range slicerRecutTests { + c.Logf("Summary: %s", test.summary) + + if _, ok := test.release["chisel.yaml"]; !ok { + test.release["chisel.yaml"] = testutil.DefaultChiselYaml + } + if test.pkgs == nil { + test.pkgs = []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }} + } + for _, pkg := range test.pkgs { + // We need to set these fields for manifest validation. + if pkg.Arch == "" { + pkg.Arch = "arch" + } + if pkg.Hash == "" { + pkg.Hash = "hash" + } + if pkg.Version == "" { + pkg.Version = "version" + } + } + + releaseDir := c.MkDir() + for path, data := range test.release { + fpath := filepath.Join(releaseDir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) + c.Assert(err, IsNil) + } + + release, err := setup.ReadRelease(releaseDir) + c.Assert(err, IsNil) + + // Create a manifest slice and add it to the selection. + manifestPackage := test.cutSlices[0].Package + manifestPath := "/chisel-data/manifest.wall" + release.Packages[manifestPackage].Slices["manifest"] = &setup.Slice{ + Package: manifestPackage, + Name: "manifest", + Essential: nil, + Contents: map[string]setup.PathInfo{ + "/chisel-data/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + Scripts: setup.SliceScripts{}, + } + test.cutSlices = append(test.cutSlices, setup.SliceKey{ + Package: manifestPackage, + Slice: "manifest", + }) + + selection, err := setup.Select(release, test.cutSlices, test.arch) + c.Assert(err, IsNil) + + archives := map[string]archive.Archive{} + for name, setupArchive := range release.Archives { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { + pkgs[pkg.Name] = pkg + } + } + archive := &testutil.TestArchive{ + Opts: archive.Options{ + Label: setupArchive.Name, + Version: setupArchive.Version, + Suites: setupArchive.Suites, + Components: setupArchive.Components, + Pro: setupArchive.Pro, + Arch: test.arch, + }, + Packages: pkgs, + } + archives[name] = archive + } + + targetDir := c.MkDir() + options := slicer.RunOptions{ + Selection: selection, + Archives: archives, + TargetDir: targetDir, + } + if test.hackopt != nil { + test.hackopt(c, &options) + } + // First run. + err = slicer.Run(&options) + c.Assert(err, IsNil) + + if test.alterFilesystem != nil { + test.alterFilesystem(c, targetDir) + } + mfest := readManifest(c, options.TargetDir, manifestPath) + + test.recutSlices = append(test.recutSlices, setup.SliceKey{ + Package: manifestPackage, + Slice: "manifest", + }) + selection, err = setup.Select(release, test.recutSlices, test.arch) + c.Assert(err, IsNil) + + options = slicer.RunOptions{ + Selection: selection, + Archives: archives, + TargetDir: targetDir, + Manifest: mfest, + } + // Second run. + err = slicer.Run(&options) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + + if test.filesystem == nil && test.manifestPaths == nil && test.manifestPkgs == nil { + continue + } + mfest = readManifest(c, options.TargetDir, manifestPath) + + // Assert state of final filesystem. + if test.filesystem != nil { + filesystem := testutil.TreeDump(options.TargetDir) + c.Assert(filesystem["/chisel-data/"], Not(HasLen), 0) + c.Assert(filesystem[manifestPath], Not(HasLen), 0) + delete(filesystem, "/chisel-data/") + delete(filesystem, manifestPath) + c.Assert(filesystem, DeepEquals, test.filesystem) + } + + // Assert state of the files recorded in the manifest. + if test.manifestPaths != nil { + pathsDump, err := treeDumpManifestPaths(mfest) + c.Assert(err, IsNil) + c.Assert(pathsDump[manifestPath], Not(HasLen), 0) + delete(pathsDump, manifestPath) + c.Assert(pathsDump, DeepEquals, test.manifestPaths) + } + + // Assert state of the packages recorded in the manifest. + if test.manifestPkgs != nil { + pkgsDump, err := dumpManifestPkgs(mfest) + c.Assert(err, IsNil) + c.Assert(pkgsDump, DeepEquals, test.manifestPkgs) + } + } +} + +type selectValidManifestTest struct { + summary string + build func() *setup.Release + setup func(c *C, targetDir string, release *setup.Release) + noMatch bool + error string +} + +var selectValidManifestTests = []selectValidManifestTest{{ + summary: "No manifest paths in release", + build: func() *setup.Release { + return &setup.Release{Packages: map[string]*setup.Package{}} + }, + noMatch: true, +}, { + summary: "Manifest path missing in target", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + noMatch: true, +}, { + summary: "Unknown schema error ignored when other valid found", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPathA := manifestPathForDir("/chisel-a/**") + manifestPathB := manifestPathForDir("/chisel-b/**") + slice := release.Packages["test-package"].Slices["manifest"] + writeManifest(c, targetDir, manifestPathA, slice, "hash1") + writeInvalidSchemaManifest(c, targetDir, manifestPathB) + }, +}, { + summary: "Unknown schema error raised when no other valid found", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := manifestPathForDir("/chisel/**") + writeInvalidSchemaManifest(c, targetDir, manifestPath) + }, + error: `cannot select a manifest: manifest\(s\) found use unknown schema`, +}, { + summary: "Valid manifest selected", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := manifestPathForDir("/chisel/**") + slice := release.Packages["test-package"].Slices["manifest"] + writeManifest(c, targetDir, manifestPath, slice, "hash1") + }, +}, { + summary: "Two consistent manifests are accepted", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPathA := manifestPathForDir("/chisel-a/**") + manifestPathB := manifestPathForDir("/chisel-b/**") + slice := release.Packages["test-package"].Slices["manifest"] + writeManifest(c, targetDir, manifestPathA, slice, "hash1") + writeManifest(c, targetDir, manifestPathB, slice, "hash1") + }, +}, { + summary: "Inconsistent manifests with same schema are rejected", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPathA := manifestPathForDir("/chisel-a/**") + manifestPathB := manifestPathForDir("/chisel-b/**") + slice := release.Packages["test-package"].Slices["manifest"] + writeManifest(c, targetDir, manifestPathA, slice, "hash1") + writeManifest(c, targetDir, manifestPathB, slice, "hash2") + }, + error: `cannot select a manifest: "/chisel-a/manifest.wall" and "/chisel-b/manifest.wall" are inconsistent`, +}, { + summary: "Invalid manifest data returns error", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := filepath.Join(targetDir, manifestPathForDir("/chisel/**")) + err := os.MkdirAll(filepath.Dir(manifestPath), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(manifestPath, []byte("not-a-zstd-manifest"), 0o644) + c.Assert(err, IsNil) + }, + error: "cannot read manifest: invalid input: .*", +}, { + summary: "Manifest validation error is returned", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := manifestPathForDir("/chisel/**") + writeInvalidManifest(c, targetDir, manifestPath) + }, + error: `invalid manifest: path /file has no matching entry in contents`, +}} + +func (s *S) TestSelectValidManifest(c *C) { + for _, test := range selectValidManifestTests { + c.Logf("Summary: %s", test.summary) + release := test.build() + targetDir := c.MkDir() + if test.setup != nil { + test.setup(c, targetDir, release) + } + mfest, err := slicer.SelectValidManifest(targetDir, release) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + if test.noMatch { + c.Assert(mfest, IsNil) + continue + } + c.Assert(mfest, NotNil) + } +} + +func buildReleaseWithManifestDirs(dirs ...string) *setup.Release { + contents := map[string]setup.PathInfo{} + for _, dir := range dirs { + contents[dir] = setup.PathInfo{Kind: "generate", Generate: "manifest"} + } + return &setup.Release{ + Packages: map[string]*setup.Package{ + "test-package": { + Name: "test-package", + Slices: map[string]*setup.Slice{ + "manifest": { + Package: "test-package", + Name: "manifest", + Contents: contents, + }, + }, + }, + }, + } +} + +func manifestPathForDir(dir string) string { + base := strings.TrimSuffix(dir, "**") + return path.Join(base, manifestutil.DefaultFilename) +} + func treeDumpManifestPaths(mfest *manifest.Manifest) (map[string]string, error) { result := make(map[string]string) err := mfest.IteratePaths("", func(path *manifest.Path) error { @@ -2241,3 +2833,68 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { return mfest } + +func writeManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, hash string) { + mfestPath := filepath.Join(targetDir, manifestPath) + err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) + c.Assert(err, IsNil) + f, err := os.Create(mfestPath) + c.Assert(err, IsNil) + zw, err := zstd.NewWriter(f) + c.Assert(err, IsNil) + options := &manifestutil.WriteOptions{ + PackageInfo: []*archive.PackageInfo{{ + Name: slice.Package, + Version: "1.0", + Arch: "amd64", + SHA256: "pkg-hash", + }}, + Selection: []*setup.Slice{slice}, + Report: &manifestutil.Report{Root: "/", Entries: map[string]manifestutil.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0o644, + SHA256: hash, + Size: 3, + Slices: map[*setup.Slice]bool{slice: true}, + }, + }}, + } + err = manifestutil.Write(options, zw) + c.Assert(err, IsNil) + c.Assert(zw.Close(), IsNil) + c.Assert(f.Close(), IsNil) + c.Assert(os.Chmod(mfestPath, 0o644), IsNil) +} + +func writeInvalidManifest(c *C, targetDir, manifestPath string) { + mfestPath := filepath.Join(targetDir, manifestPath) + err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) + c.Assert(err, IsNil) + f, err := os.Create(mfestPath) + c.Assert(err, IsNil) + zw, err := zstd.NewWriter(f) + c.Assert(err, IsNil) + dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{Schema: manifest.Schema}) + err = dbw.Add(&manifest.Path{Kind: "path", Path: "/file", Mode: "0644"}) + c.Assert(err, IsNil) + _, err = dbw.WriteTo(zw) + c.Assert(err, IsNil) + c.Assert(zw.Close(), IsNil) + c.Assert(f.Close(), IsNil) +} + +func writeInvalidSchemaManifest(c *C, targetDir, manifestPath string) { + mfestPath := filepath.Join(targetDir, manifestPath) + err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) + c.Assert(err, IsNil) + f, err := os.Create(mfestPath) + c.Assert(err, IsNil) + zw, err := zstd.NewWriter(f) + c.Assert(err, IsNil) + dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{Schema: "9.9"}) + _, err = dbw.WriteTo(zw) + c.Assert(err, IsNil) + c.Assert(zw.Close(), IsNil) + c.Assert(f.Close(), IsNil) +} diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8..5a1bb34c 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -46,12 +46,14 @@ type Manifest struct { db *jsonwall.DB } +var ErrUnknownSchema = fmt.Errorf("unknown schema version") + // Read loads a Manifest without performing any validation. The data is assumed // to be both valid jsonwall and a valid Manifest (see Validate). func Read(reader io.Reader) (manifest *Manifest, err error) { defer func() { if err != nil { - err = fmt.Errorf("cannot read manifest: %s", err) + err = fmt.Errorf("cannot read manifest: %w", err) } }() @@ -61,7 +63,7 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { } mfestSchema := db.Schema() if mfestSchema != Schema { - return nil, fmt.Errorf("unknown schema version %q", mfestSchema) + return nil, fmt.Errorf("%w %q", ErrUnknownSchema, mfestSchema) } manifest = &Manifest{db: db} diff --git a/tests/recut/chisel-releases/chisel.yaml b/tests/recut/chisel-releases/chisel.yaml new file mode 100644 index 00000000..c137ff40 --- /dev/null +++ b/tests/recut/chisel-releases/chisel.yaml @@ -0,0 +1,49 @@ +format: v2 + + +archives: + # archive.ubuntu.com/ubuntu/ (amd64, i386) + # ports.ubuntu.com/ubuntu-ports/ (other arch) + ubuntu: + priority: 10 + version: 24.04 + components: [main, universe] + suites: [noble, noble-security, noble-updates] + public-keys: [ubuntu-archive-key-2018] + + +public-keys: + # Ubuntu Archive Automatic Signing Key (2018) + # rsa4096/f6ecb3762474eda9d21b7022871920d1991bc93c 2018-09-17T15:01:46Z + ubuntu-archive-key-2018: + id: "871920D1991BC93C" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFufwdoBEADv/Gxytx/LcSXYuM0MwKojbBye81s0G1nEx+lz6VAUpIUZnbkq + dXBHC+dwrGS/CeeLuAjPRLU8AoxE/jjvZVp8xFGEWHYdklqXGZ/gJfP5d3fIUBtZ + HZEJl8B8m9pMHf/AQQdsC+YzizSG5t5Mhnotw044LXtdEEkx2t6Jz0OGrh+5Ioxq + X7pZiq6Cv19BohaUioKMdp7ES6RYfN7ol6HSLFlrMXtVfh/ijpN9j3ZhVGVeRC8k + KHQsJ5PkIbmvxBiUh7SJmfZUx0IQhNMaDHXfdZAGNtnhzzNReb1FqNLSVkrS/Pns + AQzMhG1BDm2VOSF64jebKXffFqM5LXRQTeqTLsjUbbrqR6s/GCO8UF7jfUj6I7ta + LygmsHO/JD4jpKRC0gbpUBfaiJyLvuepx3kWoqL3sN0LhlMI80+fA7GTvoOx4tpq + VlzlE6TajYu+jfW3QpOFS5ewEMdL26hzxsZg/geZvTbArcP+OsJKRmhv4kNo6Ayd + yHQ/3ZV/f3X9mT3/SPLbJaumkgp3Yzd6t5PeBu+ZQk/mN5WNNuaihNEV7llb1Zhv + Y0Fxu9BVd/BNl0rzuxp3rIinB2TX2SCg7wE5xXkwXuQ/2eTDE0v0HlGntkuZjGow + DZkxHZQSxZVOzdZCRVaX/WEFLpKa2AQpw5RJrQ4oZ/OfifXyJzP27o03wQARAQAB + tEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmluZyBLZXkgKDIwMTgpIDxm + dHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwEKACIFAlufwdoCGwMGCwkIBwMCBhUI + AgkKCwQWAgMBAh4BAheAAAoJEIcZINGZG8k8LHMQAKS2cnxz/5WaoCOWArf5g6UH + beOCgc5DBm0hCuFDZWWv427aGei3CPuLw0DGLCXZdyc5dqE8mvjMlOmmAKKlj1uG + g3TYCbQWjWPeMnBPZbkFgkZoXJ7/6CB7bWRht1sHzpt1LTZ+SYDwOwJ68QRp7DRa + Zl9Y6QiUbeuhq2DUcTofVbBxbhrckN4ZteLvm+/nG9m/ciopc66LwRdkxqfJ32Cy + q+1TS5VaIJDG7DWziG+Kbu6qCDM4QNlg3LH7p14CrRxAbc4lvohRgsV4eQqsIcdF + kuVY5HPPj2K8TqpY6STe8Gh0aprG1RV8ZKay3KSMpnyV1fAKn4fM9byiLzQAovC0 + LZ9MMMsrAS/45AvC3IEKSShjLFn1X1dRCiO6/7jmZEoZtAp53hkf8SMBsi78hVNr + BumZwfIdBA1v22+LY4xQK8q4XCoRcA9G+pvzU9YVW7cRnDZZGl0uwOw7z9PkQBF5 + KFKjWDz4fCk+K6+YtGpovGKekGBb8I7EA6UpvPgqA/QdI0t1IBP0N06RQcs1fUaA + QEtz6DGy5zkRhR4pGSZn+dFET7PdAjEK84y7BdY4t+U1jcSIvBj0F2B7LwRL7xGp + SpIKi/ekAXLs117bvFHaCvmUYN7JVp1GMmVFxhIdx6CFm3fxG8QjNb5tere/YqK+ + uOgcXny1UlwtCUzlrSaP + =9AdM + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/recut/chisel-releases/slices/base-files.yaml b/tests/recut/chisel-releases/slices/base-files.yaml new file mode 100644 index 00000000..84c30016 --- /dev/null +++ b/tests/recut/chisel-releases/slices/base-files.yaml @@ -0,0 +1,13 @@ +package: base-files +slices: + slice-a: + contents: + /etc/debian_version: + /etc/foo: + text: bar + slice-b: + contents: + /etc/issue: + manifest: + contents: + /chisel/**: {generate: manifest} diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml new file mode 100644 index 00000000..b87119c3 --- /dev/null +++ b/tests/recut/task.yaml @@ -0,0 +1,39 @@ +summary: Recut relies on manifest to upgrade existing content + +variants: + - noble + +environment: + ROOTFS: rootfs + +execute: | + # Install yq. + wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\ + chmod +x /usr/bin/yq + + mkdir -p ${ROOTFS} + + # TODO: remove this env var when final upgrade strategy is in place. + export CHISEL_RECUT_EXPERIMENTAL=1 + # First cut generates manifest and installs slice-a. + chisel cut --release ./chisel-releases/ \ + --root ${ROOTFS} \ + base-files_slice-a base-files_manifest + + test -s ${ROOTFS}/etc/debian_version + test -s ${ROOTFS}/etc/foo + cat ${ROOTFS}/etc/foo | grep "bar" + test -f ${ROOTFS}/chisel/manifest.wall + + # Update slice-a definition. + yq -i '.slices.slice-a.contents./etc/foo.text = "qux"' ./chisel-releases/slices/base-files.yaml + + # Second cut, only requesting slice-b. + chisel cut --release ./chisel-releases/ \ + --root ${ROOTFS} \ + base-files_slice-b + + test -s ${ROOTFS}/etc/debian_version + test -s ${ROOTFS}/etc/issue + cat ${ROOTFS}/etc/foo | grep "qux" + test -f ${ROOTFS}/chisel/manifest.wall