From 463bf8a0bead41e2534baf4e5a92353c4ad47b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20L=C3=B6nnegren?= Date: Thu, 26 Sep 2024 14:40:14 +0200 Subject: [PATCH 1/2] Do not error out on malformed yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make default executor Graph method not completely error out when encountering an unparseable yaml file. Signed-off-by: Fredrik Lönnegren --- go.mod | 3 +- pkg/executor/default.go | 5 ++- pkg/executor/default_test.go | 77 +++++++++++++++++++++++++++--------- 3 files changed, 64 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 11e45426..7dbe3171 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/rancher/yip -go 1.22 +go 1.22.0 + toolchain go1.22.7 require ( diff --git a/pkg/executor/default.go b/pkg/executor/default.go index 541feb63..f46177b4 100644 --- a/pkg/executor/default.go +++ b/pkg/executor/default.go @@ -189,9 +189,10 @@ func (e *DefaultExecutor) dirOps(stage, dir string, fs vfs.FS, console plugins.C config, err := schema.Load(path, fs, schema.FromFile, e.modifier) if err != nil { - return err - + e.logger.Warnf("failed to load file '%s': %s", path, err.Error()) + return nil } + ops := e.genOpFromSchema(path, stage, *config, fs, console) // mark lexicographic order dependency from previous blocks if len(prev) > 0 && len(ops) > 0 { diff --git a/pkg/executor/default_test.go b/pkg/executor/default_test.go index 8cc44395..f029a39f 100644 --- a/pkg/executor/default_test.go +++ b/pkg/executor/default_test.go @@ -487,25 +487,25 @@ stages: stages: initramfs: - users: - kairos: + user1: groups: - sudo - passwd: kairos + passwd: hunter2 - users: - kairos: + user1: groups: - sudo - passwd: kairos + passwd: hunter2 - users: - kairos: + user1: groups: - sudo - passwd: kairos + passwd: hunter2 - users: - kairos: + user1: groups: - sudo - passwd: kairos + passwd: hunter2 `, }) Expect(err).Should(BeNil()) @@ -531,28 +531,28 @@ stages: "/some/yip/01_first.yaml": ` stages: initramfs: - - name: Create Kairos User + - name: Create User users: - kairos: + user1: groups: - sudo - passwd: kairos + passwd: hunter2 - users: - kairos: + user1: groups: - sudo - passwd: kairos - - name: Create Kairos User + passwd: hunter2 + - name: Create User users: - kairos: + user1: groups: - sudo - passwd: kairos + passwd: hunter2 - users: - kairos: + user1: groups: - sudo - passwd: kairos + passwd: hunter2 `, }) Expect(err).Should(BeNil()) @@ -748,5 +748,46 @@ stages: g, _ := def.Graph("default", vfs.OSFS, testConsole, temp) Expect(len(g)).To(Equal(4)) }) + It("loads a graph with malformed yaml files", func() { + def := NewExecutor() + testConsole := console.NewStandardConsole() + + fs, cleanup, err := vfst.NewTestFS(map[string]interface{}{ + "/some/yip/01_first.yaml": ` +name: "Rootfs Layout Settings" +stages: + rootfs.before: + - name: "before roots" + commands: + - echo "rootfs.before" + rootfs: + - name: "rootfs" + commands: + - echo "rootfs" + - name: "rootfs 2" + commands: + - echo "2" + initramfs: + - name: "initramfs" + commands: + - echo "initramfs" +`, + "/some/yip/02_malformed.yaml": ` +name: "second Rootfs Layout Settings" +stages: + rootfs.before: + # bad indentation + - name: "second before roots" + commands: + - echo "second.rootfs.before" +`, + }) + Expect(err).Should(BeNil()) + defer cleanup() + + g, err := def.Graph("rootfs", fs, testConsole, "/some/yip") + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(g)).To(Equal(3)) + }) }) }) From 384d60e0329325f7f3ccb3b794858953f2ebd3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20L=C3=B6nnegren?= Date: Fri, 27 Sep 2024 09:45:39 +0200 Subject: [PATCH 2/2] Return error for errors on loading yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this commit we add back the error when trying to load a corrupt yaml file, but we still execute the entire graph first, making the command fail with an error about which path failed to load, but not breaking the execution for other yaml files. Signed-off-by: Fredrik Lönnegren --- pkg/executor/default.go | 52 +++++++++++++++++++++++++----------- pkg/executor/default_test.go | 2 +- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/pkg/executor/default.go b/pkg/executor/default.go index f46177b4..27e4b5d7 100644 --- a/pkg/executor/default.go +++ b/pkg/executor/default.go @@ -75,6 +75,20 @@ func (l opList) uniqueNames() { } } +type loaderError struct { + path string +} + +func (e loaderError) Error() string { + return fmt.Sprintf("error loading path '%s'", e.path) +} + +func newLoaderError(path string) loaderError { + return loaderError{ + path, + } +} + func (e *DefaultExecutor) applyStage(stage schema.Stage, fs vfs.FS, console plugins.Console) error { var errs error for _, p := range e.conditionals { @@ -167,10 +181,10 @@ func (e *DefaultExecutor) genOpFromSchema(file, stage string, config schema.YipC return results } -func (e *DefaultExecutor) dirOps(stage, dir string, fs vfs.FS, console plugins.Console) ([]*op, error) { - results := []*op{} +func (e *DefaultExecutor) dirOps(stage, dir string, fs vfs.FS, console plugins.Console) (results []*op, loaderError error, err error) { + results = []*op{} prev := []*op{} - err := vfs.Walk(fs, dir, + err = vfs.Walk(fs, dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -190,6 +204,7 @@ func (e *DefaultExecutor) dirOps(stage, dir string, fs vfs.FS, console plugins.C config, err := schema.Load(path, fs, schema.FromFile, e.modifier) if err != nil { e.logger.Warnf("failed to load file '%s': %s", path, err.Error()) + loaderError = multierror.Append(loaderError, newLoaderError(path)) return nil } @@ -210,7 +225,8 @@ func (e *DefaultExecutor) dirOps(stage, dir string, fs vfs.FS, console plugins.C results = append(results, ops...) return nil }) - return results, err + + return results, loaderError, err } func writeDAG(dag [][]dag.GraphEntry) { @@ -228,17 +244,17 @@ func writeDAG(dag [][]dag.GraphEntry) { } func (e *DefaultExecutor) Graph(stage string, fs vfs.FS, console plugins.Console, source string) ([][]dag.GraphEntry, error) { - g, err := e.prepareDAG(stage, source, fs, console) + g, loaderError, err := e.prepareDAG(stage, source, fs, console) if err != nil { return nil, err } - return g.Analyze(), err + return g.Analyze(), loaderError } func (e *DefaultExecutor) Analyze(stage string, fs vfs.FS, console plugins.Console, args ...string) { var errs error for _, source := range args { - g, err := e.prepareDAG(stage, source, fs, console) + g, _, err := e.prepareDAG(stage, source, fs, console) if err != nil { errs = multierror.Append(errs, err) continue @@ -256,35 +272,35 @@ func (e *DefaultExecutor) Analyze(stage string, fs vfs.FS, console plugins.Conso } } -func (e *DefaultExecutor) prepareDAG(stage, uri string, fs vfs.FS, console plugins.Console) (*dag.Graph, error) { +func (e *DefaultExecutor) prepareDAG(stage, uri string, fs vfs.FS, console plugins.Console) (g *dag.Graph, loaderError error, err error) { f, err := fs.Stat(uri) - g := dag.DAG(dag.EnableInit) + g = dag.DAG(dag.EnableInit) var ops opList switch { case err == nil && f.IsDir(): - ops, err = e.dirOps(stage, uri, fs, console) + ops, loaderError, err = e.dirOps(stage, uri, fs, console) if err != nil { - return nil, err + return nil, loaderError, err } case err == nil: config, err := schema.Load(uri, fs, schema.FromFile, e.modifier) if err != nil { - return nil, err + return nil, nil, err } ops = e.genOpFromSchema(uri, stage, *config, fs, console) case utils.IsUrl(uri): config, err := schema.Load(uri, fs, schema.FromUrl, e.modifier) if err != nil { - return nil, err + return nil, nil, err } ops = e.genOpFromSchema(uri, stage, *config, fs, console) default: config, err := schema.Load(uri, fs, nil, e.modifier) if err != nil { - return nil, err + return nil, nil, err } ops = e.genOpFromSchema("", stage, *config, fs, console) @@ -296,11 +312,11 @@ func (e *DefaultExecutor) prepareDAG(stage, uri string, fs vfs.FS, console plugi g.Add(o.name, append(o.options, dag.WithCallback(o.fn), dag.WithDeps(append(o.after, o.deps...)...))...) } - return g, nil + return g, loaderError, nil } func (e *DefaultExecutor) runStage(stage, uri string, fs vfs.FS, console plugins.Console) (err error) { - g, err := e.prepareDAG(stage, uri, fs, console) + g, loaderError, err := e.prepareDAG(stage, uri, fs, console) if err != nil { return err } @@ -322,6 +338,10 @@ func (e *DefaultExecutor) runStage(stage, uri string, fs vfs.FS, console plugins } } + if loaderError != nil { + err = multierror.Append(err, loaderError) + } + return err } diff --git a/pkg/executor/default_test.go b/pkg/executor/default_test.go index f029a39f..6060dc52 100644 --- a/pkg/executor/default_test.go +++ b/pkg/executor/default_test.go @@ -786,7 +786,7 @@ stages: defer cleanup() g, err := def.Graph("rootfs", fs, testConsole, "/some/yip") - Expect(err).ShouldNot(HaveOccurred()) + Expect(err).Should(HaveOccurred()) Expect(len(g)).To(Equal(3)) }) })