From 0b73a9fdeb5c75dc5df6acd3323c5af5b79a98c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Tue, 30 Apr 2024 12:21:10 +0300 Subject: [PATCH 1/2] Add an example manifest converter for conversion of singleton lists to embedded object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- pkg/config/example_conversions.go | 139 ++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 pkg/config/example_conversions.go diff --git a/pkg/config/example_conversions.go b/pkg/config/example_conversions.go new file mode 100644 index 00000000..2e245c65 --- /dev/null +++ b/pkg/config/example_conversions.go @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: 2024 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8sschema "k8s.io/apimachinery/pkg/runtime/schema" + kyaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" + + "github.com/crossplane/upjet/pkg/config/conversion" +) + +func ConvertSingletonListToEmbeddedObject(pc *Provider, startPath string) error { //nolint:gocyclo + resourceRegistry := prepareResourceRegistry(pc) + err := filepath.Walk(startPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return errors.Wrap(err, "walk failed") + } + + var convertedFileContent string + if !info.IsDir() && strings.HasSuffix(info.Name(), ".yaml") { + log.Printf("Converting: %s\n", path) + content, err := os.ReadFile(path) //nolint:gosec + if err != nil { + return errors.Wrap(err, "failed to read file") + } + + examples, err := decodeExamples(string(content)) + if err != nil { + return errors.Wrap(err, "failed to decode examples") + } + + rootResource := resourceRegistry[fmt.Sprintf("%s/%s", examples[0].GroupVersionKind().Kind, examples[0].GroupVersionKind().Group)] + if rootResource == nil { + return nil + } + + newPath := strings.Replace(path, examples[0].GroupVersionKind().Version, rootResource.Version, -1) //nolint:gocritic + if path == newPath { + return nil + } + annotationValue := strings.ToLower(fmt.Sprintf("%s/%s/%s", rootResource.ShortGroup, rootResource.Version, rootResource.Kind)) + for _, e := range examples { + if resource, ok := resourceRegistry[fmt.Sprintf("%s/%s", e.GroupVersionKind().Kind, e.GroupVersionKind().Group)]; ok { + conversionPaths := resource.CRDListConversionPaths() + if conversionPaths != nil && e.GroupVersionKind().Version != resource.Version { + for i, cp := range conversionPaths { + conversionPaths[i] = "spec.forProvider." + cp + } + converted, err := conversion.Convert(e.Object, conversionPaths, conversion.ToEmbeddedObject) + if err != nil { + return errors.Wrap(err, "failed to convert example to embedded object") + } + e.Object = converted + e.SetGroupVersionKind(k8sschema.GroupVersionKind{ + Group: e.GroupVersionKind().Group, + Version: resource.Version, + Kind: e.GetKind(), + }) + } + annotations := e.GetAnnotations() + annotations["meta.upbound.io/example-id"] = annotationValue + e.SetAnnotations(annotations) + } + } + convertedFileContent = "# SPDX-FileCopyrightText: 2024 The Crossplane Authors \n#\n# SPDX-License-Identifier: CC0-1.0\n\n" + for i, e := range examples { + var convertedData []byte + convertedData, err := yaml.Marshal(&e) //nolint:gosec + if err != nil { + return errors.Wrap(err, "failed to marshal example to yaml") + } + if i == len(examples)-1 { + convertedFileContent += string(convertedData) + } else { + convertedFileContent += string(convertedData) + "\n---\n\n" + } + } + dir := filepath.Dir(newPath) + + // Create all necessary directories if they do not exist + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + return errors.Wrap(err, "failed to create directory") + } + f, err := os.Create(newPath) //nolint:gosec + if err != nil { + return errors.Wrap(err, "failed to create file") + } + if _, err := f.WriteString(convertedFileContent); err != nil { + return errors.Wrap(err, "failed to write to file") + } + log.Printf("Converted: %s\n", path) + } + return nil + }) + if err != nil { + log.Printf("Error walking the path %q: %v\n", startPath, err) + } + return nil +} + +func prepareResourceRegistry(pc *Provider) map[string]*Resource { + reg := map[string]*Resource{} + for _, r := range pc.Resources { + reg[fmt.Sprintf("%s/%s.%s", r.Kind, r.ShortGroup, pc.RootGroup)] = r + } + return reg +} + +func decodeExamples(content string) ([]*unstructured.Unstructured, error) { + var manifests []*unstructured.Unstructured + decoder := kyaml.NewYAMLOrJSONDecoder(bytes.NewBufferString(content), 1024) + for { + u := &unstructured.Unstructured{} + if err := decoder.Decode(&u); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, errors.Wrap(err, "cannot decode manifest") + } + if u != nil { + manifests = append(manifests, u) + } + } + return manifests, nil +} From e3647026b877e53b7004c0f71c713fa1da64c957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergen=20Yal=C3=A7=C4=B1n?= Date: Tue, 14 May 2024 17:04:42 +0300 Subject: [PATCH 2/2] Address review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergen Yalçın --- .../conversion}/example_conversions.go | 101 ++++++++++++------ 1 file changed, 66 insertions(+), 35 deletions(-) rename pkg/{config => examples/conversion}/example_conversions.go (54%) diff --git a/pkg/config/example_conversions.go b/pkg/examples/conversion/example_conversions.go similarity index 54% rename from pkg/config/example_conversions.go rename to pkg/examples/conversion/example_conversions.go index 2e245c65..ad6196c5 100644 --- a/pkg/config/example_conversions.go +++ b/pkg/examples/conversion/example_conversions.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -package config +package conversion import ( "bytes" @@ -13,6 +13,7 @@ import ( "path/filepath" "strings" + "github.com/crossplane/upjet/pkg/config" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sschema "k8s.io/apimachinery/pkg/runtime/schema" @@ -22,19 +23,29 @@ import ( "github.com/crossplane/upjet/pkg/config/conversion" ) -func ConvertSingletonListToEmbeddedObject(pc *Provider, startPath string) error { //nolint:gocyclo +func ConvertSingletonListToEmbeddedObject(pc *config.Provider, startPath, licenseHeaderPath string) error { resourceRegistry := prepareResourceRegistry(pc) + + var license string + var lErr error + if licenseHeaderPath != "" { + license, lErr = getLicenseHeader(licenseHeaderPath) + if lErr != nil { + return errors.Wrap(lErr, "failed to get license header") + } + } + err := filepath.Walk(startPath, func(path string, info os.FileInfo, err error) error { if err != nil { - return errors.Wrap(err, "walk failed") + return errors.Wrapf(err, "walk failed: %s", startPath) } var convertedFileContent string if !info.IsDir() && strings.HasSuffix(info.Name(), ".yaml") { log.Printf("Converting: %s\n", path) - content, err := os.ReadFile(path) //nolint:gosec + content, err := os.ReadFile(filepath.Clean(path)) if err != nil { - return errors.Wrap(err, "failed to read file") + return errors.Wrapf(err, "failed to read the %s file", path) } examples, err := decodeExamples(string(content)) @@ -44,10 +55,11 @@ func ConvertSingletonListToEmbeddedObject(pc *Provider, startPath string) error rootResource := resourceRegistry[fmt.Sprintf("%s/%s", examples[0].GroupVersionKind().Kind, examples[0].GroupVersionKind().Group)] if rootResource == nil { + log.Printf("Warning: Skipping %s because the corresponding resource could not be found in the provider", path) return nil } - newPath := strings.Replace(path, examples[0].GroupVersionKind().Version, rootResource.Version, -1) //nolint:gocritic + newPath := strings.ReplaceAll(path, examples[0].GroupVersionKind().Version, rootResource.Version) if path == newPath { return nil } @@ -57,6 +69,9 @@ func ConvertSingletonListToEmbeddedObject(pc *Provider, startPath string) error conversionPaths := resource.CRDListConversionPaths() if conversionPaths != nil && e.GroupVersionKind().Version != resource.Version { for i, cp := range conversionPaths { + // Here, for the manifests to be converted, only `forProvider + // is converted, assuming the `initProvider` field is empty in the + // spec. conversionPaths[i] = "spec.forProvider." + cp } converted, err := conversion.Convert(e.Object, conversionPaths, conversion.ToEmbeddedObject) @@ -75,34 +90,10 @@ func ConvertSingletonListToEmbeddedObject(pc *Provider, startPath string) error e.SetAnnotations(annotations) } } - convertedFileContent = "# SPDX-FileCopyrightText: 2024 The Crossplane Authors \n#\n# SPDX-License-Identifier: CC0-1.0\n\n" - for i, e := range examples { - var convertedData []byte - convertedData, err := yaml.Marshal(&e) //nolint:gosec - if err != nil { - return errors.Wrap(err, "failed to marshal example to yaml") - } - if i == len(examples)-1 { - convertedFileContent += string(convertedData) - } else { - convertedFileContent += string(convertedData) + "\n---\n\n" - } + convertedFileContent = license + "\n\n" + if err := writeExampleContent(path, convertedFileContent, examples, newPath); err != nil { + return errors.Wrap(err, "failed to write example content") } - dir := filepath.Dir(newPath) - - // Create all necessary directories if they do not exist - err = os.MkdirAll(dir, os.ModePerm) - if err != nil { - return errors.Wrap(err, "failed to create directory") - } - f, err := os.Create(newPath) //nolint:gosec - if err != nil { - return errors.Wrap(err, "failed to create file") - } - if _, err := f.WriteString(convertedFileContent); err != nil { - return errors.Wrap(err, "failed to write to file") - } - log.Printf("Converted: %s\n", path) } return nil }) @@ -112,8 +103,48 @@ func ConvertSingletonListToEmbeddedObject(pc *Provider, startPath string) error return nil } -func prepareResourceRegistry(pc *Provider) map[string]*Resource { - reg := map[string]*Resource{} +func writeExampleContent(path string, convertedFileContent string, examples []*unstructured.Unstructured, newPath string) error { + for i, e := range examples { + var convertedData []byte + e := e + convertedData, err := yaml.Marshal(&e) + if err != nil { + return errors.Wrap(err, "failed to marshal example to yaml") + } + if i == len(examples)-1 { + convertedFileContent += string(convertedData) + } else { + convertedFileContent += string(convertedData) + "\n---\n\n" + } + } + dir := filepath.Dir(newPath) + + // Create all necessary directories if they do not exist + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return errors.Wrap(err, "failed to create directory") + } + f, err := os.Create(filepath.Clean(newPath)) + if err != nil { + return errors.Wrap(err, "failed to create file") + } + if _, err := f.WriteString(convertedFileContent); err != nil { + return errors.Wrap(err, "failed to write to file") + } + log.Printf("Converted: %s\n", path) + return nil +} + +func getLicenseHeader(licensePath string) (string, error) { + licenseData, err := os.ReadFile(licensePath) + if err != nil { + return "", errors.Wrapf(err, "failed to read license file: %s", licensePath) + } + + return string(licenseData), nil +} + +func prepareResourceRegistry(pc *config.Provider) map[string]*config.Resource { + reg := map[string]*config.Resource{} for _, r := range pc.Resources { reg[fmt.Sprintf("%s/%s.%s", r.Kind, r.ShortGroup, pc.RootGroup)] = r }