Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: verify storage class exists on init #2180

Merged
merged 35 commits into from
Jan 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
a556978
Add onDeploy before action to verify storage class exists for git ser…
Dec 6, 2023
e820793
Remove redundant error message when init component fails to deploy
Dec 8, 2023
1af1114
Add test function to validate zarf init fails with no storage class
Dec 9, 2023
9d3a6ec
Skip initWithoutStorageClass() in appliance mode
Dec 9, 2023
19a6ce3
Get storageclass name dynamically rather than based on distro
Dec 9, 2023
dd8a398
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Dec 12, 2023
8537426
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Dec 18, 2023
ef09390
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Dec 18, 2023
ece1927
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Dec 18, 2023
5a8e2a7
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Dec 18, 2023
ca86c51
Use wait syntax to check for storage class in cluster
Dec 19, 2023
08ac22b
Update packages/gitea/zarf.yaml
lucasrod16 Dec 19, 2023
cfd7d8a
Update packages/zarf-registry/zarf.yaml
lucasrod16 Dec 19, 2023
5b7fd0c
Ensure state values are populated before running onDeploy before actions
Dec 19, 2023
a1df190
Escape double quotes in ZARF_STORAGE_CLASS
Dec 19, 2023
aed2e0a
Check if cluster is nil before loading Zarf state
Dec 19, 2023
4a58407
Only run mismatched init test on linux
Dec 19, 2023
d95c51f
Check for more complete error message
Dec 19, 2023
3871c52
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Racer159 Dec 19, 2023
2a9b14f
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Racer159 Dec 19, 2023
5a9ffab
Add distro detection for minikube
Dec 20, 2023
3a894f8
Remove minikube from distro detection logic
Dec 20, 2023
71afc27
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Dec 20, 2023
cddea13
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Jan 3, 2024
ab631f0
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Jan 3, 2024
cef2729
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Jan 3, 2024
150be5b
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Jan 3, 2024
fb00d96
Allow resource name to be optional in zarf tools wait-for
Jan 3, 2024
3cf1f46
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Jan 3, 2024
69f7ceb
Fix wait-for args check
Jan 3, 2024
b1a833b
Do not return an error when identifier is provided
Jan 3, 2024
69c0098
Use filepath.ToSlash to ensure a forward slash appears on windows
Jan 4, 2024
421aadd
Merge branch 'main' into 1824-verify-storageclass-exists-on-init
Jan 8, 2024
16cd18b
Update src/pkg/utils/wait.go
lucasrod16 Jan 8, 2024
5aad4bf
Log a debug message when no resources are found
Jan 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions packages/gitea/zarf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ components:
actions:
onDeploy:
before:
- description: Check that the cluster has the specified storage class
maxTotalSeconds: 3
wait:
cluster:
kind: storageclass
name: "\"${ZARF_STORAGE_CLASS}\""
- cmd: ./zarf internal update-gitea-pvc --no-progress
setVariables:
- name: GIT_SERVER_CREATE_PVC
Expand Down
9 changes: 9 additions & 0 deletions packages/zarf-registry/zarf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,15 @@ components:
images:
# The seed image (or images) that will be injected (see zarf-config.toml)
- "###ZARF_PKG_TMPL_REGISTRY_IMAGE_DOMAIN######ZARF_PKG_TMPL_REGISTRY_IMAGE###:###ZARF_PKG_TMPL_REGISTRY_IMAGE_TAG###"
actions:
onDeploy:
before:
- description: Check that the cluster has the specified storage class
maxTotalSeconds: 3
wait:
cluster:
kind: storageclass
name: "\"${ZARF_STORAGE_CLASS}\""

- name: zarf-registry
description: |
Expand Down
15 changes: 11 additions & 4 deletions src/cmd/tools/wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,21 @@ var waitForCmd = &cobra.Command{
Short: lang.CmdToolsWaitForShort,
Long: lang.CmdToolsWaitForLong,
Example: lang.CmdToolsWaitForExample,
Args: cobra.MinimumNArgs(2),
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
// Parse the timeout string
timeout, err := time.ParseDuration(waitTimeout)
if err != nil {
message.Fatalf(err, lang.CmdToolsWaitForErrTimeoutString, waitTimeout)
}

// Parse the kind type and identifier.
kind, identifier := args[0], args[1]
kind := args[0]

// identifier is optional to allow for commands like `zarf tools wait-for storageclass` without specifying a name.
identifier := ""
if len(args) > 1 {
identifier = args[1]
}

// Condition is optional, default to "exists".
condition := ""
Expand All @@ -45,7 +50,9 @@ var waitForCmd = &cobra.Command{
}

// Execute the wait command.
utils.ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier, timeout)
if err := utils.ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier, timeout); err != nil {
message.Fatal(err, err.Error())
}
},
}

Expand Down
36 changes: 19 additions & 17 deletions src/pkg/packager/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ func (p *Packager) deployInitComponent(component types.ZarfComponent) (charts []

charts, err = p.deployComponent(component, isAgent /* skip img checksum if isAgent */, isSeedRegistry /* skip image push if isSeedRegistry */)
if err != nil {
return charts, fmt.Errorf("unable to deploy component %q: %w", component.Name, err)
return charts, err
}

// Do cleanup for when we inject the seed registry during initialization
Expand Down Expand Up @@ -255,16 +255,6 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum

onDeploy := component.Actions.OnDeploy

if err = p.runActions(onDeploy.Defaults, onDeploy.Before, p.valueTemplate); err != nil {
return charts, fmt.Errorf("unable to run component before action: %w", err)
}

if hasFiles {
if err := p.processComponentFiles(component, componentPath.Files); err != nil {
return charts, fmt.Errorf("unable to process the component files: %w", err)
}
}

if !p.valueTemplate.Ready() && requiresCluster(component) {
// Setup the state in the config and get the valuesTemplate
p.valueTemplate, err = p.setupStateValuesTemplate()
Expand All @@ -282,6 +272,16 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum
}
}

if err = p.runActions(onDeploy.Defaults, onDeploy.Before, p.valueTemplate); err != nil {
return charts, fmt.Errorf("unable to run component before action: %w", err)
}

if hasFiles {
if err := p.processComponentFiles(component, componentPath.Files); err != nil {
return charts, fmt.Errorf("unable to process the component files: %w", err)
}
}

if hasImages {
if err := p.pushImagesToRegistry(component.Images, noImgChecksum); err != nil {
return charts, fmt.Errorf("unable to push images to the registry: %w", err)
Expand Down Expand Up @@ -620,12 +620,14 @@ func (p *Packager) printTablesForDeployment(componentsToDeploy []types.DeployedC
if !p.isInitConfig() {
message.PrintConnectStringTable(p.connectStrings)
} else {
// Grab a fresh copy of the state (if we are able) to print the most up-to-date version of the creds
freshState, err := p.cluster.LoadZarfState()
if err != nil {
freshState = p.cfg.State
if p.cluster != nil {
// Grab a fresh copy of the state (if we are able) to print the most up-to-date version of the creds
freshState, err := p.cluster.LoadZarfState()
if err != nil {
freshState = p.cfg.State
}
// otherwise, print the init config connection and passwords
message.PrintCredentialTable(freshState, componentsToDeploy)
}
// otherwise, print the init config connection and passwords
message.PrintCredentialTable(freshState, componentsToDeploy)
}
}
23 changes: 16 additions & 7 deletions src/pkg/utils/wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"fmt"
"net"
"net/http"
"path"
"strconv"
"strings"
"time"
Expand All @@ -29,12 +30,12 @@ func isJSONPathWaitType(condition string) bool {
}

// ExecuteWait executes the wait-for command.
func ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier string, timeout time.Duration) {
func ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier string, timeout time.Duration) error {
// Handle network endpoints.
switch kind {
case "http", "https", "tcp":
waitForNetworkEndpoint(kind, identifier, condition, timeout)
return
return nil
}

// Type of wait, condition or JSONPath
Expand All @@ -53,8 +54,9 @@ func ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier string,
message.Fatal(err, lang.CmdToolsWaitForErrZarfPath)
}

identifierMsg := identifier

// If the identifier contains an equals sign, convert to a label selector.
identifierMsg := fmt.Sprintf("/%s", identifier)
if strings.ContainsRune(identifier, '=') {
identifierMsg = fmt.Sprintf(" with label `%s`", identifier)
identifier = fmt.Sprintf("-l %s", identifier)
Expand All @@ -73,7 +75,7 @@ func ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier string,

// Setup the spinner messages.
conditionMsg := fmt.Sprintf("Waiting for %s%s%s to be %s.", kind, identifierMsg, namespaceMsg, condition)
existMsg := fmt.Sprintf("Waiting for %s%s%s to exist.", kind, identifierMsg, namespaceMsg)
existMsg := fmt.Sprintf("Waiting for %s%s to exist.", path.Join(kind, identifierMsg), namespaceMsg)
spinner := message.NewProgressSpinner(existMsg)

// Get the OS shell to execute commands in
Expand All @@ -93,7 +95,14 @@ func ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier string,
spinner.Updatef(existMsg)
// Check if the resource exists.
zarfKubectlGet := fmt.Sprintf("%s tools kubectl get %s %s %s", zarfCommand, namespaceFlag, kind, identifier)
if stdout, stderr, err := exec.Cmd(shell, append(shellArgs, zarfKubectlGet)...); err != nil {
stdout, stderr, err := exec.Cmd(shell, append(shellArgs, zarfKubectlGet)...)
if err != nil {
message.Debug(stdout, stderr, err)
continue
}

resourceNotFound := strings.Contains(stderr, "No resources found") && identifier == ""
if resourceNotFound {
message.Debug(stdout, stderr, err)
continue
}
Expand All @@ -102,7 +111,7 @@ func ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier string,
switch condition {
case "", "exist", "exists":
spinner.Success()
return
return nil
}

spinner.Updatef(conditionMsg)
Expand All @@ -118,7 +127,7 @@ func ExecuteWait(waitTimeout, waitNamespace, condition, kind, identifier string,

// And just like that, success!
spinner.Successf(conditionMsg)
return
return nil
}
}
}
Expand Down
12 changes: 7 additions & 5 deletions src/test/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,17 @@ var logRegex = regexp.MustCompile(`Saving log file to (?P<logFile>.*?\.log)`)
// GetCLIName looks at the OS and CPU architecture to determine which Zarf binary needs to be run.
func GetCLIName() string {
var binaryName string
if runtime.GOOS == "linux" {
switch runtime.GOOS {
case "linux":
binaryName = "zarf"
} else if runtime.GOOS == "darwin" {
if runtime.GOARCH == "arm64" {
case "darwin":
switch runtime.GOARCH {
case "arm64":
binaryName = "zarf-mac-apple"
} else {
default:
binaryName = "zarf-mac-intel"
}
} else if runtime.GOOS == "windows" {
case "windows":
if runtime.GOARCH == "amd64" {
binaryName = "zarf.exe"
}
Expand Down
74 changes: 57 additions & 17 deletions src/test/e2e/20_zarf_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ package test
import (
"encoding/base64"
"fmt"
"runtime"
"testing"

"encoding/json"

"github.com/defenseunicorns/zarf/src/pkg/utils"
"github.com/defenseunicorns/zarf/src/types"
"github.com/stretchr/testify/require"
)
Expand All @@ -30,32 +32,29 @@ func TestZarfInit(t *testing.T) {
var (
mismatchedArch = e2e.GetMismatchedArch()
mismatchedInitPackage = fmt.Sprintf("zarf-init-%s-%s.tar.zst", mismatchedArch, initPackageVersion)
expectedErrorMessage = fmt.Sprintf("this package architecture is %s", mismatchedArch)
expectedErrorMessage = "unable to run component before action: command \"Check that the host architecture matches the package architecture\""
)
t.Cleanup(func() {
e2e.CleanFiles(mismatchedInitPackage)
})

// Build init package with different arch than the cluster arch.
stdOut, stdErr, err := e2e.Zarf("package", "create", "src/test/packages/20-mismatched-arch-init", "--architecture", mismatchedArch, "--confirm")
require.NoError(t, err, stdOut, stdErr)
if runtime.GOOS == "linux" {
// Build init package with different arch than the cluster arch.
stdOut, stdErr, err := e2e.Zarf("package", "create", "src/test/packages/20-mismatched-arch-init", "--architecture", mismatchedArch, "--confirm")
require.NoError(t, err, stdOut, stdErr)

componentsFlag := ""
if e2e.ApplianceMode {
// make sure init fails in appliance mode when we try to initialize a k3s cluster
// with behavior from the k3s component's actions
componentsFlag = "--components=k3s"
// Check that `zarf init` returns an error because of the mismatched architectures.
// We need to use the --architecture flag here to force zarf to find the package.
_, stdErr, err = e2e.Zarf("init", "--architecture", mismatchedArch, "--components=k3s", "--confirm")
require.Error(t, err, stdErr)
require.Contains(t, stdErr, expectedErrorMessage)
lucasrod16 marked this conversation as resolved.
Show resolved Hide resolved
}

// Check that `zarf init` returns an error because of the mismatched architectures.
// We need to use the --architecture flag here to force zarf to find the package.
_, stdErr, err = e2e.Zarf("init", "--architecture", mismatchedArch, componentsFlag, "--confirm")
require.Error(t, err, stdErr)
require.Contains(t, stdErr, expectedErrorMessage)
initWithoutStorageClass(t)

if !e2e.ApplianceMode {
// throw a pending pod into the cluster to ensure we can properly ignore them when selecting images
_, _, err = e2e.Kubectl("apply", "-f", "https://github.com/kubernetes/website/main/content/en/examples/pods/pod-with-node-affinity.yaml")
_, _, err := e2e.Kubectl("apply", "-f", "https://github.com/kubernetes/website/main/content/en/examples/pods/pod-with-node-affinity.yaml")
require.NoError(t, err)
}

Expand All @@ -65,7 +64,8 @@ func TestZarfInit(t *testing.T) {
if err == nil {
oldStateJSON, err := base64.StdEncoding.DecodeString(base64State)
require.NoError(t, err)
json.Unmarshal(oldStateJSON, &oldState)
err = json.Unmarshal(oldStateJSON, &oldState)
require.NoError(t, err)
}

// run `zarf init`
Expand Down Expand Up @@ -98,7 +98,7 @@ func TestZarfInit(t *testing.T) {
}

// Check that the registry is running on the correct NodePort
stdOut, _, err = e2e.Kubectl("get", "service", "-n", "zarf", "zarf-docker-registry", "-o=jsonpath='{.spec.ports[*].nodePort}'")
stdOut, _, err := e2e.Kubectl("get", "service", "-n", "zarf", "zarf-docker-registry", "-o=jsonpath='{.spec.ports[*].nodePort}'")
require.NoError(t, err)
require.Contains(t, stdOut, "31337")

Expand Down Expand Up @@ -127,3 +127,43 @@ func checkLogForSensitiveState(t *testing.T, logText string, zarfState types.Zar
require.NotContains(t, logText, zarfState.RegistryInfo.Secret)
require.NotContains(t, logText, zarfState.LoggingSecret)
}

// Verify `zarf init` produces an error when there is no storage class in cluster.
func initWithoutStorageClass(t *testing.T) {
/*
Exit early if testing with Zarf-deployed k3s cluster.
This is a chicken-egg problem because we can't interact with a cluster that Zarf hasn't created yet.
Zarf deploys k3s with the Rancher local-path storage class out of the box,
so we don't expect any problems with no storage class in this case.
*/
if e2e.ApplianceMode {
return
}

jsonPathQuery := `{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}{end}`
defaultStorageClassName, _, err := e2e.Kubectl("get", "storageclass", "-o=jsonpath="+jsonPathQuery)
require.NoError(t, err)
require.NotEmpty(t, defaultStorageClassName)

storageClassYaml, _, err := e2e.Kubectl("get", "storageclass", defaultStorageClassName, "-o=yaml")
require.NoError(t, err)

storageClassFileName := "storage-class.yaml"

err = utils.WriteFile(storageClassFileName, []byte(storageClassYaml))
require.NoError(t, err)
defer e2e.CleanFiles(storageClassFileName)

_, _, err = e2e.Kubectl("delete", "storageclass", defaultStorageClassName)
require.NoError(t, err)

_, stdErr, err := e2e.Zarf("init", "--confirm")
require.Error(t, err, stdErr)
require.Contains(t, stdErr, "unable to run component before action: command \"Check that the cluster has the specified storage class\"")

_, _, err = e2e.Zarf("destroy", "--confirm")
require.NoError(t, err)

_, _, err = e2e.Kubectl("apply", "-f", storageClassFileName)
require.NoError(t, err)
}
6 changes: 4 additions & 2 deletions src/test/e2e/22_git_and_gitops_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ func testGitServerReadOnly(t *testing.T, gitURL string) {

// Make sure the only permissions are pull (read)
var bodyMap map[string]interface{}
json.Unmarshal(getRepoResponseBody, &bodyMap)
err = json.Unmarshal(getRepoResponseBody, &bodyMap)
require.NoError(t, err)
permissionsMap := bodyMap["permissions"].(map[string]interface{})
require.False(t, permissionsMap["admin"].(bool))
require.False(t, permissionsMap["push"].(bool))
Expand All @@ -105,7 +106,8 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) {

// Make sure the pushed tag exists
var tagMap map[string]interface{}
json.Unmarshal(getRepoTagsResponseBody, &tagMap)
err = json.Unmarshal(getRepoTagsResponseBody, &tagMap)
require.NoError(t, err)
require.Equal(t, repoTag, tagMap["name"])

// Get the Zarf repo commit
Expand Down
5 changes: 4 additions & 1 deletion src/test/e2e/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ const (
// TestMain lets us customize the test run. See https://medium.com/goingogo/why-use-testmain-for-testing-in-go-dafb52b406bc.
func TestMain(m *testing.M) {
// Work from the root directory of the project
os.Chdir("../../../")
err := os.Chdir("../../../")
if err != nil {
fmt.Println(err) //nolint:forbidigo
}

// K3d use the intern package, which requires this to be set in go 1.19
os.Setenv("ASSUME_NO_MOVING_GC_UNSAFE_RISK_IT_WITH", "go1.19")
Expand Down
Loading