diff --git a/.gitignore b/.gitignore index c58a5b6d00..d80906dc93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ out/ .idea +*.iml diff --git a/integration/benchmark_fs/Dockerfile_fs_benchmark b/integration/benchmark_fs/Dockerfile_fs_benchmark new file mode 100644 index 0000000000..4065467c4a --- /dev/null +++ b/integration/benchmark_fs/Dockerfile_fs_benchmark @@ -0,0 +1,8 @@ +FROM bash:4.4 + +ARG NUM +COPY context.txt . +COPY make.sh . +SHELL ["/usr/local/bin/bash", "-c"] +RUN ./make.sh $NUM +RUN ls -al /workdir | wc diff --git a/integration/benchmark_fs/context.txt b/integration/benchmark_fs/context.txt new file mode 100644 index 0000000000..3b18e512db --- /dev/null +++ b/integration/benchmark_fs/context.txt @@ -0,0 +1 @@ +hello world diff --git a/integration/benchmark_fs/make.sh b/integration/benchmark_fs/make.sh new file mode 100755 index 0000000000..828f420ba8 --- /dev/null +++ b/integration/benchmark_fs/make.sh @@ -0,0 +1,24 @@ +#!/usr/local/bin/bash + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir /workdir + +i=1 +while [ $i -le $1 ] +do + cat context.txt > /workdir/somefile$i + i=$(( $i + 1 )) +done diff --git a/integration/benchmark_test.go b/integration/benchmark_test.go new file mode 100644 index 0000000000..036e96beb1 --- /dev/null +++ b/integration/benchmark_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + "time" +) + +type result struct { + totalBuildTime float64 + resolvingFiles float64 + walkingFiles float64 +} + +func TestSnapshotBenchmark(t *testing.T) { + if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err != nil || !b { + t.SkipNow() + } + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + contextDir := filepath.Join(cwd, "benchmark_fs") + + nums := []int{10000, 50000, 100000, 200000, 300000, 500000, 700000, 800000} + + var timeMap sync.Map + var wg sync.WaitGroup + for _, num := range nums { + t.Run(fmt.Sprintf("test_benchmark_%d", num), func(t *testing.T) { + wg.Add(1) + var err error + go func(num int, err *error) { + dockerfile := "Dockerfile_fs_benchmark" + kanikoImage := fmt.Sprintf("%s_%d", GetKanikoImage(config.imageRepo, dockerfile), num) + buildArgs := []string{"--build-arg", fmt.Sprintf("NUM=%d", num)} + var benchmarkDir string + benchmarkDir, *err = buildKanikoImage("", dockerfile, + buildArgs, []string{}, kanikoImage, contextDir, config.gcsBucket, + config.serviceAccount, false) + if *err != nil { + return + } + r := newResult(t, filepath.Join(benchmarkDir, dockerfile)) + timeMap.Store(num, r) + wg.Done() + defer os.Remove(benchmarkDir) + }(num, &err) + if err != nil { + t.Errorf("could not run benchmark results for num %d due to %s", num, err) + } + }) + } + wg.Wait() + + fmt.Println("Number of Files,Total Build Time,Walking Filesystem, Resolving Files") + timeMap.Range(func(key interface{}, value interface{}) bool { + d, _ := key.(int) + v, _ := value.(result) + fmt.Println(fmt.Sprintf("%d,%f,%f,%f", d, v.totalBuildTime, v.walkingFiles, v.resolvingFiles)) + return true + }) + +} + +func newResult(t *testing.T, f string) result { + var current map[string]time.Duration + jsonFile, err := os.Open(f) + defer jsonFile.Close() + if err != nil { + t.Errorf("could not read benchmark file %s", f) + } + byteValue, _ := ioutil.ReadAll(jsonFile) + if err := json.Unmarshal(byteValue, ¤t); err != nil { + t.Errorf("could not unmarshal benchmark file") + } + r := result{} + if c, ok := current["Resolving Paths"]; ok { + r.resolvingFiles = c.Seconds() + } + if c, ok := current["Walking filesystem"]; ok { + r.walkingFiles = c.Seconds() + } + if c, ok := current["Total Build Time"]; ok { + r.totalBuildTime = c.Seconds() + } + return r +} diff --git a/integration/images.go b/integration/images.go index bb254f1102..8865e836aa 100644 --- a/integration/images.go +++ b/integration/images.go @@ -259,72 +259,22 @@ func (d *DockerFileBuilder) BuildImageWithContext(config *integrationTestConfig, } } - reproducibleFlag := "" + additionalKanikoFlags := additionalKanikoFlagsMap[dockerfile] + additionalKanikoFlags = append(additionalKanikoFlags, contextFlag, contextPath) for _, d := range reproducibleTests { if d == dockerfile { - reproducibleFlag = "--reproducible" + additionalKanikoFlags = append(additionalKanikoFlags, "--reproducible") break } } - benchmarkEnv := "BENCHMARK_FILE=false" - benchmarkDir, err := ioutil.TempDir("", "") - if err != nil { - return err - } - if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b { - benchmarkEnv = "BENCHMARK_FILE=/kaniko/benchmarks/" + dockerfile - benchmarkFile := path.Join(benchmarkDir, dockerfile) - fileName := fmt.Sprintf("run_%s_%s", time.Now().Format("2006-01-02-15:04"), dockerfile) - dst := path.Join("benchmarks", fileName) - defer UploadFileToBucket(gcsBucket, benchmarkFile, dst) - } - - // build kaniko image - additionalFlags := append(buildArgs, additionalKanikoFlagsMap[dockerfile]...) kanikoImage := GetKanikoImage(imageRepo, dockerfile) - fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) - - dockerRunFlags := []string{"run", "--net=host", - "-e", benchmarkEnv, - "-v", contextDir + ":/workspace", - "-v", benchmarkDir + ":/kaniko/benchmarks", - } - - if env, ok := envsMap[dockerfile]; ok { - for _, envVariable := range env { - dockerRunFlags = append(dockerRunFlags, "-e", envVariable) - } - } - - dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) - - kanikoDockerfilePath := path.Join(buildContextPath, dockerfilesPath, dockerfile) - if dockerfilesPath == "" { - kanikoDockerfilePath = path.Join(buildContextPath, "Dockerfile") - } - - dockerRunFlags = append(dockerRunFlags, ExecutorImage, - "-f", kanikoDockerfilePath, - "-d", kanikoImage, reproducibleFlag, - contextFlag, contextPath) - dockerRunFlags = append(dockerRunFlags, additionalFlags...) - - kanikoCmd := exec.Command("docker", dockerRunFlags...) - timer = timing.Start(dockerfile + "_kaniko") - out, err := RunCommandWithoutTest(kanikoCmd) - timing.DefaultRun.Stop(timer) - - if err != nil { - return fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) - } - - if outputCheck := outputChecks[dockerfile]; outputCheck != nil { - if err := outputCheck(dockerfile, out); err != nil { - return fmt.Errorf("Output check failed for image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) - } + if _, err := buildKanikoImage(dockerfilesPath, dockerfile, buildArgs, additionalKanikoFlags, kanikoImage, + contextDir, gcsBucket, serviceAccount, true); err != nil { + return err } + timing.DefaultRun.Stop(timer) d.filesBuilt[dockerfile] = struct{}{} @@ -381,9 +331,7 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac "--cache-dir", cacheDir) kanikoCmd := exec.Command("docker", dockerRunFlags...) - timer := timing.Start(dockerfile + "_kaniko_cached_" + strconv.Itoa(version)) _, err := RunCommandWithoutTest(kanikoCmd) - timing.DefaultRun.Stop(timer) if err != nil { return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err) } @@ -435,3 +383,71 @@ func (d *DockerFileBuilder) buildRelativePathsImage(imageRepo, dockerfile, servi return nil } + +func buildKanikoImage( + dockerfilesPath string, + dockerfile string, + buildArgs []string, + kanikoArgs []string, + kanikoImage string, + contextDir string, + gcsBucket string, + serviceAccount string, + shdUpload bool, +) (string, error) { + benchmarkEnv := "BENCHMARK_FILE=false" + benchmarkDir, err := ioutil.TempDir("", "") + if err != nil { + return "", err + } + if b, err := strconv.ParseBool(os.Getenv("BENCHMARK")); err == nil && b { + benchmarkEnv = "BENCHMARK_FILE=/kaniko/benchmarks/" + dockerfile + if shdUpload { + benchmarkFile := path.Join(benchmarkDir, dockerfile) + fileName := fmt.Sprintf("run_%s_%s", time.Now().Format("2006-01-02-15:04"), dockerfile) + dst := path.Join("benchmarks", fileName) + defer UploadFileToBucket(gcsBucket, benchmarkFile, dst) + } + } + + // build kaniko image + additionalFlags := append(buildArgs, kanikoArgs...) + fmt.Printf("Going to build image with kaniko: %s, flags: %s \n", kanikoImage, additionalFlags) + + dockerRunFlags := []string{"run", "--net=host", + "-e", benchmarkEnv, + "-v", contextDir + ":/workspace", + "-v", benchmarkDir + ":/kaniko/benchmarks", + } + + if env, ok := envsMap[dockerfile]; ok { + for _, envVariable := range env { + dockerRunFlags = append(dockerRunFlags, "-e", envVariable) + } + } + + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) + + kanikoDockerfilePath := path.Join(buildContextPath, dockerfilesPath, dockerfile) + if dockerfilesPath == "" { + kanikoDockerfilePath = path.Join(buildContextPath, "Dockerfile") + } + + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", kanikoDockerfilePath, + "-d", kanikoImage) + dockerRunFlags = append(dockerRunFlags, additionalFlags...) + + kanikoCmd := exec.Command("docker", dockerRunFlags...) + + out, err := RunCommandWithoutTest(kanikoCmd) + if err != nil { + return "", fmt.Errorf("Failed to build image %s with kaniko command \"%s\": %s %s", kanikoImage, kanikoCmd.Args, err, string(out)) + } + if outputCheck := outputChecks[dockerfile]; outputCheck != nil { + if err := outputCheck(dockerfile, out); err != nil { + return "", fmt.Errorf("Output check failed for image %s with kaniko command : %s %s", kanikoImage, err, string(out)) + } + } + return benchmarkDir, nil +} diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh index 3af24c922f..07ac75bdb4 100755 --- a/scripts/integration-test.sh +++ b/scripts/integration-test.sh @@ -21,19 +21,6 @@ IMAGE_REPO="${IMAGE_REPO:-gcr.io/kaniko-test}" docker version # Sets up a kokoro (Google internal integration testing tool) environment -if [ -f "$KOKORO_GFILE_DIR"/common.sh ]; then - echo "Installing dependencies..." - source "$KOKORO_GFILE_DIR/common.sh" - mkdir -p /usr/local/go/src/github.com/GoogleContainerTools/ - cp -r github/kaniko /usr/local/go/src/github.com/GoogleContainerTools/ - pushd /usr/local/go/src/github.com/GoogleContainerTools/kaniko - echo "Installing container-diff..." - mv $KOKORO_GFILE_DIR/container-diff-linux-amd64 $KOKORO_GFILE_DIR/container-diff - chmod +x $KOKORO_GFILE_DIR/container-diff - export PATH=$PATH:$KOKORO_GFILE_DIR - cp $KOKORO_ROOT/src/keystore/72508_gcr_application_creds $HOME/.config/gcloud/application_default_credentials.json -fi - echo "Running integration tests..." make out/executor make out/warmer