Remove tarball.WithCompressedCaching flag to resolve OOM Killed error (…

…#1722)

* Remove tarball.WithCompressedCaching flag to resolve OOM Killed error

Large images cannot be build as the kaniko container will be killed due to an OOM error. Removing the tarball compression drastically reduces the memory required to push large image layers. Fixes #1680

This change may increase the build time for smaller images. Therefore a command line option to trigger the compression or a more intelligent behaviour may be useful.

* Add new command line flag to toggle compressed caching

* Add unittest for build with --compressed-caching command line flag set to false

README.md

@@ -58,6 +58,7 @@ _If you are interested in contributing to kaniko, see [DEVELOPMENT.md](DEVELOPME
     - [--cache-ttl duration](#--cache-ttl-duration)
     - [--cleanup](#--cleanup)
     - [--context-sub-path](#--context-sub-path)
+    - [--compressed-caching](#--compressed-caching)
     - [--customPlatform](#--customPlatform)
     - [--digest-file](#--digest-file)
     - [--dockerfile](#--dockerfile)
@@ -613,6 +614,11 @@ Cache timeout in hours. Defaults to two weeks.
 
 Set this flag to clean the filesystem at the end of the build.
 
+#### --compressed-caching
+
+Set this to false in order to prevent tar compression for cached layers. This will increase the runtime of the build, but decrease the memory usage especially for large builds.
+Try to use `--compressed-caching=false` if your build fails with an out of memory error. Defaults to true.
+
 #### --context-sub-path
 
 Set a sub path within the given `--context`.

cmd/executor/cmd/root.go

@@ -202,6 +202,7 @@ func addKanikoOptionsFlags() {
 	RootCmd.PersistentFlags().StringVarP(&opts.ImageNameTagDigestFile, "image-name-tag-with-digest-file", "", "", "Specify a file to save the image name w/ image tag w/ digest of the built image to.")
 	RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image")
+	RootCmd.PersistentFlags().BoolVarP(&opts.CompressedCaching, "compressed-caching", "", true, "Compress the cached layers. Decreases build time, but increases memory usage.")
 	RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end")
 	RootCmd.PersistentFlags().DurationVarP(&opts.CacheTTL, "cache-ttl", "", time.Hour*336, "Cache timeout in hours. Defaults to two weeks.")
 	RootCmd.PersistentFlags().VarP(&opts.InsecureRegistries, "insecure-registry", "", "Insecure registry using plain HTTP to push and pull. Set it repeatedly for multiple registries.")

pkg/config/options.go

@@ -70,6 +70,7 @@ type KanikoOptions struct {
 	NoPush                 bool
 	Cache                  bool
 	Cleanup                bool
+	CompressedCaching      bool
 	IgnoreVarRun           bool
 	SkipUnusedStages       bool
 	RunV2                  bool

pkg/executor/build.go

@@ -477,7 +477,12 @@ func (s *stageBuilder) saveSnapshotToLayer(tarPath string) (v1.Layer, error) {
 		return nil, nil
 	}
 
-	layer, err := tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching)
+	var layer v1.Layer
+	if s.opts.CompressedCaching == true {
+		layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching)
+	} else {
+		layer, err = tarball.LayerFromFile(tarPath)
+	}
 	if err != nil {
 		return nil, err
 	}

pkg/executor/build_test.go

@@ -784,6 +784,42 @@ func Test_stageBuilder_build(t *testing.T) {
 				rootDir:           dir,
 			}
 		}(),
+		func() testcase {
+			dir, files := tempDirAndFile(t)
+			file := files[0]
+			filePath := filepath.Join(dir, file)
+			ch := NewCompositeCache("", "meow")
+
+			ch.AddPath(filePath, util.FileContext{})
+			hash, err := ch.Hash()
+			if err != nil {
+				t.Errorf("couldn't create hash %v", err)
+			}
+			command := MockDockerCommand{
+				command:      "meow",
+				contextFiles: []string{filePath},
+				cacheCommand: MockCachedDockerCommand{
+					contextFiles: []string{filePath},
+				},
+			}
+
+			destDir, err := ioutil.TempDir("", "baz")
+			if err != nil {
+				t.Errorf("could not create temp dir %v", err)
+			}
+			return testcase{
+				description: "fake command cache enabled with tar compression disabled and key in cache",
+				opts:        &config.KanikoOptions{Cache: true, CompressedCaching: false},
+				config:      &v1.ConfigFile{Config: v1.Config{WorkingDir: destDir}},
+				layerCache: &fakeLayerCache{
+					retrieve: true,
+				},
+				expectedCacheKeys: []string{hash},
+				pushedCacheKeys:   []string{},
+				commands:          []commands.DockerCommand{command},
+				rootDir:           dir,
+			}
+		}(),
 		{
 			description: "fake command cache disabled and key not in cache",
 			opts:        &config.KanikoOptions{Cache: false},

pkg/executor/push.go

@@ -327,7 +327,13 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error {
 // pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache
 // if opts.Cache doesn't exist, infer the cache from the given destination
 func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error {
-	layer, err := tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching)
+	var layer v1.Layer
+	var err error
+	if opts.CompressedCaching == true {
+		layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching)
+	} else {
+		layer, err = tarball.LayerFromFile(tarPath)
+	}
 	if err != nil {
 		return err
 	}