diff --git a/core/src/main/java/hudson/FilePath.java b/core/src/main/java/hudson/FilePath.java index b0cf56bf907d..b5131906e417 100644 --- a/core/src/main/java/hudson/FilePath.java +++ b/core/src/main/java/hudson/FilePath.java @@ -131,14 +131,14 @@ import jenkins.util.ContextResettingExecutorService; import jenkins.util.SystemProperties; import jenkins.util.VirtualFile; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.fileupload2.core.FileItem; import org.apache.commons.io.input.CountingInputStream; import org.apache.tools.ant.BuildException; import org.apache.tools.ant.DirectoryScanner; import org.apache.tools.ant.Project; import org.apache.tools.ant.types.FileSet; +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarInputStream; import org.apache.tools.zip.ZipEntry; import org.apache.tools.zip.ZipFile; import org.jenkinsci.remoting.RoleChecker; @@ -3079,14 +3079,13 @@ private static void readFromTar(String name, File baseDir, InputStream in) throw /** * Reads from a tar stream and stores obtained files to the base dir. - * Supports large files > 10 GB since 1.627 when this was migrated to use commons-compress. + * Supports large files > 10 GB since 1.627. */ private static void readFromTar(String name, File baseDir, InputStream in, Charset filenamesEncoding) throws IOException { - // TarInputStream t = new TarInputStream(in); - try (TarArchiveInputStream t = new TarArchiveInputStream(in, filenamesEncoding.name())) { - TarArchiveEntry te; - while ((te = t.getNextTarEntry()) != null) { + try (TarInputStream t = new TarInputStream(in, filenamesEncoding.name())) { + TarEntry te; + while ((te = t.getNextEntry()) != null) { File f = new File(baseDir, te.getName()); if (!f.toPath().normalize().startsWith(baseDir.toPath())) { throw new IOException( diff --git a/core/src/main/java/hudson/util/io/TarArchiver.java b/core/src/main/java/hudson/util/io/TarArchiver.java index 00b3d1b55d12..597b4ff47339 100644 --- a/core/src/main/java/hudson/util/io/TarArchiver.java +++ b/core/src/main/java/hudson/util/io/TarArchiver.java @@ -36,10 +36,10 @@ import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.attribute.BasicFileAttributes; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.archivers.tar.TarConstants; -import org.apache.commons.compress.utils.BoundedInputStream; +import org.apache.commons.io.input.BoundedInputStream; +import org.apache.tools.tar.TarConstants; +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarOutputStream; /** * {@link FileVisitor} that creates a tar archive. @@ -48,17 +48,17 @@ */ final class TarArchiver extends Archiver { private final byte[] buf = new byte[8192]; - private final TarArchiveOutputStream tar; + private final TarOutputStream tar; TarArchiver(OutputStream out, Charset filenamesEncoding) { - tar = new TarArchiveOutputStream(out, filenamesEncoding.name()); - tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR); - tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); + tar = new TarOutputStream(out, filenamesEncoding.name()); + tar.setBigNumberMode(TarOutputStream.BIGNUMBER_STAR); + tar.setLongFileMode(TarOutputStream.LONGFILE_GNU); } @Override public void visitSymlink(File link, String target, String relativePath) throws IOException { - TarArchiveEntry e = new TarArchiveEntry(relativePath, TarConstants.LF_SYMLINK); + TarEntry e = new TarEntry(relativePath, TarConstants.LF_SYMLINK); try { int mode = IOUtils.mode(link); if (mode != -1) { @@ -70,8 +70,8 @@ public void visitSymlink(File link, String target, String relativePath) throws I e.setLinkName(target); - tar.putArchiveEntry(e); - tar.closeArchiveEntry(); + tar.putNextEntry(e); + tar.closeEntry(); entriesWritten++; } @@ -88,7 +88,7 @@ public void visit(File file, String relativePath) throws IOException { BasicFileAttributes basicFileAttributes = Files.readAttributes(Util.fileToPath(file), BasicFileAttributes.class); if (basicFileAttributes.isDirectory()) relativePath += '/'; - TarArchiveEntry te = new TarArchiveEntry(relativePath); + TarEntry te = new TarEntry(relativePath); int mode = IOUtils.mode(file); if (mode != -1) te.setMode(mode); te.setModTime(basicFileAttributes.lastModifiedTime().toMillis()); @@ -98,7 +98,7 @@ public void visit(File file, String relativePath) throws IOException { size = basicFileAttributes.size(); te.setSize(size); } - tar.putArchiveEntry(te); + tar.putNextEntry(te); try { if (!basicFileAttributes.isDirectory()) { // ensure we don't write more bytes than the declared when we created the entry @@ -118,7 +118,7 @@ public void visit(File file, String relativePath) throws IOException { } } } finally { // always close the entry - tar.closeArchiveEntry(); + tar.closeEntry(); } entriesWritten++; } diff --git a/test/src/test/java/hudson/tasks/ArtifactArchiverTest.java b/test/src/test/java/hudson/tasks/ArtifactArchiverTest.java index 2157e89492d7..6ceca40aba88 100644 --- a/test/src/test/java/hudson/tasks/ArtifactArchiverTest.java +++ b/test/src/test/java/hudson/tasks/ArtifactArchiverTest.java @@ -26,6 +26,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -51,6 +52,8 @@ import hudson.slaves.DumbSlave; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.HttpURLConnection; import java.util.Collections; import java.util.Comparator; @@ -61,6 +64,7 @@ import org.hamcrest.Matchers; import org.jenkinsci.plugins.structs.describable.DescribableModel; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.jvnet.hudson.test.BuildWatcher; @@ -392,6 +396,59 @@ public void testDefaultExcludesOff() throws Exception { assertTrue(artifacts.child("dir").child(".svn").child("file").exists()); } + @Ignore("Test is too slow and requires a lot of disk space") + @Issue("JENKINS-10629") + @Test + public void testLargeArchiveFromAgent() throws Exception { + final String filename = "large"; + final long size = 10L * 1024L * 1024L * 1024L; // 10 GB + + Slave agent = j.createOnlineSlave(); + FreeStyleProject project = j.createFreeStyleProject(); + project.setAssignedNode(agent); + project.getBuildersList().add(new TestBuilder() { + @Override + public boolean perform(AbstractBuild build, Launcher launcher, BuildListener listener) throws InterruptedException, IOException { + FilePath filePath = build.getWorkspace().child(filename); + try (OutputStream os = filePath.write()) { + // Create byte array and fill it with data + byte[] megabyte = new byte[1024 * 1024]; + for (int i = 0; i < megabyte.length; i++) { + megabyte[i] = (byte) (i % 128); + } + // Fill file with 1 MB chunks + for (int i = 0; i < size / megabyte.length; i++) { + os.write(megabyte); + } + } + return true; + } + }); + project.getPublishersList().add(new ArtifactArchiver(filename)); + + // Assert that the build succeeded + FreeStyleBuild build = j.buildAndAssertSuccess(project); + VirtualFile virtualFile = build.getArtifactManager().root().child(filename); + + // Assert that the artifact was copied + assertTrue(virtualFile.exists()); + + // Assert that it has the right size + assertEquals(size, virtualFile.length()); + + // Assert that the data at the end of the file is the expected data + try (InputStream is = virtualFile.open()) { + is.skip(size - 1024 * 1024); + byte[] expected = new byte[1024 * 1024]; + for (int i = 0; i < expected.length; i++) { + expected[i] = (byte) (i % 128); + } + byte[] actual = new byte[1024 * 1024]; + is.read(actual); + assertArrayEquals(expected, actual); + } + } + @LocalData @Test public void latestOnlyMigration() throws Exception { FreeStyleProject p = j.jenkins.getItemByFullName("sample", FreeStyleProject.class);