Skip to content

Commit

Permalink
[JENKINS-10629] Migrate from Commons Compress to Ant (#9311)
Browse files Browse the repository at this point in the history
  • Loading branch information
@basil
basil committed May 27, 2024
1 parent dae1737 commit c10cafc
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 21 deletions.
13 changes: 6 additions & 7 deletions core/src/main/java/hudson/FilePath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,14 +131,14 @@
import jenkins.util.ContextResettingExecutorService;
import jenkins.util.SystemProperties;
import jenkins.util.VirtualFile;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.fileupload2.core.FileItem;
import org.apache.commons.io.input.CountingInputStream;
import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.DirectoryScanner;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.types.FileSet;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;
import org.apache.tools.zip.ZipEntry;
import org.apache.tools.zip.ZipFile;
import org.jenkinsci.remoting.RoleChecker;
Expand Down Expand Up @@ -3079,14 +3079,13 @@ private static void readFromTar(String name, File baseDir, InputStream in) throw

/**
* Reads from a tar stream and stores obtained files to the base dir.
* Supports large files > 10 GB since 1.627 when this was migrated to use commons-compress.
* Supports large files > 10 GB since 1.627.
*/
private static void readFromTar(String name, File baseDir, InputStream in, Charset filenamesEncoding) throws IOException {

// TarInputStream t = new TarInputStream(in);
try (TarArchiveInputStream t = new TarArchiveInputStream(in, filenamesEncoding.name())) {
TarArchiveEntry te;
while ((te = t.getNextTarEntry()) != null) {
try (TarInputStream t = new TarInputStream(in, filenamesEncoding.name())) {
TarEntry te;
while ((te = t.getNextEntry()) != null) {
File f = new File(baseDir, te.getName());
if (!f.toPath().normalize().startsWith(baseDir.toPath())) {
throw new IOException(
Expand Down
28 changes: 14 additions & 14 deletions core/src/main/java/hudson/util/io/TarArchiver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.attribute.BasicFileAttributes;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.tar.TarConstants;
import org.apache.commons.compress.utils.BoundedInputStream;
import org.apache.commons.io.input.BoundedInputStream;
import org.apache.tools.tar.TarConstants;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarOutputStream;

/**
* {@link FileVisitor} that creates a tar archive.
Expand All @@ -48,17 +48,17 @@
*/
final class TarArchiver extends Archiver {
private final byte[] buf = new byte[8192];
private final TarArchiveOutputStream tar;
private final TarOutputStream tar;

TarArchiver(OutputStream out, Charset filenamesEncoding) {
tar = new TarArchiveOutputStream(out, filenamesEncoding.name());
tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
tar = new TarOutputStream(out, filenamesEncoding.name());
tar.setBigNumberMode(TarOutputStream.BIGNUMBER_STAR);
tar.setLongFileMode(TarOutputStream.LONGFILE_GNU);
}

@Override
public void visitSymlink(File link, String target, String relativePath) throws IOException {
TarArchiveEntry e = new TarArchiveEntry(relativePath, TarConstants.LF_SYMLINK);
TarEntry e = new TarEntry(relativePath, TarConstants.LF_SYMLINK);
try {
int mode = IOUtils.mode(link);
if (mode != -1) {
Expand All @@ -70,8 +70,8 @@ public void visitSymlink(File link, String target, String relativePath) throws I

e.setLinkName(target);

tar.putArchiveEntry(e);
tar.closeArchiveEntry();
tar.putNextEntry(e);
tar.closeEntry();
entriesWritten++;
}

Expand All @@ -88,7 +88,7 @@ public void visit(File file, String relativePath) throws IOException {
BasicFileAttributes basicFileAttributes = Files.readAttributes(Util.fileToPath(file), BasicFileAttributes.class);
if (basicFileAttributes.isDirectory())
relativePath += '/';
TarArchiveEntry te = new TarArchiveEntry(relativePath);
TarEntry te = new TarEntry(relativePath);
int mode = IOUtils.mode(file);
if (mode != -1) te.setMode(mode);
te.setModTime(basicFileAttributes.lastModifiedTime().toMillis());
Expand All @@ -98,7 +98,7 @@ public void visit(File file, String relativePath) throws IOException {
size = basicFileAttributes.size();
te.setSize(size);
}
tar.putArchiveEntry(te);
tar.putNextEntry(te);
try {
if (!basicFileAttributes.isDirectory()) {
// ensure we don't write more bytes than the declared when we created the entry
Expand All @@ -118,7 +118,7 @@ public void visit(File file, String relativePath) throws IOException {
}
}
} finally { // always close the entry
tar.closeArchiveEntry();
tar.closeEntry();
}
entriesWritten++;
}
Expand Down
57 changes: 57 additions & 0 deletions test/src/test/java/hudson/tasks/ArtifactArchiverTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.lessThan;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
Expand All @@ -51,6 +52,8 @@
import hudson.slaves.DumbSlave;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.util.Collections;
import java.util.Comparator;
Expand All @@ -61,6 +64,7 @@
import org.hamcrest.Matchers;
import org.jenkinsci.plugins.structs.describable.DescribableModel;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.jvnet.hudson.test.BuildWatcher;
Expand Down Expand Up @@ -392,6 +396,59 @@ public void testDefaultExcludesOff() throws Exception {
assertTrue(artifacts.child("dir").child(".svn").child("file").exists());
}

@Ignore("Test is too slow and requires a lot of disk space")
@Issue("JENKINS-10629")
@Test
public void testLargeArchiveFromAgent() throws Exception {
final String filename = "large";
final long size = 10L * 1024L * 1024L * 1024L; // 10 GB

Slave agent = j.createOnlineSlave();
FreeStyleProject project = j.createFreeStyleProject();
project.setAssignedNode(agent);
project.getBuildersList().add(new TestBuilder() {
@Override
public boolean perform(AbstractBuild<?, ?> build, Launcher launcher, BuildListener listener) throws InterruptedException, IOException {
FilePath filePath = build.getWorkspace().child(filename);
try (OutputStream os = filePath.write()) {
// Create byte array and fill it with data
byte[] megabyte = new byte[1024 * 1024];
for (int i = 0; i < megabyte.length; i++) {
megabyte[i] = (byte) (i % 128);
}
// Fill file with 1 MB chunks
for (int i = 0; i < size / megabyte.length; i++) {
os.write(megabyte);
}
}
return true;
}
});
project.getPublishersList().add(new ArtifactArchiver(filename));

// Assert that the build succeeded
FreeStyleBuild build = j.buildAndAssertSuccess(project);
VirtualFile virtualFile = build.getArtifactManager().root().child(filename);

// Assert that the artifact was copied
assertTrue(virtualFile.exists());

// Assert that it has the right size
assertEquals(size, virtualFile.length());

// Assert that the data at the end of the file is the expected data
try (InputStream is = virtualFile.open()) {
is.skip(size - 1024 * 1024);
byte[] expected = new byte[1024 * 1024];
for (int i = 0; i < expected.length; i++) {
expected[i] = (byte) (i % 128);
}
byte[] actual = new byte[1024 * 1024];
is.read(actual);
assertArrayEquals(expected, actual);
}
}

@LocalData
@Test public void latestOnlyMigration() throws Exception {
FreeStyleProject p = j.jenkins.getItemByFullName("sample", FreeStyleProject.class);
Expand Down

0 comments on commit c10cafc

Please sign in to comment.