diff --git a/src/main/java/rife/bld/dependencies/DependencySet.java b/src/main/java/rife/bld/dependencies/DependencySet.java index 47749b2..a654b6e 100644 --- a/src/main/java/rife/bld/dependencies/DependencySet.java +++ b/src/main/java/rife/bld/dependencies/DependencySet.java @@ -8,7 +8,6 @@ import rife.bld.dependencies.exceptions.DependencyTransferException; import java.io.File; import java.util.*; -import java.util.function.Supplier; /** * Convenience class to handle a set of {@link Dependency} objects. @@ -146,56 +145,9 @@ public class DependencySet extends AbstractSet implements Set transferIntoDirectory(VersionResolution resolution, ArtifactRetriever retriever, List repositories, File directory, File modulesDirectory, String... classifiers) { - var transfers = new ArrayList>>(); - for (var dependency : this) { - var transfer_directory = directory; - if (dependency.isModularJar()) { - if (modulesDirectory == null) { - throw new DependencyTransferException(dependency, "modules directory is not provided"); - } - transfer_directory = modulesDirectory; - } - else if (directory == null) { - throw new DependencyTransferException(dependency, "artifacts directory is not provided"); - } - - if (!transfer_directory.exists()) { - if (!transfer_directory.mkdirs()) { - throw new DependencyTransferException(dependency, transfer_directory, "couldn't create directory"); - } - } - - final var target_directory = transfer_directory; - transfers.add(() -> { - var artifacts = new ArrayList(); - var artifact = new DependencyResolver(resolution, retriever, repositories, dependency).transferIntoDirectory(target_directory); - if (artifact != null) { - artifacts.add(artifact); - } - - if (classifiers != null) { - for (var classifier : classifiers) { - if (classifier != null && !dependency.excludedClassifiers().contains(classifier)) { - var classifier_artifact = new DependencyResolver(resolution, retriever, repositories, dependency.withClassifier(classifier)).transferIntoDirectory(target_directory); - if (classifier_artifact != null) { - artifacts.add(classifier_artifact); - } - } - } - } - return artifacts; - }); - } - - return executeTransfers(transfers, resolution.transferParallelism()); - } - - private static List executeTransfers(List>> transfers, int transferParallelism) { - var result = new ArrayList(); - for (var artifacts : ParallelExecution.execute(transfers, transferParallelism)) { - result.addAll(artifacts); - } - return result; + return new DependencyTransferBatch() + .add(this, directory, modulesDirectory, classifiers) + .transfer(resolution, retriever, repositories); } /** diff --git a/src/main/java/rife/bld/dependencies/DependencyTransferBatch.java b/src/main/java/rife/bld/dependencies/DependencyTransferBatch.java new file mode 100644 index 0000000..7166a0d --- /dev/null +++ b/src/main/java/rife/bld/dependencies/DependencyTransferBatch.java @@ -0,0 +1,129 @@ +/* + * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com) + * Licensed under the Apache License, Version 2.0 (the "License") + */ +package rife.bld.dependencies; + +import rife.bld.dependencies.exceptions.DependencyTransferException; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; + +/** + * Collects the artifact transfers of one or more dependency sets so that + * they can be transferred together in a single parallel batch, instead of + * separate consecutive batches per dependency set. + *

+ * The batch itself is a passive collector, the resolution context is only + * provided when the transfers are {@linkplain #transfer performed}. The + * parallelism is determined by {@link VersionResolution#transferParallelism()}, + * setting it to {@code 1} makes the transfers sequential. Identical + * transfers into the same directory are only performed once. + * + * @author Geert Bevin (gbevin[remove] at uwyn dot com) + * @since 2.3.1 + */ +public class DependencyTransferBatch { + private record TransferRequest(Dependency dependency, File directory, String[] classifiers) { + } + + private final List requests_ = new ArrayList<>(); + private final Set transferTargets_ = new HashSet<>(); + + /** + * Adds the artifact transfers for a dependency set to this batch. + *

+ * The destination directory must exist and be writable. + * + * @param dependencies the dependencies whose artifacts to transfer + * @param directory the directory to transfer the artifacts into + * @param modulesDirectory the directory to download the modules into + * @param classifiers the additional classifiers to transfer + * @return this batch instance + * @throws DependencyTransferException when the transfer couldn't be prepared + * @since 2.3.1 + */ + public DependencyTransferBatch add(DependencySet dependencies, File directory, File modulesDirectory, String... classifiers) { + for (var dependency : dependencies) { + var transfer_directory = directory; + if (dependency.isModularJar()) { + if (modulesDirectory == null) { + throw new DependencyTransferException(dependency, "modules directory is not provided"); + } + transfer_directory = modulesDirectory; + } + else if (directory == null) { + throw new DependencyTransferException(dependency, "artifacts directory is not provided"); + } + + if (!transfer_directory.exists()) { + if (!transfer_directory.mkdirs()) { + throw new DependencyTransferException(dependency, transfer_directory, "couldn't create directory"); + } + } + + // skip transfers that are already batched for the same directory + if (!transferTargets_.add(dependency + " -> " + transfer_directory.getAbsolutePath())) { + continue; + } + + requests_.add(new TransferRequest(dependency, transfer_directory, classifiers)); + } + return this; + } + + /** + * Performs all the collected artifact transfers in a single parallel + * batch, in the order they were added. + *

+ * This empties the batch, transfers can be collected and transferred + * again with the same instance. + * + * @param resolution the version resolution state that can be cached + * @param retriever the retriever to use to get artifacts + * @param repositories the repositories to use for the transfer + * @return the list of artifacts that were transferred successfully + * @throws DependencyTransferException when an error occurred during the transfer + * @since 2.3.1 + */ + public List transfer(VersionResolution resolution, ArtifactRetriever retriever, List repositories) { + final var repos = (repositories == null ? List.of() : repositories); + try { + var transfers = new ArrayList>>(requests_.size()); + for (var request : requests_) { + transfers.add(() -> { + var artifacts = new ArrayList(); + var artifact = new DependencyResolver(resolution, retriever, repos, request.dependency()).transferIntoDirectory(request.directory()); + if (artifact != null) { + artifacts.add(artifact); + } + + if (request.classifiers() != null) { + for (var classifier : request.classifiers()) { + if (classifier != null && !request.dependency().excludedClassifiers().contains(classifier)) { + var classifier_artifact = new DependencyResolver(resolution, retriever, repos, request.dependency().withClassifier(classifier)).transferIntoDirectory(request.directory()); + if (classifier_artifact != null) { + artifacts.add(classifier_artifact); + } + } + } + } + return artifacts; + }); + } + + var result = new ArrayList(); + for (var artifacts : ParallelExecution.execute(transfers, resolution.transferParallelism())) { + result.addAll(artifacts); + } + return result; + } finally { + requests_.clear(); + transferTargets_.clear(); + } + } +} diff --git a/src/main/java/rife/bld/operations/DownloadOperation.java b/src/main/java/rife/bld/operations/DownloadOperation.java index be6d397..e2034b7 100644 --- a/src/main/java/rife/bld/operations/DownloadOperation.java +++ b/src/main/java/rife/bld/operations/DownloadOperation.java @@ -42,9 +42,13 @@ public class DownloadOperation extends AbstractOperation { private File libTestModulesDirectory_; private boolean downloadSources_ = false; private boolean downloadJavadoc_ = false; + private final DependencyTransferBatch transfers_ = new DependencyTransferBatch(); /** * Performs the download operation. + *

+ * The artifact transfers of all the scopes are collected first and then + * performed together in a single parallel batch. * * @since 1.5 */ @@ -59,6 +63,7 @@ public class DownloadOperation extends AbstractOperation { executeDownloadRuntimeDependencies(); executeDownloadStandaloneDependencies(); executeDownloadTestDependencies(); + executeTransferDependencies(); if (!silent()) { System.out.println("Downloading finished successfully."); } @@ -110,7 +115,11 @@ public class DownloadOperation extends AbstractOperation { } /** - * Part of the {@link #execute} operation, download the artifacts for a particular dependency scope. + * Part of the {@link #execute} operation, collect the artifact transfers + * for a particular dependency scope into the {@linkplain #transfers() + * transfer batch}. + *

+ * The transfers are performed by {@link #executeTransferDependencies}. * * @param destinationDirectory the directory in which the artifacts should be downloaded * @param modulesDirectory the directory in which the modules should be downloaded @@ -128,7 +137,29 @@ public class DownloadOperation extends AbstractOperation { additional_classifiers = classifiers.toArray(new String[0]); } - dependencies.transferIntoDirectory(new VersionResolution(properties()), artifactRetriever(), repositories(), destinationDirectory, modulesDirectory, additional_classifiers); + transfers().add(dependencies, destinationDirectory, modulesDirectory, additional_classifiers); + } + + /** + * Part of the {@link #execute} operation, perform all the collected + * artifact transfers of the {@linkplain #transfers() transfer batch} + * together in parallel. + * + * @since 2.3.1 + */ + protected void executeTransferDependencies() { + transfers().transfer(new VersionResolution(properties()), artifactRetriever(), repositories()); + } + + /** + * Returns the batch that collects the artifact transfers of this + * operation. + * + * @return the artifact transfer batch of this operation + * @since 2.3.1 + */ + protected DependencyTransferBatch transfers() { + return transfers_; } /** diff --git a/src/test/java/rife/bld/dependencies/TestDependencySet.java b/src/test/java/rife/bld/dependencies/TestDependencySet.java index f6e9025..8615550 100644 --- a/src/test/java/rife/bld/dependencies/TestDependencySet.java +++ b/src/test/java/rife/bld/dependencies/TestDependencySet.java @@ -4,21 +4,17 @@ */ package rife.bld.dependencies; -import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; import rife.ioc.HierarchicalProperties; import rife.tools.FileUtils; import rife.tools.StringUtils; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; import java.nio.file.Files; import java.util.List; -import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; +import static rife.bld.dependencies.TransferTestHelper.*; import static rife.bld.dependencies.RepositoryTestHelper.getNextRepository; import static rife.bld.dependencies.Scope.compile; import static rife.bld.dependencies.Scope.runtime; @@ -390,8 +386,8 @@ public class TestDependencySet { server.start(); var tmp = Files.createTempDirectory("transfers").toFile(); try { - var dependencies = createTransferDependencies(); - var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/")); + var dependencies = createTransferDependencies(1, 6); + var repositories = List.of(transferRepository(server)); var artifacts = dependencies.transferIntoDirectory(new VersionResolution(null), ArtifactRetriever.instance(), repositories, tmp, tmp); @@ -415,8 +411,8 @@ public class TestDependencySet { var resolution = new VersionResolution(properties); assertEquals(1, resolution.transferParallelism()); - var dependencies = createTransferDependencies(); - var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/")); + var dependencies = createTransferDependencies(1, 6); + var repositories = List.of(transferRepository(server)); var artifacts = dependencies.transferIntoDirectory(resolution, ArtifactRetriever.instance(), repositories, tmp, tmp); @@ -428,47 +424,4 @@ public class TestDependencySet { } } - private static DependencySet createTransferDependencies() { - var dependencies = new DependencySet(); - for (var i = 1; i <= 6; i++) { - dependencies.include(new Dependency("com.example", "artifact" + i, new VersionNumber(1, 0, 0))); - } - return dependencies; - } - - private static HttpServer createTransferServer(AtomicInteger maxConcurrentTransfers) - throws IOException { - var active_transfers = new AtomicInteger(); - var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); - server.createContext("/", exchange -> { - var active = active_transfers.incrementAndGet(); - maxConcurrentTransfers.accumulateAndGet(active, Math::max); - try { - // delay the response so that parallel transfers overlap - Thread.sleep(200); - - var body = exchange.getRequestURI().getPath().getBytes(); - exchange.sendResponseHeaders(200, body.length); - exchange.getResponseBody().write(body); - exchange.close(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - active_transfers.decrementAndGet(); - } - }); - server.setExecutor(Executors.newCachedThreadPool()); - return server; - } - - private static void assertTransferredArtifacts(DependencySet dependencies, List artifacts, File directory) { - assertEquals(dependencies.size(), artifacts.size()); - var index = 0; - for (var dependency : dependencies) { - var filename = dependency.artifactId() + "-" + dependency.version() + ".jar"; - assertTrue(artifacts.get(index).location().endsWith(filename), "expected artifact " + filename + " at index " + index); - assertTrue(new File(directory, filename).exists(), "expected file " + filename + " to be transferred"); - ++index; - } - } } diff --git a/src/test/java/rife/bld/dependencies/TestDependencyTransferBatch.java b/src/test/java/rife/bld/dependencies/TestDependencyTransferBatch.java new file mode 100644 index 0000000..10115f3 --- /dev/null +++ b/src/test/java/rife/bld/dependencies/TestDependencyTransferBatch.java @@ -0,0 +1,143 @@ +/* + * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com) + * Licensed under the Apache License, Version 2.0 (the "License") + */ +package rife.bld.dependencies; + +import org.junit.jupiter.api.Test; +import rife.bld.dependencies.exceptions.DependencyTransferException; +import rife.tools.FileUtils; + +import java.io.File; +import java.nio.file.Files; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; +import static rife.bld.dependencies.TransferTestHelper.*; + +public class TestDependencyTransferBatch { + @Test + void testTransferAcrossSets() throws Exception { + var max_concurrent_transfers = new AtomicInteger(); + var server = createTransferServer(max_concurrent_transfers); + server.start(); + var tmp1 = Files.createTempDirectory("transfers1").toFile(); + var tmp2 = Files.createTempDirectory("transfers2").toFile(); + try { + var set1 = createTransferDependencies(1, 3); + var set2 = createTransferDependencies(4, 6); + var repositories = List.of(transferRepository(server)); + + var batch = new DependencyTransferBatch(); + batch.add(set1, tmp1, tmp1) + .add(set2, tmp2, tmp2) + // identical transfers into the same directory are only performed once + .add(set1, tmp1, tmp1); + var artifacts = batch.transfer(new VersionResolution(null), ArtifactRetriever.instance(), repositories); + + assertEquals(6, artifacts.size()); + assertTransferredArtifacts(set1, artifacts.subList(0, 3), tmp1); + assertTransferredArtifacts(set2, artifacts.subList(3, 6), tmp2); + assertTrue(max_concurrent_transfers.get() > 1, "expected concurrent transfers, max was " + max_concurrent_transfers.get()); + } finally { + server.stop(0); + FileUtils.deleteDirectory(tmp1); + FileUtils.deleteDirectory(tmp2); + } + } + + @Test + void testTransferEmptiesTheBatch() throws Exception { + var server = createTransferServer(new AtomicInteger()); + server.start(); + var tmp = Files.createTempDirectory("transfers").toFile(); + try { + var repositories = List.of(transferRepository(server)); + var resolution = new VersionResolution(null); + var batch = new DependencyTransferBatch(); + + var set1 = createTransferDependencies(1, 3); + batch.add(set1, tmp, tmp); + assertTransferredArtifacts(set1, batch.transfer(resolution, ArtifactRetriever.instance(), repositories), tmp); + + // the batch was emptied, nothing is transferred again + assertTrue(batch.transfer(resolution, ArtifactRetriever.instance(), repositories).isEmpty()); + + // the same instance can collect and transfer again, + // including targets that were transferred before + var set2 = createTransferDependencies(3, 5); + batch.add(set2, tmp, tmp); + assertTransferredArtifacts(set2, batch.transfer(resolution, ArtifactRetriever.instance(), repositories), tmp); + } finally { + server.stop(0); + FileUtils.deleteDirectory(tmp); + } + } + + @Test + void testTransferModularJarsIntoModulesDirectory() throws Exception { + var server = createTransferServer(new AtomicInteger()); + server.start(); + var artifacts_dir = Files.createTempDirectory("artifacts").toFile(); + var modules_dir = Files.createTempDirectory("modules").toFile(); + try { + var dependencies = new DependencySet() + .include(new Dependency("com.example", "artifact1", new VersionNumber(1, 0, 0))) + .include(new Dependency("com.example", "module1", new VersionNumber(1, 0, 0), "", Dependency.TYPE_MODULAR_JAR)); + + var artifacts = new DependencyTransferBatch() + .add(dependencies, artifacts_dir, modules_dir) + .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of(transferRepository(server))); + + assertEquals(2, artifacts.size()); + assertTrue(new File(artifacts_dir, "artifact1-1.0.0.jar").exists()); + assertTrue(new File(modules_dir, "module1-1.0.0.jar").exists()); + assertFalse(new File(artifacts_dir, "module1-1.0.0.jar").exists()); + } finally { + server.stop(0); + FileUtils.deleteDirectory(artifacts_dir); + FileUtils.deleteDirectory(modules_dir); + } + } + + @Test + void testTransferClassifiers() throws Exception { + var server = createTransferServer(new AtomicInteger()); + server.start(); + var tmp = Files.createTempDirectory("transfers").toFile(); + try { + var dependencies = createTransferDependencies(1, 2); + var artifacts = new DependencyTransferBatch() + .add(dependencies, tmp, tmp, "sources") + .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of(transferRepository(server))); + + assertEquals(4, artifacts.size()); + for (var i = 1; i <= 2; i++) { + assertTrue(new File(tmp, "artifact" + i + "-1.0.0.jar").exists()); + assertTrue(new File(tmp, "artifact" + i + "-1.0.0-sources.jar").exists()); + } + } finally { + server.stop(0); + FileUtils.deleteDirectory(tmp); + } + } + + @Test + void testMissingDirectories() { + var regular = new DependencySet() + .include(new Dependency("com.example", "artifact1", new VersionNumber(1, 0, 0))); + var modular = new DependencySet() + .include(new Dependency("com.example", "module1", new VersionNumber(1, 0, 0), "", Dependency.TYPE_MODULAR_JAR)); + + assertThrows(DependencyTransferException.class, () -> new DependencyTransferBatch().add(regular, null, null)); + assertThrows(DependencyTransferException.class, () -> new DependencyTransferBatch().add(modular, Files.createTempDirectory("artifacts").toFile(), null)); + } + + @Test + void testEmptyBatch() { + var artifacts = new DependencyTransferBatch() + .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of()); + assertTrue(artifacts.isEmpty()); + } +} diff --git a/src/test/java/rife/bld/dependencies/TransferTestHelper.java b/src/test/java/rife/bld/dependencies/TransferTestHelper.java new file mode 100644 index 0000000..6469d17 --- /dev/null +++ b/src/test/java/rife/bld/dependencies/TransferTestHelper.java @@ -0,0 +1,71 @@ +/* + * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com) + * Licensed under the Apache License, Version 2.0 (the "License") + */ +package rife.bld.dependencies; + +import com.sun.net.httpserver.HttpServer; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Provides a local artifact server and dependency fixtures for testing + * transfers without relying on remote repositories. + */ +abstract class TransferTestHelper { + static HttpServer createTransferServer(AtomicInteger maxConcurrentTransfers) + throws IOException { + var active_transfers = new AtomicInteger(); + var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + server.createContext("/", exchange -> { + var active = active_transfers.incrementAndGet(); + maxConcurrentTransfers.accumulateAndGet(active, Math::max); + try { + // delay the response so that parallel transfers overlap + Thread.sleep(200); + + var body = exchange.getRequestURI().getPath().getBytes(); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + active_transfers.decrementAndGet(); + } + }); + server.setExecutor(Executors.newCachedThreadPool()); + return server; + } + + static Repository transferRepository(HttpServer server) { + return new Repository("http://localhost:" + server.getAddress().getPort() + "/"); + } + + static DependencySet createTransferDependencies(int from, int to) { + var dependencies = new DependencySet(); + for (var i = from; i <= to; i++) { + dependencies.include(new Dependency("com.example", "artifact" + i, new VersionNumber(1, 0, 0))); + } + return dependencies; + } + + static void assertTransferredArtifacts(DependencySet dependencies, List artifacts, File directory) { + assertEquals(dependencies.size(), artifacts.size()); + var index = 0; + for (var dependency : dependencies) { + var filename = dependency.artifactId() + "-" + dependency.version() + ".jar"; + assertTrue(artifacts.get(index).location().endsWith(filename), "expected artifact " + filename + " at index " + index); + assertTrue(new File(directory, filename).exists(), "expected file " + filename + " to be transferred"); + ++index; + } + } +}