Batch artifact transfers across scopes in the download operation

src/main/java/rife/bld/dependencies/DependencySet.java

@@ -8,7 +8,6 @@ import rife.bld.dependencies.exceptions.DependencyTransferException;
 
 import java.io.File;
 import java.util.*;
-import java.util.function.Supplier;
 
 /**
  * Convenience class to handle a set of {@link Dependency} objects.
@@ -146,56 +145,9 @@ public class DependencySet extends AbstractSet<Dependency> implements Set<Depend
      * @since 2.1
      */
     public List<RepositoryArtifact> transferIntoDirectory(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories, File directory, File modulesDirectory, String... classifiers) {
-        var transfers = new ArrayList<Supplier<List<RepositoryArtifact>>>();
-        for (var dependency : this) {
-            var transfer_directory = directory;
-            if (dependency.isModularJar()) {
-                if (modulesDirectory == null) {
-                    throw new DependencyTransferException(dependency, "modules directory is not provided");
-                }
-                transfer_directory = modulesDirectory;
-            }
-            else if (directory == null) {
-                throw new DependencyTransferException(dependency, "artifacts directory is not provided");
-            }
-
-            if (!transfer_directory.exists()) {
-                if (!transfer_directory.mkdirs()) {
-                    throw new DependencyTransferException(dependency, transfer_directory, "couldn't create directory");
-                }
-            }
-
-            final var target_directory = transfer_directory;
-            transfers.add(() -> {
-                var artifacts = new ArrayList<RepositoryArtifact>();
-                var artifact = new DependencyResolver(resolution, retriever, repositories, dependency).transferIntoDirectory(target_directory);
-                if (artifact != null) {
-                    artifacts.add(artifact);
-                }
-
-                if (classifiers != null) {
-                    for (var classifier : classifiers) {
-                        if (classifier != null && !dependency.excludedClassifiers().contains(classifier)) {
-                            var classifier_artifact = new DependencyResolver(resolution, retriever, repositories, dependency.withClassifier(classifier)).transferIntoDirectory(target_directory);
-                            if (classifier_artifact != null) {
-                                artifacts.add(classifier_artifact);
-                            }
-                        }
-                    }
-                }
-                return artifacts;
-            });
-        }
-
-        return executeTransfers(transfers, resolution.transferParallelism());
-    }
-
-    private static List<RepositoryArtifact> executeTransfers(List<Supplier<List<RepositoryArtifact>>> transfers, int transferParallelism) {
-        var result = new ArrayList<RepositoryArtifact>();
-        for (var artifacts : ParallelExecution.execute(transfers, transferParallelism)) {
-            result.addAll(artifacts);
-        }
-        return result;
+        return new DependencyTransferBatch()
+            .add(this, directory, modulesDirectory, classifiers)
+            .transfer(resolution, retriever, repositories);
     }
 
     /**

src/main/java/rife/bld/dependencies/DependencyTransferBatch.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import rife.bld.dependencies.exceptions.DependencyTransferException;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
+
+/**
+ * Collects the artifact transfers of one or more dependency sets so that
+ * they can be transferred together in a single parallel batch, instead of
+ * separate consecutive batches per dependency set.
+ * <p>
+ * The batch itself is a passive collector, the resolution context is only
+ * provided when the transfers are {@linkplain #transfer performed}. The
+ * parallelism is determined by {@link VersionResolution#transferParallelism()},
+ * setting it to {@code 1} makes the transfers sequential. Identical
+ * transfers into the same directory are only performed once.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.3.1
+ */
+public class DependencyTransferBatch {
+    private record TransferRequest(Dependency dependency, File directory, String[] classifiers) {
+    }
+
+    private final List<TransferRequest> requests_ = new ArrayList<>();
+    private final Set<String> transferTargets_ = new HashSet<>();
+
+    /**
+     * Adds the artifact transfers for a dependency set to this batch.
+     * <p>
+     * The destination directory must exist and be writable.
+     *
+     * @param dependencies     the dependencies whose artifacts to transfer
+     * @param directory        the directory to transfer the artifacts into
+     * @param modulesDirectory the directory to download the modules into
+     * @param classifiers      the additional classifiers to transfer
+     * @return this batch instance
+     * @throws DependencyTransferException when the transfer couldn't be prepared
+     * @since 2.3.1
+     */
+    public DependencyTransferBatch add(DependencySet dependencies, File directory, File modulesDirectory, String... classifiers) {
+        for (var dependency : dependencies) {
+            var transfer_directory = directory;
+            if (dependency.isModularJar()) {
+                if (modulesDirectory == null) {
+                    throw new DependencyTransferException(dependency, "modules directory is not provided");
+                }
+                transfer_directory = modulesDirectory;
+            }
+            else if (directory == null) {
+                throw new DependencyTransferException(dependency, "artifacts directory is not provided");
+            }
+
+            if (!transfer_directory.exists()) {
+                if (!transfer_directory.mkdirs()) {
+                    throw new DependencyTransferException(dependency, transfer_directory, "couldn't create directory");
+                }
+            }
+
+            // skip transfers that are already batched for the same directory
+            if (!transferTargets_.add(dependency + " -> " + transfer_directory.getAbsolutePath())) {
+                continue;
+            }
+
+            requests_.add(new TransferRequest(dependency, transfer_directory, classifiers));
+        }
+        return this;
+    }
+
+    /**
+     * Performs all the collected artifact transfers in a single parallel
+     * batch, in the order they were added.
+     * <p>
+     * This empties the batch, transfers can be collected and transferred
+     * again with the same instance.
+     *
+     * @param resolution   the version resolution state that can be cached
+     * @param retriever    the retriever to use to get artifacts
+     * @param repositories the repositories to use for the transfer
+     * @return the list of artifacts that were transferred successfully
+     * @throws DependencyTransferException when an error occurred during the transfer
+     * @since 2.3.1
+     */
+    public List<RepositoryArtifact> transfer(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories) {
+        final var repos = (repositories == null ? List.<Repository>of() : repositories);
+        try {
+            var transfers = new ArrayList<Supplier<List<RepositoryArtifact>>>(requests_.size());
+            for (var request : requests_) {
+                transfers.add(() -> {
+                    var artifacts = new ArrayList<RepositoryArtifact>();
+                    var artifact = new DependencyResolver(resolution, retriever, repos, request.dependency()).transferIntoDirectory(request.directory());
+                    if (artifact != null) {
+                        artifacts.add(artifact);
+                    }
+
+                    if (request.classifiers() != null) {
+                        for (var classifier : request.classifiers()) {
+                            if (classifier != null && !request.dependency().excludedClassifiers().contains(classifier)) {
+                                var classifier_artifact = new DependencyResolver(resolution, retriever, repos, request.dependency().withClassifier(classifier)).transferIntoDirectory(request.directory());
+                                if (classifier_artifact != null) {
+                                    artifacts.add(classifier_artifact);
+                                }
+                            }
+                        }
+                    }
+                    return artifacts;
+                });
+            }
+
+            var result = new ArrayList<RepositoryArtifact>();
+            for (var artifacts : ParallelExecution.execute(transfers, resolution.transferParallelism())) {
+                result.addAll(artifacts);
+            }
+            return result;
+        } finally {
+            requests_.clear();
+            transferTargets_.clear();
+        }
+    }
+}

src/main/java/rife/bld/operations/DownloadOperation.java

@@ -42,9 +42,13 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
     private File libTestModulesDirectory_;
     private boolean downloadSources_ = false;
     private boolean downloadJavadoc_ = false;
+    private final DependencyTransferBatch transfers_ = new DependencyTransferBatch();
 
     /**
      * Performs the download operation.
+     * <p>
+     * The artifact transfers of all the scopes are collected first and then
+     * performed together in a single parallel batch.
      *
      * @since 1.5
      */
@@ -59,6 +63,7 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
         executeDownloadRuntimeDependencies();
         executeDownloadStandaloneDependencies();
         executeDownloadTestDependencies();
+        executeTransferDependencies();
         if (!silent()) {
             System.out.println("Downloading finished successfully.");
         }
@@ -110,7 +115,11 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
     }
 
     /**
-     * Part of the {@link #execute} operation, download the artifacts for a particular dependency scope.
+     * Part of the {@link #execute} operation, collect the artifact transfers
+     * for a particular dependency scope into the {@linkplain #transfers()
+     * transfer batch}.
+     * <p>
+     * The transfers are performed by {@link #executeTransferDependencies}.
      *
      * @param destinationDirectory the directory in which the artifacts should be downloaded
      * @param modulesDirectory     the directory in which the modules should be downloaded
@@ -128,7 +137,29 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
             additional_classifiers = classifiers.toArray(new String[0]);
         }
 
-        dependencies.transferIntoDirectory(new VersionResolution(properties()), artifactRetriever(), repositories(), destinationDirectory, modulesDirectory, additional_classifiers);
+        transfers().add(dependencies, destinationDirectory, modulesDirectory, additional_classifiers);
+    }
+
+    /**
+     * Part of the {@link #execute} operation, perform all the collected
+     * artifact transfers of the {@linkplain #transfers() transfer batch}
+     * together in parallel.
+     *
+     * @since 2.3.1
+     */
+    protected void executeTransferDependencies() {
+        transfers().transfer(new VersionResolution(properties()), artifactRetriever(), repositories());
+    }
+
+    /**
+     * Returns the batch that collects the artifact transfers of this
+     * operation.
+     *
+     * @return the artifact transfer batch of this operation
+     * @since 2.3.1
+     */
+    protected DependencyTransferBatch transfers() {
+        return transfers_;
     }
 
     /**

src/test/java/rife/bld/dependencies/TestDependencySet.java

@@ -4,21 +4,17 @@
  */
 package rife.bld.dependencies;
 
-import com.sun.net.httpserver.HttpServer;
 import org.junit.jupiter.api.Test;
 import rife.ioc.HierarchicalProperties;
 import rife.tools.FileUtils;
 import rife.tools.StringUtils;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.nio.file.Files;
 import java.util.List;
-import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.jupiter.api.Assertions.*;
+import static rife.bld.dependencies.TransferTestHelper.*;
 import static rife.bld.dependencies.RepositoryTestHelper.getNextRepository;
 import static rife.bld.dependencies.Scope.compile;
 import static rife.bld.dependencies.Scope.runtime;
@@ -390,8 +386,8 @@ public class TestDependencySet {
         server.start();
         var tmp = Files.createTempDirectory("transfers").toFile();
         try {
-            var dependencies = createTransferDependencies();
-            var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/"));
+            var dependencies = createTransferDependencies(1, 6);
+            var repositories = List.of(transferRepository(server));
 
             var artifacts = dependencies.transferIntoDirectory(new VersionResolution(null), ArtifactRetriever.instance(), repositories, tmp, tmp);
 
@@ -415,8 +411,8 @@ public class TestDependencySet {
             var resolution = new VersionResolution(properties);
             assertEquals(1, resolution.transferParallelism());
 
-            var dependencies = createTransferDependencies();
-            var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/"));
+            var dependencies = createTransferDependencies(1, 6);
+            var repositories = List.of(transferRepository(server));
 
             var artifacts = dependencies.transferIntoDirectory(resolution, ArtifactRetriever.instance(), repositories, tmp, tmp);
 
@@ -428,47 +424,4 @@ public class TestDependencySet {
         }
     }
 
-    private static DependencySet createTransferDependencies() {
-        var dependencies = new DependencySet();
-        for (var i = 1; i <= 6; i++) {
-            dependencies.include(new Dependency("com.example", "artifact" + i, new VersionNumber(1, 0, 0)));
-        }
-        return dependencies;
-    }
-
-    private static HttpServer createTransferServer(AtomicInteger maxConcurrentTransfers)
-    throws IOException {
-        var active_transfers = new AtomicInteger();
-        var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
-        server.createContext("/", exchange -> {
-            var active = active_transfers.incrementAndGet();
-            maxConcurrentTransfers.accumulateAndGet(active, Math::max);
-            try {
-                // delay the response so that parallel transfers overlap
-                Thread.sleep(200);
-
-                var body = exchange.getRequestURI().getPath().getBytes();
-                exchange.sendResponseHeaders(200, body.length);
-                exchange.getResponseBody().write(body);
-                exchange.close();
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            } finally {
-                active_transfers.decrementAndGet();
-            }
-        });
-        server.setExecutor(Executors.newCachedThreadPool());
-        return server;
-    }
-
-    private static void assertTransferredArtifacts(DependencySet dependencies, List<RepositoryArtifact> artifacts, File directory) {
-        assertEquals(dependencies.size(), artifacts.size());
-        var index = 0;
-        for (var dependency : dependencies) {
-            var filename = dependency.artifactId() + "-" + dependency.version() + ".jar";
-            assertTrue(artifacts.get(index).location().endsWith(filename), "expected artifact " + filename + " at index " + index);
-            assertTrue(new File(directory, filename).exists(), "expected file " + filename + " to be transferred");
-            ++index;
-        }
-    }
 }

src/test/java/rife/bld/dependencies/TestDependencyTransferBatch.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import org.junit.jupiter.api.Test;
+import rife.bld.dependencies.exceptions.DependencyTransferException;
+import rife.tools.FileUtils;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static rife.bld.dependencies.TransferTestHelper.*;
+
+public class TestDependencyTransferBatch {
+    @Test
+    void testTransferAcrossSets() throws Exception {
+        var max_concurrent_transfers = new AtomicInteger();
+        var server = createTransferServer(max_concurrent_transfers);
+        server.start();
+        var tmp1 = Files.createTempDirectory("transfers1").toFile();
+        var tmp2 = Files.createTempDirectory("transfers2").toFile();
+        try {
+            var set1 = createTransferDependencies(1, 3);
+            var set2 = createTransferDependencies(4, 6);
+            var repositories = List.of(transferRepository(server));
+
+            var batch = new DependencyTransferBatch();
+            batch.add(set1, tmp1, tmp1)
+                 .add(set2, tmp2, tmp2)
+                 // identical transfers into the same directory are only performed once
+                 .add(set1, tmp1, tmp1);
+            var artifacts = batch.transfer(new VersionResolution(null), ArtifactRetriever.instance(), repositories);
+
+            assertEquals(6, artifacts.size());
+            assertTransferredArtifacts(set1, artifacts.subList(0, 3), tmp1);
+            assertTransferredArtifacts(set2, artifacts.subList(3, 6), tmp2);
+            assertTrue(max_concurrent_transfers.get() > 1, "expected concurrent transfers, max was " + max_concurrent_transfers.get());
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp1);
+            FileUtils.deleteDirectory(tmp2);
+        }
+    }
+
+    @Test
+    void testTransferEmptiesTheBatch() throws Exception {
+        var server = createTransferServer(new AtomicInteger());
+        server.start();
+        var tmp = Files.createTempDirectory("transfers").toFile();
+        try {
+            var repositories = List.of(transferRepository(server));
+            var resolution = new VersionResolution(null);
+            var batch = new DependencyTransferBatch();
+
+            var set1 = createTransferDependencies(1, 3);
+            batch.add(set1, tmp, tmp);
+            assertTransferredArtifacts(set1, batch.transfer(resolution, ArtifactRetriever.instance(), repositories), tmp);
+
+            // the batch was emptied, nothing is transferred again
+            assertTrue(batch.transfer(resolution, ArtifactRetriever.instance(), repositories).isEmpty());
+
+            // the same instance can collect and transfer again,
+            // including targets that were transferred before
+            var set2 = createTransferDependencies(3, 5);
+            batch.add(set2, tmp, tmp);
+            assertTransferredArtifacts(set2, batch.transfer(resolution, ArtifactRetriever.instance(), repositories), tmp);
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp);
+        }
+    }
+
+    @Test
+    void testTransferModularJarsIntoModulesDirectory() throws Exception {
+        var server = createTransferServer(new AtomicInteger());
+        server.start();
+        var artifacts_dir = Files.createTempDirectory("artifacts").toFile();
+        var modules_dir = Files.createTempDirectory("modules").toFile();
+        try {
+            var dependencies = new DependencySet()
+                .include(new Dependency("com.example", "artifact1", new VersionNumber(1, 0, 0)))
+                .include(new Dependency("com.example", "module1", new VersionNumber(1, 0, 0), "", Dependency.TYPE_MODULAR_JAR));
+
+            var artifacts = new DependencyTransferBatch()
+                .add(dependencies, artifacts_dir, modules_dir)
+                .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of(transferRepository(server)));
+
+            assertEquals(2, artifacts.size());
+            assertTrue(new File(artifacts_dir, "artifact1-1.0.0.jar").exists());
+            assertTrue(new File(modules_dir, "module1-1.0.0.jar").exists());
+            assertFalse(new File(artifacts_dir, "module1-1.0.0.jar").exists());
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(artifacts_dir);
+            FileUtils.deleteDirectory(modules_dir);
+        }
+    }
+
+    @Test
+    void testTransferClassifiers() throws Exception {
+        var server = createTransferServer(new AtomicInteger());
+        server.start();
+        var tmp = Files.createTempDirectory("transfers").toFile();
+        try {
+            var dependencies = createTransferDependencies(1, 2);
+            var artifacts = new DependencyTransferBatch()
+                .add(dependencies, tmp, tmp, "sources")
+                .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of(transferRepository(server)));
+
+            assertEquals(4, artifacts.size());
+            for (var i = 1; i <= 2; i++) {
+                assertTrue(new File(tmp, "artifact" + i + "-1.0.0.jar").exists());
+                assertTrue(new File(tmp, "artifact" + i + "-1.0.0-sources.jar").exists());
+            }
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp);
+        }
+    }
+
+    @Test
+    void testMissingDirectories() {
+        var regular = new DependencySet()
+            .include(new Dependency("com.example", "artifact1", new VersionNumber(1, 0, 0)));
+        var modular = new DependencySet()
+            .include(new Dependency("com.example", "module1", new VersionNumber(1, 0, 0), "", Dependency.TYPE_MODULAR_JAR));
+
+        assertThrows(DependencyTransferException.class, () -> new DependencyTransferBatch().add(regular, null, null));
+        assertThrows(DependencyTransferException.class, () -> new DependencyTransferBatch().add(modular, Files.createTempDirectory("artifacts").toFile(), null));
+    }
+
+    @Test
+    void testEmptyBatch() {
+        var artifacts = new DependencyTransferBatch()
+            .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of());
+        assertTrue(artifacts.isEmpty());
+    }
+}

src/test/java/rife/bld/dependencies/TransferTestHelper.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import com.sun.net.httpserver.HttpServer;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Provides a local artifact server and dependency fixtures for testing
+ * transfers without relying on remote repositories.
+ */
+abstract class TransferTestHelper {
+    static HttpServer createTransferServer(AtomicInteger maxConcurrentTransfers)
+    throws IOException {
+        var active_transfers = new AtomicInteger();
+        var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+        server.createContext("/", exchange -> {
+            var active = active_transfers.incrementAndGet();
+            maxConcurrentTransfers.accumulateAndGet(active, Math::max);
+            try {
+                // delay the response so that parallel transfers overlap
+                Thread.sleep(200);
+
+                var body = exchange.getRequestURI().getPath().getBytes();
+                exchange.sendResponseHeaders(200, body.length);
+                exchange.getResponseBody().write(body);
+                exchange.close();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            } finally {
+                active_transfers.decrementAndGet();
+            }
+        });
+        server.setExecutor(Executors.newCachedThreadPool());
+        return server;
+    }
+
+    static Repository transferRepository(HttpServer server) {
+        return new Repository("http://localhost:" + server.getAddress().getPort() + "/");
+    }
+
+    static DependencySet createTransferDependencies(int from, int to) {
+        var dependencies = new DependencySet();
+        for (var i = from; i <= to; i++) {
+            dependencies.include(new Dependency("com.example", "artifact" + i, new VersionNumber(1, 0, 0)));
+        }
+        return dependencies;
+    }
+
+    static void assertTransferredArtifacts(DependencySet dependencies, List<RepositoryArtifact> artifacts, File directory) {
+        assertEquals(dependencies.size(), artifacts.size());
+        var index = 0;
+        for (var dependency : dependencies) {
+            var filename = dependency.artifactId() + "-" + dependency.version() + ".jar";
+            assertTrue(artifacts.get(index).location().endsWith(filename), "expected artifact " + filename + " at index " + index);
+            assertTrue(new File(directory, filename).exists(), "expected file " + filename + " to be transferred");
+            ++index;
+        }
+    }
+}