Merge 385b0cb2a8 into 27c45033fb

src/main/java/rife/bld/BaseProject.java

@@ -31,7 +31,7 @@ public class BaseProject extends BuildExecutor {
     /**
      * The CLI option to trigger automatic dependency download and purge.
      *
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public static final String AUTO_DOWNLOAD_PURGE_OPTION = "--auto-download-purge";
 

src/main/java/rife/bld/BuildExecutor.java

@@ -144,7 +144,7 @@ public class BuildExecutor {
      *
      * @return {@code true} if the execution is verbose;
      *         or {@code false} otherwise
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public boolean verbose() {
         return verbose_;

src/main/java/rife/bld/dependencies/ArtifactRetriever.java

@@ -19,6 +19,7 @@ import java.net.URLConnection;
 import java.nio.channels.Channels;
 import java.security.MessageDigest;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 
 import static rife.tools.HttpUtils.HEADER_AUTHORIZATION;
 import static rife.tools.HttpUtils.basicAuthorizationHeader;
@@ -35,6 +36,8 @@ import static rife.tools.StringUtils.encodeHexLower;
  * @since 1.5.18
  */
 public abstract class ArtifactRetriever {
+    private static final int TRANSFER_CHUNK_SIZE = 128 * 1024;
+
     private final static ArtifactRetriever UNCACHED = new ArtifactRetriever() {
         String getCached(RepositoryArtifact artifact) {
             return null;
@@ -42,6 +45,10 @@ public abstract class ArtifactRetriever {
 
         void cache(RepositoryArtifact artifact, String content) {
         }
+
+        boolean isCaching() {
+            return false;
+        }
     };
 
     /**
@@ -63,7 +70,7 @@ public abstract class ArtifactRetriever {
      */
     public static ArtifactRetriever cachingInstance() {
         return new ArtifactRetriever() {
-            private final Map<RepositoryArtifact, String> artifactCache = new HashMap<>();
+            private final Map<RepositoryArtifact, String> artifactCache = new ConcurrentHashMap<>();
 
             String getCached(RepositoryArtifact artifact) {
                 return artifactCache.get(artifact);
@@ -73,6 +80,9 @@ public abstract class ArtifactRetriever {
                 artifactCache.put(artifact, content);
             }
 
+            boolean isCaching() {
+                return true;
+            }
         };
     }
 
@@ -83,6 +93,8 @@ public abstract class ArtifactRetriever {
 
     abstract void cache(RepositoryArtifact artifact, String content);
 
+    abstract boolean isCaching();
+
     /**
      * Reads the contents of an artifact as a string.
      *
@@ -141,17 +153,17 @@ public abstract class ArtifactRetriever {
 
         var download_filename = artifact.location().substring(artifact.location().lastIndexOf("/") + 1);
         var download_file = new File(directory, download_filename);
-        System.out.print("Downloading: " + artifact.location() + " ... ");
-        System.out.flush();
+        var transfer = TransferOutput.instance().start(artifact.location());
+        var status = "";
         try {
             if (artifact.repository().isLocal()) {
                 var source = new File(artifact.location());
                 if (source.exists()) {
                     FileUtils.copy(source, download_file);
-                    System.out.print("done");
+                    status = "done";
                     return true;
                 } else {
-                    System.out.print("not found");
+                    status = "not found";
                     return false;
                 }
             } else {
@@ -159,7 +171,7 @@ public abstract class ArtifactRetriever {
                     if (download_file.exists() && download_file.canRead()) {
                         if (checkHash(artifact, download_file, ".sha256", "SHA-256") ||
                             checkHash(artifact, download_file, ".md5", "MD5")) {
-                            System.out.print("exists");
+                            status = "exists";
                             return true;
                         }
                     }
@@ -170,23 +182,29 @@ public abstract class ArtifactRetriever {
                             HEADER_AUTHORIZATION,
                             basicAuthorizationHeader(artifact.repository().username(), artifact.repository().password()));
                     }
+                    var content_length = connection.getContentLengthLong();
                     try (var input_stream = connection.getInputStream()) {
                         var readableByteChannel = Channels.newChannel(input_stream);
                         try (var fileOutputStream = new FileOutputStream(download_file)) {
                             var fileChannel = fileOutputStream.getChannel();
-                            fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
+                            var position = 0L;
+                            long transferred;
+                            while ((transferred = fileChannel.transferFrom(readableByteChannel, position, TRANSFER_CHUNK_SIZE)) > 0) {
+                                position += transferred;
+                                transfer.progress(position, content_length);
+                            }
 
-                            System.out.print("done");
+                            status = "done";
                             return true;
                         }
                     }
                 } catch (FileNotFoundException e) {
-                    System.out.print("not found");
+                    status = "not found";
                     return false;
                 }
             }
         } finally {
-            System.out.println();
+            transfer.finish(status);
         }
     }
 

src/main/java/rife/bld/dependencies/DependencyResolver.java

@@ -121,6 +121,17 @@ public class DependencyResolver {
      * @since 1.5
      */
     public DependencySet getAllDependencies(Scope... scopes) {
+        var prefetcher = PomPrefetcher.create(resolution_, retriever_, repositories_);
+        try {
+            return getAllDependencies(prefetcher, scopes);
+        } finally {
+            if (prefetcher != null) {
+                prefetcher.shutdown();
+            }
+        }
+    }
+
+    DependencySet getAllDependencies(PomPrefetcher prefetcher, Scope... scopes) {
         var result = new DependencySet();
         var overridden = resolution_.overrideDependency(dependency_);
         result.add(overridden);
@@ -138,6 +149,11 @@ public class DependencyResolver {
             next_dependencies.removeIf(it -> matchesExclusions(exclusion_context, it));
             // add all next dependencies to the queue
             dependency_queue.addAll(next_dependencies);
+            // speculatively retrieve the POMs of the queued dependencies in
+            // parallel so that they are cached when they're processed in order
+            if (prefetcher != null) {
+                prefetcher.prefetch(next_dependencies);
+            }
 
             // unless we find a next set of dependencies to add, stop resolving
             parent = null;

src/main/java/rife/bld/dependencies/DependencyScopes.java

@@ -6,6 +6,7 @@ package rife.bld.dependencies;
 
 import rife.ioc.HierarchicalProperties;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 
@@ -147,15 +148,14 @@ public class DependencyScopes extends LinkedHashMap<Scope, DependencySet> {
 
     private DependencySet resolveScopedDependencies(HierarchicalProperties properties, ArtifactRetriever retriever, List<Repository> repositories, Scope[] resolvedScopes, Scope[] transitiveScopes, DependencySet excluded) {
         var resolution = new VersionResolution(properties);
-        var dependencies = new DependencySet();
+        var roots = new ArrayList<Dependency>();
         for (var scope : resolvedScopes) {
             var scoped_dependencies = get(scope);
             if (scoped_dependencies != null) {
-                for (var dependency : scoped_dependencies) {
-                    dependencies.addAll(new DependencyResolver(resolution, retriever, repositories, dependency).getAllDependencies(transitiveScopes));
-                }
+                roots.addAll(scoped_dependencies);
             }
         }
+        var dependencies = new ParallelDependencyResolver(resolution, retriever, repositories).resolveAllDependencies(roots, transitiveScopes);
         if (excluded != null) {
             dependencies.removeAll(excluded);
         }

src/main/java/rife/bld/dependencies/DependencySet.java

@@ -128,6 +128,11 @@ public class DependencySet extends AbstractSet<Dependency> implements Set<Depend
      * including other classifiers.
      * <p>
      * The destination directory must exist and be writable.
+     * <p>
+     * The artifacts of different dependencies are transferred in parallel,
+     * the {@value VersionResolution#PROPERTY_TRANSFER_PARALLELISM} property
+     * can be used to change the number of simultaneous transfers, setting it
+     * to {@code 1} makes the transfers sequential.
      *
      * @param resolution        the version resolution state that can be cached
      * @param retriever         the retriever to use to get artifacts
@@ -140,42 +145,9 @@ public class DependencySet extends AbstractSet<Dependency> implements Set<Depend
      * @since 2.1
      */
     public List<RepositoryArtifact> transferIntoDirectory(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories, File directory, File modulesDirectory, String... classifiers) {
-        var result = new ArrayList<RepositoryArtifact>();
-        for (var dependency : this) {
-            var transfer_directory = directory;
-            if (dependency.isModularJar()) {
-                if (modulesDirectory == null) {
-                    throw new DependencyTransferException(dependency, "modules directory is not provided");
-                }
-                transfer_directory = modulesDirectory;
-            }
-            else if (directory == null) {
-                throw new DependencyTransferException(dependency, "artifacts directory is not provided");
-            }
-
-            if (!transfer_directory.exists()) {
-                if (!transfer_directory.mkdirs()) {
-                    throw new DependencyTransferException(dependency, transfer_directory, "couldn't create directory");
-                }
-            }
-
-            var artifact = new DependencyResolver(resolution, retriever, repositories, dependency).transferIntoDirectory(transfer_directory);
-            if (artifact != null) {
-                result.add(artifact);
-            }
-
-            if (classifiers != null) {
-                for (var classifier : classifiers) {
-                    if (classifier != null && !dependency.excludedClassifiers().contains(classifier)) {
-                        var classifier_artifact = new DependencyResolver(resolution, retriever, repositories, dependency.withClassifier(classifier)).transferIntoDirectory(transfer_directory);
-                        if (classifier_artifact != null) {
-                            result.add(classifier_artifact);
-                        }
-                    }
-                }
-            }
-        }
-        return result;
+        return new DependencyTransferBatch()
+            .add(this, directory, modulesDirectory, classifiers)
+            .transfer(resolution, retriever, repositories);
     }
 
     /**
@@ -206,11 +178,7 @@ public class DependencySet extends AbstractSet<Dependency> implements Set<Depend
      * @since 2.0
      */
     public String generateTransitiveDependencyTree(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories, Scope... scopes) {
-        var compile_dependencies = new DependencySet();
-        for (var dependency : this) {
-            compile_dependencies.addAll(new DependencyResolver(resolution, retriever, repositories, dependency).getAllDependencies(scopes));
-        }
-        return compile_dependencies.generateDependencyTree();
+        return new ParallelDependencyResolver(resolution, retriever, repositories).resolveAllDependencies(this, scopes).generateDependencyTree();
     }
 
     /**

src/main/java/rife/bld/dependencies/DependencyTransferBatch.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import rife.bld.dependencies.exceptions.DependencyTransferException;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Supplier;
+
+/**
+ * Collects the artifact transfers of one or more dependency sets so that
+ * they can be transferred together in a single parallel batch, instead of
+ * separate consecutive batches per dependency set.
+ * <p>
+ * The batch itself is a passive collector, the resolution context is only
+ * provided when the transfers are {@linkplain #transfer performed}. The
+ * parallelism is determined by {@link VersionResolution#transferParallelism()},
+ * setting it to {@code 1} makes the transfers sequential. Identical
+ * transfers into the same directory are only performed once.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.4.0
+ */
+public class DependencyTransferBatch {
+    private record TransferRequest(Dependency dependency, File directory, String[] classifiers) {
+    }
+
+    private final List<TransferRequest> requests_ = new ArrayList<>();
+    private final Set<String> transferTargets_ = new HashSet<>();
+
+    /**
+     * Adds the artifact transfers for a dependency set to this batch.
+     * <p>
+     * The destination directory must exist and be writable.
+     *
+     * @param dependencies     the dependencies whose artifacts to transfer
+     * @param directory        the directory to transfer the artifacts into
+     * @param modulesDirectory the directory to download the modules into
+     * @param classifiers      the additional classifiers to transfer
+     * @return this batch instance
+     * @throws DependencyTransferException when the transfer couldn't be prepared
+     * @since 2.4.0
+     */
+    public DependencyTransferBatch add(DependencySet dependencies, File directory, File modulesDirectory, String... classifiers) {
+        for (var dependency : dependencies) {
+            var transfer_directory = directory;
+            if (dependency.isModularJar()) {
+                if (modulesDirectory == null) {
+                    throw new DependencyTransferException(dependency, "modules directory is not provided");
+                }
+                transfer_directory = modulesDirectory;
+            }
+            else if (directory == null) {
+                throw new DependencyTransferException(dependency, "artifacts directory is not provided");
+            }
+
+            if (!transfer_directory.exists()) {
+                if (!transfer_directory.mkdirs()) {
+                    throw new DependencyTransferException(dependency, transfer_directory, "couldn't create directory");
+                }
+            }
+
+            // skip transfers that are already batched for the same directory
+            if (!transferTargets_.add(dependency + " -> " + transfer_directory.getAbsolutePath())) {
+                continue;
+            }
+
+            requests_.add(new TransferRequest(dependency, transfer_directory, classifiers));
+        }
+        return this;
+    }
+
+    /**
+     * Performs all the collected artifact transfers in a single parallel
+     * batch, in the order they were added.
+     * <p>
+     * This empties the batch, transfers can be collected and transferred
+     * again with the same instance.
+     *
+     * @param resolution   the version resolution state that can be cached
+     * @param retriever    the retriever to use to get artifacts
+     * @param repositories the repositories to use for the transfer
+     * @return the list of artifacts that were transferred successfully
+     * @throws DependencyTransferException when an error occurred during the transfer
+     * @since 2.4.0
+     */
+    public List<RepositoryArtifact> transfer(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories) {
+        final var repos = (repositories == null ? List.<Repository>of() : repositories);
+        try {
+            var transfers = new ArrayList<Supplier<List<RepositoryArtifact>>>(requests_.size());
+            for (var request : requests_) {
+                transfers.add(() -> {
+                    var artifacts = new ArrayList<RepositoryArtifact>();
+                    var artifact = new DependencyResolver(resolution, retriever, repos, request.dependency()).transferIntoDirectory(request.directory());
+                    if (artifact != null) {
+                        artifacts.add(artifact);
+                    }
+
+                    if (request.classifiers() != null) {
+                        for (var classifier : request.classifiers()) {
+                            if (classifier != null && !request.dependency().excludedClassifiers().contains(classifier)) {
+                                var classifier_artifact = new DependencyResolver(resolution, retriever, repos, request.dependency().withClassifier(classifier)).transferIntoDirectory(request.directory());
+                                if (classifier_artifact != null) {
+                                    artifacts.add(classifier_artifact);
+                                }
+                            }
+                        }
+                    }
+                    return artifacts;
+                });
+            }
+
+            var result = new ArrayList<RepositoryArtifact>();
+            for (var artifacts : ParallelExecution.execute(transfers, resolution.transferParallelism())) {
+                result.addAll(artifacts);
+            }
+            return result;
+        } finally {
+            requests_.clear();
+            transferTargets_.clear();
+        }
+    }
+}

src/main/java/rife/bld/dependencies/ParallelDependencyResolver.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Supplier;
+
+/**
+ * Resolves multiple dependencies in parallel within a list of
+ * Maven-compatible repositories.
+ * <p>
+ * The parallelism is determined by {@link VersionResolution#resolutionParallelism()},
+ * setting it to {@code 1} makes the resolution sequential. The results are
+ * always identical to resolving each dependency sequentially with a
+ * {@link DependencyResolver}, in the same order.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.4.0
+ */
+public class ParallelDependencyResolver {
+    private final VersionResolution resolution_;
+    private final ArtifactRetriever retriever_;
+    private final List<Repository> repositories_;
+
+    /**
+     * Creates a new parallel resolver.
+     * <p>
+     * The repositories will be checked in the order they're listed.
+     *
+     * @param resolution   the version resolution state that can be cached
+     * @param retriever    the retriever to use to get artifacts
+     * @param repositories the repositories to use for the resolution
+     * @since 2.4.0
+     */
+    public ParallelDependencyResolver(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories) {
+        resolution_ = resolution;
+        retriever_ = retriever;
+        if (repositories == null) {
+            repositories = List.of();
+        }
+        repositories_ = repositories;
+    }
+
+    /**
+     * Resolves the transitive dependencies of multiple root dependencies,
+     * merging the results in the order of the provided roots.
+     * <p>
+     * The roots are resolved in parallel while a shared prefetcher
+     * speculatively warms the retriever cache across all of them.
+     *
+     * @param roots  the root dependencies to resolve
+     * @param scopes the scopes to return the transitive dependencies for
+     * @return the merged transitive dependencies of all the roots
+     * @since 2.4.0
+     */
+    public DependencySet resolveAllDependencies(Collection<Dependency> roots, Scope... scopes) {
+        var result = new DependencySet();
+        if (roots.isEmpty()) {
+            return result;
+        }
+
+        var prefetcher = PomPrefetcher.create(resolution_, retriever_, repositories_);
+        try {
+            var resolutions = new ArrayList<Supplier<DependencySet>>(roots.size());
+            for (var root : roots) {
+                resolutions.add(() -> new DependencyResolver(resolution_, retriever_, repositories_, root).getAllDependencies(prefetcher, scopes));
+            }
+            for (var dependencies : ParallelExecution.execute(resolutions, resolution_.resolutionParallelism())) {
+                result.addAll(dependencies);
+            }
+        } finally {
+            if (prefetcher != null) {
+                prefetcher.shutdown();
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Resolves the latest versions of multiple dependencies, returning them
+     * in the same order as the provided dependencies.
+     *
+     * @param dependencies the dependencies to resolve the latest versions of
+     * @return the latest versions in the order of the provided dependencies
+     * @since 2.4.0
+     */
+    public List<Version> resolveLatestVersions(List<Dependency> dependencies) {
+        var resolutions = new ArrayList<Supplier<Version>>(dependencies.size());
+        for (var dependency : dependencies) {
+            resolutions.add(() -> new DependencyResolver(resolution_, retriever_, repositories_, dependency).latestVersion());
+        }
+        return ParallelExecution.execute(resolutions, resolution_.resolutionParallelism());
+    }
+
+    /**
+     * Returns the version resolution state that can be cached.
+     *
+     * @return the version resolution state
+     * @since 2.4.0
+     */
+    public VersionResolution resolution() {
+        return resolution_;
+    }
+
+    /**
+     * Retrieve the repositories that are used by this resolver.
+     *
+     * @return the resolver's repositories
+     * @since 2.4.0
+     */
+    public List<Repository> repositories() {
+        return repositories_;
+    }
+}

src/main/java/rife/bld/dependencies/ParallelExecution.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Supplier;
+
+/**
+ * Executes tasks in parallel while preserving the order of their results
+ * and the sequential semantics of failures: the first task that fails in
+ * order will have its exception rethrown.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.4.0
+ */
+final class ParallelExecution {
+    private ParallelExecution() {
+    }
+
+    static <T> List<T> execute(List<Supplier<T>> tasks, int parallelism) {
+        var result = new ArrayList<T>(tasks.size());
+
+        parallelism = Math.min(tasks.size(), parallelism);
+        if (parallelism <= 1) {
+            for (var task : tasks) {
+                result.add(task.get());
+            }
+            return result;
+        }
+
+        var executor = Executors.newFixedThreadPool(parallelism);
+        try {
+            var futures = new ArrayList<Future<T>>(tasks.size());
+            for (var task : tasks) {
+                futures.add(executor.submit(task::get));
+            }
+            for (var future : futures) {
+                try {
+                    result.add(future.get());
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    throw new IllegalStateException("Parallel execution was interrupted", e);
+                } catch (ExecutionException e) {
+                    if (e.getCause() instanceof RuntimeException runtime) {
+                        throw runtime;
+                    }
+                    if (e.getCause() instanceof Error error) {
+                        throw error;
+                    }
+                    throw new IllegalStateException(e.getCause());
+                }
+            }
+        } finally {
+            executor.shutdownNow();
+        }
+        return result;
+    }
+}

src/main/java/rife/bld/dependencies/PomPrefetcher.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Speculatively retrieves POMs in parallel so that they are already cached
+ * by the artifact retriever when the sequential dependency resolution
+ * processes them, without influencing the resolution semantics.
+ * <p>
+ * A single instance can be shared by multiple concurrent resolutions, the
+ * POM of each unique dependency will only be prefetched once.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.4.0
+ */
+class PomPrefetcher {
+    private final VersionResolution resolution_;
+    private final ArtifactRetriever retriever_;
+    private final List<Repository> repositories_;
+    private final ExecutorService executor_;
+    private final Set<Dependency> submitted_ = ConcurrentHashMap.newKeySet();
+
+    /**
+     * Creates a prefetcher when it can be beneficial.
+     *
+     * @return the prefetcher; or {@code null} when the retriever doesn't
+     * cache the retrieved POMs or when the resolution parallelism disables it
+     * @since 2.4.0
+     */
+    static PomPrefetcher create(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories) {
+        // prefetching only has benefits when the retrieved POMs are
+        // cached for the sequential resolution that follows
+        if (!retriever.isCaching() || resolution.resolutionParallelism() <= 1) {
+            return null;
+        }
+        return new PomPrefetcher(resolution, retriever, repositories);
+    }
+
+    private PomPrefetcher(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories) {
+        resolution_ = resolution;
+        retriever_ = retriever;
+        repositories_ = repositories;
+        executor_ = Executors.newFixedThreadPool(resolution.resolutionParallelism());
+    }
+
+    void prefetch(Collection<PomDependency> candidates) {
+        for (var candidate : candidates) {
+            var dependency = resolution_.overrideDependency(candidate.convertToDependency());
+            if (submitted_.add(dependency)) {
+                executor_.submit(() -> {
+                    try {
+                        new DependencyResolver(resolution_, retriever_, repositories_, dependency).getMavenPom(dependency);
+                    } catch (Throwable e) {
+                        // failures are ignored since they will resurface
+                        // with the proper context when the dependency is
+                        // resolved in order
+                    }
+                });
+            }
+        }
+    }
+
+    void shutdown() {
+        executor_.shutdownNow();
+    }
+}

src/main/java/rife/bld/dependencies/TransferOutput.java

@@ -0,0 +1,185 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Outputs the status of artifact transfers to the console.
+ * <p>
+ * When the console is an interactive terminal, the transfers that are in
+ * progress are displayed in a live updating block at the bottom of the
+ * output, with each transfer reporting its progress as it happens.
+ * Otherwise, a single line is printed when each transfer finishes, keeping
+ * the output stable for CI logs, pipes and IDE consoles, while still never
+ * interleaving the lines of parallel transfers.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.4.0
+ */
+abstract class TransferOutput {
+    private static final TransferOutput INSTANCE = create();
+
+    static TransferOutput instance() {
+        return INSTANCE;
+    }
+
+    private static TransferOutput create() {
+        if (System.console() != null && !"dumb".equals(System.getenv("TERM"))) {
+            return new AnsiTransferOutput();
+        }
+        return new PlainTransferOutput();
+    }
+
+    /**
+     * Starts tracking a single artifact transfer.
+     *
+     * @param location the location of the artifact that is being transferred
+     * @return the transfer to report progress and completion on
+     * @since 2.4.0
+     */
+    abstract Transfer start(String location);
+
+    /**
+     * A single artifact transfer whose progress is being tracked.
+     *
+     * @since 2.4.0
+     */
+    abstract static class Transfer {
+        protected final String location_;
+
+        protected Transfer(String location) {
+            location_ = location;
+        }
+
+        /**
+         * Reports transfer progress.
+         *
+         * @param transferred the number of bytes transferred so far
+         * @param total       the total number of bytes; or {@code -1} when unknown
+         * @since 2.4.0
+         */
+        abstract void progress(long transferred, long total);
+
+        /**
+         * Reports that the transfer finished.
+         *
+         * @param status the final status of the transfer, like
+         *               {@code done}, {@code exists} or {@code not found}
+         * @since 2.4.0
+         */
+        abstract void finish(String status);
+    }
+
+    static String describe(String location, String status) {
+        return "Downloading: " + location + " ... " + status;
+    }
+
+    private static class PlainTransferOutput extends TransferOutput {
+        Transfer start(String location) {
+            return new Transfer(location) {
+                void progress(long transferred, long total) {
+                    // no-op, only the finished state is printed
+                }
+
+                void finish(String status) {
+                    System.out.println(describe(location_, status));
+                }
+            };
+        }
+    }
+
+    private static class AnsiTransferOutput extends TransferOutput {
+        private static final long REPAINT_INTERVAL_MS = 100;
+        private static final int MAX_LINE_WIDTH = 79;
+
+        private final List<AnsiTransfer> active_ = new ArrayList<>();
+        private int paintedLines_ = 0;
+        private long lastRepaint_ = 0;
+
+        synchronized Transfer start(String location) {
+            var transfer = new AnsiTransfer(location);
+            active_.add(transfer);
+            repaint(null);
+            return transfer;
+        }
+
+        private synchronized void reportProgress() {
+            if (System.currentTimeMillis() - lastRepaint_ >= REPAINT_INTERVAL_MS) {
+                repaint(null);
+            }
+        }
+
+        private synchronized void reportFinished(AnsiTransfer transfer, String status) {
+            active_.remove(transfer);
+            repaint(describe(transfer.location_, status));
+        }
+
+        private void repaint(String finishedLine) {
+            var output = new StringBuilder();
+            if (paintedLines_ > 0) {
+                // move the cursor to the beginning of the live block
+                // and clear everything below it
+                output.append("\u001B[").append(paintedLines_).append("F\u001B[0J");
+            }
+            if (finishedLine != null) {
+                output.append(finishedLine).append('\n');
+            }
+            for (var transfer : active_) {
+                output.append(transfer.statusLine()).append('\n');
+            }
+            paintedLines_ = active_.size();
+            lastRepaint_ = System.currentTimeMillis();
+            System.out.print(output);
+            System.out.flush();
+        }
+
+        private class AnsiTransfer extends Transfer {
+            private volatile long transferred_ = 0;
+            private volatile long total_ = -1;
+
+            AnsiTransfer(String location) {
+                super(location);
+            }
+
+            void progress(long transferred, long total) {
+                transferred_ = transferred;
+                total_ = total;
+                reportProgress();
+            }
+
+            void finish(String status) {
+                reportFinished(this, status);
+            }
+
+            String statusLine() {
+                var filename = location_.substring(location_.lastIndexOf('/') + 1);
+                var line = new StringBuilder("Downloading: ").append(filename).append(" ... ");
+                if (total_ > 0) {
+                    line.append(transferred_ * 100 / total_).append('%');
+                } else if (transferred_ > 0) {
+                    line.append(humanBytes(transferred_));
+                }
+                // prevent lines from wrapping since that would break
+                // the cursor repositioning of the repaints
+                if (line.length() > MAX_LINE_WIDTH) {
+                    line.setLength(MAX_LINE_WIDTH);
+                }
+                return line.toString();
+            }
+
+            private String humanBytes(long bytes) {
+                if (bytes >= 1024 * 1024) {
+                    return (bytes / (1024 * 1024)) + " MB";
+                }
+                if (bytes >= 1024) {
+                    return (bytes / 1024) + " KB";
+                }
+                return bytes + " B";
+            }
+        }
+    }
+}

src/main/java/rife/bld/dependencies/VersionResolution.java

@@ -8,6 +8,7 @@ import rife.ioc.HierarchicalProperties;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.logging.Logger;
 
 /**
  * This class is responsible for managing version overrides for dependencies.
@@ -27,6 +28,10 @@ import java.util.Map;
  * bld.override-tests=com.uwyn.rife2:bld-tests-badge:1.4.7
  * bld.override-h2=com.h2database:h2:2.2.222
  * </pre>
+ * <p>
+ * It also captures other dependency resolution preferences, like the number
+ * of parallel artifact transfers through the "{@code bld.transferParallelism}"
+ * property.
  * @since 2.0
  */
 public class VersionResolution {
@@ -36,7 +41,26 @@ public class VersionResolution {
      */
     public static final String PROPERTY_OVERRIDE_PREFIX = "bld.override";
 
+    /**
+     * The property key that determines how many artifact transfers are
+     * performed in parallel, {@code 1} makes them sequential.
+     * @since 2.4.0
+     */
+    public static final String PROPERTY_TRANSFER_PARALLELISM = "bld.transferParallelism";
+    private static final int DEFAULT_TRANSFER_PARALLELISM = 6;
+
+    /**
+     * The property key that determines how many POMs are speculatively
+     * retrieved in parallel during transitive dependency resolution,
+     * {@code 1} disables the parallel retrieval.
+     * @since 2.4.0
+     */
+    public static final String PROPERTY_RESOLUTION_PARALLELISM = "bld.resolutionParallelism";
+    private static final int DEFAULT_RESOLUTION_PARALLELISM = 6;
+
     private final Map<String, Version> versionOverrides_ = new HashMap<>();
+    private final int transferParallelism_;
+    private final int resolutionParallelism_;
 
     /**
      * Returns a dummy {@code VersionResolution} instance that doesn't override anything.
@@ -74,6 +98,22 @@ public class VersionResolution {
                 }
             }
         }
+        transferParallelism_ = parseParallelism(properties, PROPERTY_TRANSFER_PARALLELISM, DEFAULT_TRANSFER_PARALLELISM);
+        resolutionParallelism_ = parseParallelism(properties, PROPERTY_RESOLUTION_PARALLELISM, DEFAULT_RESOLUTION_PARALLELISM);
+    }
+
+    private static int parseParallelism(HierarchicalProperties properties, String property, int defaultValue) {
+        if (properties != null) {
+            var parallelism = properties.getValueString(property);
+            if (parallelism != null && !parallelism.isBlank()) {
+                try {
+                    return Math.max(1, Integer.parseInt(parallelism.trim()));
+                } catch (NumberFormatException e) {
+                    Logger.getLogger("rife.bld").warning("Unable to parse the " + property + " property as an integer: '" + parallelism + "', using " + defaultValue + " instead");
+                }
+            }
+        }
+        return defaultValue;
     }
 
     /**
@@ -122,4 +162,27 @@ public class VersionResolution {
     public Map<String, Version> versionOverrides() {
         return versionOverrides_;
     }
+
+    /**
+     * Returns the number of artifact transfers that are performed in parallel,
+     * {@code 1} means transfers are sequential.
+     *
+     * @return the number of parallel artifact transfers
+     * @since 2.4.0
+     */
+    public int transferParallelism() {
+        return transferParallelism_;
+    }
+
+    /**
+     * Returns the number of POMs that are speculatively retrieved in parallel
+     * during transitive dependency resolution, {@code 1} means the parallel
+     * retrieval is disabled.
+     *
+     * @return the number of parallel POM retrievals
+     * @since 2.4.0
+     */
+    public int resolutionParallelism() {
+        return resolutionParallelism_;
+    }
 }

src/main/java/rife/bld/operations/AbstractCreateOperation.java

@@ -179,7 +179,7 @@ public abstract class AbstractCreateOperation<T extends AbstractCreateOperation<
      * parents, outputting the location when the operation is {@link #verbose()}.
      *
      * @param directory the directory to create
-     * @since 2.3.1
+     * @since 2.4.0
      */
     protected void executeCreateDirectory(File directory) {
         if (verbose()) {
@@ -194,7 +194,7 @@ public abstract class AbstractCreateOperation<T extends AbstractCreateOperation<
      *
      * @param content the content to write
      * @param file    the file to write the content into
-     * @since 2.3.1
+     * @since 2.4.0
      */
     protected void executeWriteProjectFile(String content, File file)
     throws FileUtilsErrorException {

src/main/java/rife/bld/operations/AbstractOperation.java

@@ -55,7 +55,7 @@ public abstract class AbstractOperation<T extends AbstractOperation<T>> {
      * @param verbose {@code true} if the operation should be verbose;
      *                {@code false} otherwise
      * @return this operation instance
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public T verbose(boolean verbose) {
         verbose_ = verbose;
@@ -68,7 +68,7 @@ public abstract class AbstractOperation<T extends AbstractOperation<T>> {
      *
      * @return {@code true} if the operation should be verbose;
      * {@code false} otherwise
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public boolean verbose() {
         return verbose_;

src/main/java/rife/bld/operations/AbstractProcessOperation.java

@@ -241,7 +241,7 @@ public abstract class AbstractProcessOperation<T extends AbstractProcessOperatio
      *
      * @param classpath classpath entries for the operation
      * @return this operation instance
-     * @since 2.3.1
+     * @since 2.4.0
      *
      */
     public T classpath(File... classpath) {
@@ -269,7 +269,7 @@ public abstract class AbstractProcessOperation<T extends AbstractProcessOperatio
      *
      * @param classpath a list of classpath entries for the operation
      * @return this operation instance
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public T classpath(Collection<File> classpath) {
         classpath_.addAll(classpath.stream().map(File::getAbsolutePath).toList());
@@ -293,7 +293,7 @@ public abstract class AbstractProcessOperation<T extends AbstractProcessOperatio
      *
      * @param modulePath module path entries for the operation
      * @return this operation instance
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public T modulePath(File... modulePath) {
         return modulePath(List.of(modulePath));
@@ -320,7 +320,7 @@ public abstract class AbstractProcessOperation<T extends AbstractProcessOperatio
      *
      * @param modulePath a list of module path entries for the operation
      * @return this operation instance
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public T modulePath(Collection<File> modulePath) {
         modulePath_.addAll(modulePath.stream().map(File::getAbsolutePath).toList());

src/main/java/rife/bld/operations/DownloadOperation.java

@@ -42,9 +42,13 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
     private File libTestModulesDirectory_;
     private boolean downloadSources_ = false;
     private boolean downloadJavadoc_ = false;
+    private final DependencyTransferBatch transfers_ = new DependencyTransferBatch();
 
     /**
      * Performs the download operation.
+     * <p>
+     * The artifact transfers of all the scopes are collected first and then
+     * performed together in a single parallel batch.
      *
      * @since 1.5
      */
@@ -59,6 +63,7 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
         executeDownloadRuntimeDependencies();
         executeDownloadStandaloneDependencies();
         executeDownloadTestDependencies();
+        executeTransferDependencies();
         if (!silent()) {
             System.out.println("Downloading finished successfully.");
         }
@@ -110,7 +115,11 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
     }
 
     /**
-     * Part of the {@link #execute} operation, download the artifacts for a particular dependency scope.
+     * Part of the {@link #execute} operation, collect the artifact transfers
+     * for a particular dependency scope into the {@linkplain #transfers()
+     * transfer batch}.
+     * <p>
+     * The transfers are performed by {@link #executeTransferDependencies}.
      *
      * @param destinationDirectory the directory in which the artifacts should be downloaded
      * @param modulesDirectory     the directory in which the modules should be downloaded
@@ -128,7 +137,29 @@ public class DownloadOperation extends AbstractOperation<DownloadOperation> {
             additional_classifiers = classifiers.toArray(new String[0]);
         }
 
-        dependencies.transferIntoDirectory(new VersionResolution(properties()), artifactRetriever(), repositories(), destinationDirectory, modulesDirectory, additional_classifiers);
+        transfers().add(dependencies, destinationDirectory, modulesDirectory, additional_classifiers);
+    }
+
+    /**
+     * Part of the {@link #execute} operation, perform all the collected
+     * artifact transfers of the {@linkplain #transfers() transfer batch}
+     * together in parallel.
+     *
+     * @since 2.4.0
+     */
+    protected void executeTransferDependencies() {
+        transfers().transfer(new VersionResolution(properties()), artifactRetriever(), repositories());
+    }
+
+    /**
+     * Returns the batch that collects the artifact transfers of this
+     * operation.
+     *
+     * @return the artifact transfer batch of this operation
+     * @since 2.4.0
+     */
+    protected DependencyTransferBatch transfers() {
+        return transfers_;
     }
 
     /**

src/main/java/rife/bld/operations/JavacOptions.java

@@ -136,7 +136,7 @@ public class JavacOptions extends ArrayList<String> {
      * of other-module is ALL-UNNAMED
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions addExports(String... modules) {
         if (isNotEmpty(modules)) {
@@ -151,7 +151,7 @@ public class JavacOptions extends ArrayList<String> {
      * of other-module is ALL-UNNAMED
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions addExports(Collection<String> modules) {
         return addCommaSeparatedOption("--add-exports", modules);
@@ -161,7 +161,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specifies additional modules to be considered as required by a given module
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions addReads(String... modules) {
         if (isNotEmpty(modules)) {
@@ -174,7 +174,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specifies additional modules to be considered as required by a given module
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions addReads(Collection<String> modules) {
         return addCommaSeparatedOption("--add-reads", modules);
@@ -222,7 +222,7 @@ public class JavacOptions extends ArrayList<String> {
      * if none specified or inferred
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions defaultModuleForCreatedFiles(String module) {
         add("--default-module-for-created-files");
@@ -417,7 +417,7 @@ public class JavacOptions extends ArrayList<String> {
      * Overrides or augments a module with classes and resources in JAR files or directories
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions patchModule(String module) {
         add("--patch-module");
@@ -441,7 +441,7 @@ public class JavacOptions extends ArrayList<String> {
      * Provide source compatibility with the specified Java SE release
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions source(int version) {
         add("--source");
@@ -453,7 +453,7 @@ public class JavacOptions extends ArrayList<String> {
      * Generate class files suitable for the specified Java SE release
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions target(int version) {
         add("--target");
@@ -645,7 +645,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify where to find input source files for multiple modules
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions moduleSourcePathStrings(Collection<String> paths) {
         return addPathOption("--module-source-path", paths);
@@ -655,7 +655,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify where to find input source files for multiple modules
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions moduleSourcePathPaths(Collection<Path> paths) {
         if (isNotEmpty(paths)) {
@@ -668,7 +668,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify where to find input source files for multiple modules
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions moduleSourcePath(Collection<File> paths) {
         if (isNotEmpty(paths)) {
@@ -827,7 +827,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify a module path where to find annotation processors
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions processorModulePathPaths(Collection<Path> paths) {
         if (isNotEmpty(paths)) {
@@ -840,7 +840,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify a module path where to find annotation processors
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions processorModulePathStrings(Collection<String> paths) {
         return addPathOption("--processor-module-path", paths);
@@ -850,7 +850,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify a module path where to find annotation processors
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions processorModulePath(Collection<File> paths) {
         if (isNotEmpty(paths)) {
@@ -902,7 +902,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify where to find annotation processors
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions processorPathStrings(Collection<String> paths) {
         return addPathOption("--processor-path", paths);
@@ -912,7 +912,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify where to find annotation processors
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions processorPath(Collection<File> paths) {
         if (isNotEmpty(paths)) {
@@ -925,7 +925,7 @@ public class JavacOptions extends ArrayList<String> {
      * Specify where to find annotation processors
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions processorPathPaths(Collection<Path> paths) {
         if (isNotEmpty(paths)) {
@@ -1036,7 +1036,7 @@ public class JavacOptions extends ArrayList<String> {
      * Override location of upgradeable modules
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions upgradeModulePathStrings(Collection<String> paths) {
         return addPathOption("--upgrade-module-path", paths);
@@ -1046,7 +1046,7 @@ public class JavacOptions extends ArrayList<String> {
      * Override location of upgradeable modules
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions upgradeModulePath(Collection<File> paths) {
         if (isNotEmpty(paths)) {
@@ -1059,7 +1059,7 @@ public class JavacOptions extends ArrayList<String> {
      * Override location of upgradeable modules
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions upgradeModulePathPaths(Collection<Path> paths) {
         if (isNotEmpty(paths)) {
@@ -1083,7 +1083,7 @@ public class JavacOptions extends ArrayList<String> {
      * Enable recommended warning categories
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions xLint() {
         add("-Xlint");
@@ -1094,7 +1094,7 @@ public class JavacOptions extends ArrayList<String> {
      * Warning categories to enable
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions xLint(XLintKey... keys) {
         if (isNotEmpty(keys)) {
@@ -1107,7 +1107,7 @@ public class JavacOptions extends ArrayList<String> {
      * Warning categories to enable
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions xLint(Collection<XLintKey> keys) {
         if (isNotEmpty(keys)) {
@@ -1120,7 +1120,7 @@ public class JavacOptions extends ArrayList<String> {
      * Warning categories to disable
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions xLintDisable(XLintKey... keys) {
         if (isNotEmpty(keys)) {
@@ -1133,7 +1133,7 @@ public class JavacOptions extends ArrayList<String> {
      * Warning categories to disable
      *
      * @return this list of options
-     * @since 2.3.1
+     * @since 2.4.0
      */
     public JavacOptions xLintDisable(Collection<XLintKey> keys) {
         if (isNotEmpty(keys)) {

src/main/java/rife/bld/operations/UpdatesOperation.java

@@ -31,16 +31,26 @@ public class UpdatesOperation extends AbstractOperation<UpdatesOperation> {
      */
     public void execute() {
         var resolution = new VersionResolution(properties());
-        var result = new DependencyScopes();
+
+        var scopes = new ArrayList<Scope>();
+        var dependencies = new ArrayList<Dependency>();
         for (var entry : dependencies_.entrySet()) {
-            var scope = entry.getKey();
             for (var dependency : entry.getValue()) {
-                var latest = new DependencyResolver(resolution, artifactRetriever(), repositories(), dependency).latestVersion();
-                if (latest.compareTo(dependency.version()) > 0) {
-                    var latest_dependency = new Dependency(dependency.groupId(), dependency.artifactId(), latest,
-                        dependency.classifier(), dependency.type());
-                    result.scope(scope).include(latest_dependency);
-                }
+                scopes.add(entry.getKey());
+                dependencies.add(dependency);
+            }
+        }
+
+        var latest_versions = new ParallelDependencyResolver(resolution, artifactRetriever(), repositories()).resolveLatestVersions(dependencies);
+
+        var result = new DependencyScopes();
+        for (var i = 0; i < dependencies.size(); ++i) {
+            var dependency = dependencies.get(i);
+            var latest = latest_versions.get(i);
+            if (latest.compareTo(dependency.version()) > 0) {
+                var latest_dependency = new Dependency(dependency.groupId(), dependency.artifactId(), latest,
+                    dependency.classifier(), dependency.type());
+                result.scope(scopes.get(i)).include(latest_dependency);
             }
         }
 

src/main/java/rife/bld/wrapper/WrapperExtensionResolver.java

@@ -89,12 +89,13 @@ public class WrapperExtensionResolver {
 
     private Set<String> transferExtensionDependencies() {
         var filenames = new HashSet<String>();
-        var dependencies = new DependencySet();
+        var roots = new ArrayList<Dependency>();
         for (var d : dependencies_) {
             if (d != null) {
-                dependencies.addAll(new DependencyResolver(resolution_, retriever_, repositories_, d).getAllDependencies(Scope.compile, Scope.runtime));
+                roots.add(d);
             }
         }
+        var dependencies = new ParallelDependencyResolver(resolution_, retriever_, repositories_).resolveAllDependencies(roots, Scope.compile, Scope.runtime);
         if (!dependencies.isEmpty()) {
             ensurePrintedHeader();
 

src/main/resources/BLD_VERSION

@@ -1 +1 @@
-2.3.1-SNAPSHOT
+2.4.0-SNAPSHOT

src/test/java/rife/bld/dependencies/TestDependencyResolver.java

@@ -4,15 +4,20 @@
  */
 package rife.bld.dependencies;
 
+import com.sun.net.httpserver.HttpServer;
 import org.junit.jupiter.api.Test;
 import rife.ioc.HierarchicalProperties;
 import rife.tools.FileUtils;
 import rife.tools.StringUtils;
 
 import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.jupiter.api.Assertions.*;
 import static rife.bld.dependencies.Dependency.CLASSIFIER_JAVADOC;
@@ -2341,4 +2346,98 @@ public class TestDependencyResolver {
             FileUtils.deleteDirectory(tmp2);
         }
     }
+
+    @Test
+    void testGetAllDependenciesParallelPomPrefetching() throws Exception {
+        var max_concurrent_retrievals = new AtomicInteger();
+        var server = createPomServer(max_concurrent_retrievals);
+        server.start();
+        try {
+            var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/"));
+            var root = new Dependency("com.example", "root", new VersionNumber(1, 0, 0));
+
+            // the caching retriever enables parallel POM prefetching
+            var resolver = new DependencyResolver(VersionResolution.dummy(), ArtifactRetriever.cachingInstance(), repositories, root);
+            assertEquals(StringUtils.convertLineSeparator("""
+                com.example:root:1.0.0
+                com.example:child1:1.0.0
+                com.example:child2:1.0.0
+                com.example:child3:1.0.0
+                com.example:child4:1.0.0
+                com.example:child5:1.0.0
+                com.example:child6:1.0.0
+                com.example:shared:1.0.0"""), StringUtils.join(resolver.getAllDependencies(compile), System.lineSeparator()));
+            assertTrue(max_concurrent_retrievals.get() > 1, "expected concurrent POM retrievals, max was " + max_concurrent_retrievals.get());
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    @Test
+    void testGetAllDependenciesSequentialWithoutCachingRetriever() throws Exception {
+        var max_concurrent_retrievals = new AtomicInteger();
+        var server = createPomServer(max_concurrent_retrievals);
+        server.start();
+        try {
+            var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/"));
+            var root = new Dependency("com.example", "root", new VersionNumber(1, 0, 0));
+
+            // the uncached retriever disables prefetching, retrievals stay sequential
+            var resolver = new DependencyResolver(VersionResolution.dummy(), ArtifactRetriever.instance(), repositories, root);
+            assertEquals(8, resolver.getAllDependencies(compile).size());
+            assertEquals(1, max_concurrent_retrievals.get(), "expected sequential POM retrievals, max was " + max_concurrent_retrievals.get());
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    private static HttpServer createPomServer(AtomicInteger maxConcurrentRetrievals)
+    throws IOException {
+        var graph = new HashMap<String, List<String>>();
+        graph.put("root", List.of("child1", "child2", "child3", "child4", "child5", "child6"));
+        graph.put("child1", List.of("shared"));
+
+        var active_retrievals = new AtomicInteger();
+        var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+        server.createContext("/", exchange -> {
+            var active = active_retrievals.incrementAndGet();
+            maxConcurrentRetrievals.accumulateAndGet(active, Math::max);
+            try {
+                // delay the response so that parallel retrievals overlap
+                Thread.sleep(100);
+
+                // serve the POM of the artifact in the request path
+                var segments = exchange.getRequestURI().getPath().split("/");
+                var artifact = segments[segments.length - 3];
+                var body = buildPom(artifact, graph.getOrDefault(artifact, List.of())).getBytes();
+                exchange.sendResponseHeaders(200, body.length);
+                exchange.getResponseBody().write(body);
+                exchange.close();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            } finally {
+                active_retrievals.decrementAndGet();
+            }
+        });
+        server.setExecutor(Executors.newCachedThreadPool());
+        return server;
+    }
+
+    private static String buildPom(String artifact, List<String> children) {
+        var dependencies = new StringBuilder();
+        for (var child : children) {
+            dependencies.append("<dependency><groupId>com.example</groupId><artifactId>")
+                .append(child)
+                .append("</artifactId><version>1.0.0</version></dependency>");
+        }
+        return """
+            <?xml version="1.0" encoding="UTF-8"?>
+            <project>
+                <modelVersion>4.0.0</modelVersion>
+                <groupId>com.example</groupId>
+                <artifactId>%s</artifactId>
+                <version>1.0.0</version>
+                <dependencies>%s</dependencies>
+            </project>""".formatted(artifact, dependencies);
+    }
 }

src/test/java/rife/bld/dependencies/TestDependencySet.java

@@ -5,11 +5,16 @@
 package rife.bld.dependencies;
 
 import org.junit.jupiter.api.Test;
+import rife.ioc.HierarchicalProperties;
+import rife.tools.FileUtils;
 import rife.tools.StringUtils;
 
+import java.nio.file.Files;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.jupiter.api.Assertions.*;
+import static rife.bld.dependencies.TransferTestHelper.*;
 import static rife.bld.dependencies.RepositoryTestHelper.getNextRepository;
 import static rife.bld.dependencies.Scope.compile;
 import static rife.bld.dependencies.Scope.runtime;
@@ -373,4 +378,50 @@ public class TestDependencySet {
                └─ org.json:json:20250107
             """), dependencies.generateTransitiveDependencyTree(VersionResolution.dummy(), ArtifactRetriever.instance(), RepositoryTestHelper.getNextRepositories(), compile, runtime));
     }
+
+    @Test
+    void testTransferIntoDirectoryParallel() throws Exception {
+        var max_concurrent_transfers = new AtomicInteger();
+        var server = createTransferServer(max_concurrent_transfers);
+        server.start();
+        var tmp = Files.createTempDirectory("transfers").toFile();
+        try {
+            var dependencies = createTransferDependencies(1, 6);
+            var repositories = List.of(transferRepository(server));
+
+            var artifacts = dependencies.transferIntoDirectory(new VersionResolution(null), ArtifactRetriever.instance(), repositories, tmp, tmp);
+
+            assertTransferredArtifacts(dependencies, artifacts, tmp);
+            assertTrue(max_concurrent_transfers.get() > 1, "expected concurrent transfers, max was " + max_concurrent_transfers.get());
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp);
+        }
+    }
+
+    @Test
+    void testTransferIntoDirectorySequential() throws Exception {
+        var max_concurrent_transfers = new AtomicInteger();
+        var server = createTransferServer(max_concurrent_transfers);
+        server.start();
+        var tmp = Files.createTempDirectory("transfers").toFile();
+        try {
+            var properties = new HierarchicalProperties();
+            properties.put(VersionResolution.PROPERTY_TRANSFER_PARALLELISM, "1");
+            var resolution = new VersionResolution(properties);
+            assertEquals(1, resolution.transferParallelism());
+
+            var dependencies = createTransferDependencies(1, 6);
+            var repositories = List.of(transferRepository(server));
+
+            var artifacts = dependencies.transferIntoDirectory(resolution, ArtifactRetriever.instance(), repositories, tmp, tmp);
+
+            assertTransferredArtifacts(dependencies, artifacts, tmp);
+            assertEquals(1, max_concurrent_transfers.get(), "expected sequential transfers, max was " + max_concurrent_transfers.get());
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp);
+        }
+    }
+
 }

src/test/java/rife/bld/dependencies/TestDependencyTransferBatch.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import org.junit.jupiter.api.Test;
+import rife.bld.dependencies.exceptions.DependencyTransferException;
+import rife.tools.FileUtils;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static rife.bld.dependencies.TransferTestHelper.*;
+
+public class TestDependencyTransferBatch {
+    @Test
+    void testTransferAcrossSets() throws Exception {
+        var max_concurrent_transfers = new AtomicInteger();
+        var server = createTransferServer(max_concurrent_transfers);
+        server.start();
+        var tmp1 = Files.createTempDirectory("transfers1").toFile();
+        var tmp2 = Files.createTempDirectory("transfers2").toFile();
+        try {
+            var set1 = createTransferDependencies(1, 3);
+            var set2 = createTransferDependencies(4, 6);
+            var repositories = List.of(transferRepository(server));
+
+            var batch = new DependencyTransferBatch();
+            batch.add(set1, tmp1, tmp1)
+                 .add(set2, tmp2, tmp2)
+                 // identical transfers into the same directory are only performed once
+                 .add(set1, tmp1, tmp1);
+            var artifacts = batch.transfer(new VersionResolution(null), ArtifactRetriever.instance(), repositories);
+
+            assertEquals(6, artifacts.size());
+            assertTransferredArtifacts(set1, artifacts.subList(0, 3), tmp1);
+            assertTransferredArtifacts(set2, artifacts.subList(3, 6), tmp2);
+            assertTrue(max_concurrent_transfers.get() > 1, "expected concurrent transfers, max was " + max_concurrent_transfers.get());
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp1);
+            FileUtils.deleteDirectory(tmp2);
+        }
+    }
+
+    @Test
+    void testTransferEmptiesTheBatch() throws Exception {
+        var server = createTransferServer(new AtomicInteger());
+        server.start();
+        var tmp = Files.createTempDirectory("transfers").toFile();
+        try {
+            var repositories = List.of(transferRepository(server));
+            var resolution = new VersionResolution(null);
+            var batch = new DependencyTransferBatch();
+
+            var set1 = createTransferDependencies(1, 3);
+            batch.add(set1, tmp, tmp);
+            assertTransferredArtifacts(set1, batch.transfer(resolution, ArtifactRetriever.instance(), repositories), tmp);
+
+            // the batch was emptied, nothing is transferred again
+            assertTrue(batch.transfer(resolution, ArtifactRetriever.instance(), repositories).isEmpty());
+
+            // the same instance can collect and transfer again,
+            // including targets that were transferred before
+            var set2 = createTransferDependencies(3, 5);
+            batch.add(set2, tmp, tmp);
+            assertTransferredArtifacts(set2, batch.transfer(resolution, ArtifactRetriever.instance(), repositories), tmp);
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp);
+        }
+    }
+
+    @Test
+    void testTransferModularJarsIntoModulesDirectory() throws Exception {
+        var server = createTransferServer(new AtomicInteger());
+        server.start();
+        var artifacts_dir = Files.createTempDirectory("artifacts").toFile();
+        var modules_dir = Files.createTempDirectory("modules").toFile();
+        try {
+            var dependencies = new DependencySet()
+                .include(new Dependency("com.example", "artifact1", new VersionNumber(1, 0, 0)))
+                .include(new Dependency("com.example", "module1", new VersionNumber(1, 0, 0), "", Dependency.TYPE_MODULAR_JAR));
+
+            var artifacts = new DependencyTransferBatch()
+                .add(dependencies, artifacts_dir, modules_dir)
+                .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of(transferRepository(server)));
+
+            assertEquals(2, artifacts.size());
+            assertTrue(new File(artifacts_dir, "artifact1-1.0.0.jar").exists());
+            assertTrue(new File(modules_dir, "module1-1.0.0.jar").exists());
+            assertFalse(new File(artifacts_dir, "module1-1.0.0.jar").exists());
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(artifacts_dir);
+            FileUtils.deleteDirectory(modules_dir);
+        }
+    }
+
+    @Test
+    void testTransferClassifiers() throws Exception {
+        var server = createTransferServer(new AtomicInteger());
+        server.start();
+        var tmp = Files.createTempDirectory("transfers").toFile();
+        try {
+            var dependencies = createTransferDependencies(1, 2);
+            var artifacts = new DependencyTransferBatch()
+                .add(dependencies, tmp, tmp, "sources")
+                .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of(transferRepository(server)));
+
+            assertEquals(4, artifacts.size());
+            for (var i = 1; i <= 2; i++) {
+                assertTrue(new File(tmp, "artifact" + i + "-1.0.0.jar").exists());
+                assertTrue(new File(tmp, "artifact" + i + "-1.0.0-sources.jar").exists());
+            }
+        } finally {
+            server.stop(0);
+            FileUtils.deleteDirectory(tmp);
+        }
+    }
+
+    @Test
+    void testMissingDirectories() {
+        var regular = new DependencySet()
+            .include(new Dependency("com.example", "artifact1", new VersionNumber(1, 0, 0)));
+        var modular = new DependencySet()
+            .include(new Dependency("com.example", "module1", new VersionNumber(1, 0, 0), "", Dependency.TYPE_MODULAR_JAR));
+
+        assertThrows(DependencyTransferException.class, () -> new DependencyTransferBatch().add(regular, null, null));
+        assertThrows(DependencyTransferException.class, () -> new DependencyTransferBatch().add(modular, Files.createTempDirectory("artifacts").toFile(), null));
+    }
+
+    @Test
+    void testEmptyBatch() {
+        var artifacts = new DependencyTransferBatch()
+            .transfer(new VersionResolution(null), ArtifactRetriever.instance(), List.of());
+        assertTrue(artifacts.isEmpty());
+    }
+}

src/test/java/rife/bld/dependencies/TransferTestHelper.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import com.sun.net.httpserver.HttpServer;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Provides a local artifact server and dependency fixtures for testing
+ * transfers without relying on remote repositories.
+ */
+abstract class TransferTestHelper {
+    static HttpServer createTransferServer(AtomicInteger maxConcurrentTransfers)
+    throws IOException {
+        var active_transfers = new AtomicInteger();
+        var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+        server.createContext("/", exchange -> {
+            var active = active_transfers.incrementAndGet();
+            maxConcurrentTransfers.accumulateAndGet(active, Math::max);
+            try {
+                // delay the response so that parallel transfers overlap
+                Thread.sleep(200);
+
+                var body = exchange.getRequestURI().getPath().getBytes();
+                exchange.sendResponseHeaders(200, body.length);
+                exchange.getResponseBody().write(body);
+                exchange.close();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            } finally {
+                active_transfers.decrementAndGet();
+            }
+        });
+        server.setExecutor(Executors.newCachedThreadPool());
+        return server;
+    }
+
+    static Repository transferRepository(HttpServer server) {
+        return new Repository("http://localhost:" + server.getAddress().getPort() + "/");
+    }
+
+    static DependencySet createTransferDependencies(int from, int to) {
+        var dependencies = new DependencySet();
+        for (var i = from; i <= to; i++) {
+            dependencies.include(new Dependency("com.example", "artifact" + i, new VersionNumber(1, 0, 0)));
+        }
+        return dependencies;
+    }
+
+    static void assertTransferredArtifacts(DependencySet dependencies, List<RepositoryArtifact> artifacts, File directory) {
+        assertEquals(dependencies.size(), artifacts.size());
+        var index = 0;
+        for (var dependency : dependencies) {
+            var filename = dependency.artifactId() + "-" + dependency.version() + ".jar";
+            assertTrue(artifacts.get(index).location().endsWith(filename), "expected artifact " + filename + " at index " + index);
+            assertTrue(new File(directory, filename).exists(), "expected file " + filename + " to be transferred");
+            ++index;
+        }
+    }
+}