Parallelize transitive dependency resolution with speculative POM prefetching

src/main/java/rife/bld/dependencies/ArtifactRetriever.java

@@ -45,6 +45,10 @@ public abstract class ArtifactRetriever {
 
         void cache(RepositoryArtifact artifact, String content) {
         }
+
+        boolean isCaching() {
+            return false;
+        }
     };
 
     /**
@@ -76,6 +80,9 @@ public abstract class ArtifactRetriever {
                 artifactCache.put(artifact, content);
             }
 
+            boolean isCaching() {
+                return true;
+            }
         };
     }
 
@@ -86,6 +93,8 @@ public abstract class ArtifactRetriever {
 
     abstract void cache(RepositoryArtifact artifact, String content);
 
+    abstract boolean isCaching();
+
     /**
      * Reads the contents of an artifact as a string.
      *

src/main/java/rife/bld/dependencies/DependencyResolver.java

@@ -9,6 +9,7 @@ import rife.tools.exceptions.FileUtilsErrorException;
 
 import java.io.*;
 import java.util.*;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static rife.bld.dependencies.Dependency.*;
@@ -121,6 +122,17 @@ public class DependencyResolver {
      * @since 1.5
      */
     public DependencySet getAllDependencies(Scope... scopes) {
+        var prefetcher = PomPrefetcher.create(resolution_, retriever_, repositories_);
+        try {
+            return getAllDependencies(prefetcher, scopes);
+        } finally {
+            if (prefetcher != null) {
+                prefetcher.shutdown();
+            }
+        }
+    }
+
+    DependencySet getAllDependencies(PomPrefetcher prefetcher, Scope... scopes) {
         var result = new DependencySet();
         var overridden = resolution_.overrideDependency(dependency_);
         result.add(overridden);
@@ -138,6 +150,11 @@ public class DependencyResolver {
             next_dependencies.removeIf(it -> matchesExclusions(exclusion_context, it));
             // add all next dependencies to the queue
             dependency_queue.addAll(next_dependencies);
+            // speculatively retrieve the POMs of the queued dependencies in
+            // parallel so that they are cached when they're processed in order
+            if (prefetcher != null) {
+                prefetcher.prefetch(next_dependencies);
+            }
 
             // unless we find a next set of dependencies to add, stop resolving
             parent = null;
@@ -163,6 +180,46 @@ public class DependencyResolver {
         return result;
     }
 
+    /**
+     * Resolves the transitive dependencies of multiple root dependencies,
+     * merging the results in the order of the provided roots.
+     * <p>
+     * The roots are resolved in parallel according to the resolution
+     * parallelism, while a shared {@code PomPrefetcher} speculatively warms
+     * the retriever cache across all of them. The merged result is identical
+     * to resolving each root sequentially.
+     *
+     * @param resolution   the version resolution state that can be cached
+     * @param retriever    the retriever to use to get artifacts
+     * @param repositories the repositories to use for the resolution
+     * @param roots        the root dependencies to resolve
+     * @param scopes       the scopes to return the transitive dependencies for
+     * @return the merged transitive dependencies of all the roots
+     * @since 2.3.1
+     */
+    static DependencySet resolveAllDependencies(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories, Collection<Dependency> roots, Scope... scopes) {
+        var result = new DependencySet();
+        if (roots.isEmpty()) {
+            return result;
+        }
+
+        var prefetcher = PomPrefetcher.create(resolution, retriever, repositories);
+        try {
+            var resolutions = new ArrayList<Supplier<DependencySet>>(roots.size());
+            for (var root : roots) {
+                resolutions.add(() -> new DependencyResolver(resolution, retriever, repositories, root).getAllDependencies(prefetcher, scopes));
+            }
+            for (var dependencies : ParallelExecution.execute(resolutions, resolution.resolutionParallelism())) {
+                result.addAll(dependencies);
+            }
+        } finally {
+            if (prefetcher != null) {
+                prefetcher.shutdown();
+            }
+        }
+        return result;
+    }
+
     private boolean matchesExclusions(Dependency context, PomDependency checked) {
         while (context != null) {
             if (context.exclusions() != null) {

src/main/java/rife/bld/dependencies/DependencyScopes.java

@@ -6,6 +6,7 @@ package rife.bld.dependencies;
 
 import rife.ioc.HierarchicalProperties;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 
@@ -147,15 +148,14 @@ public class DependencyScopes extends LinkedHashMap<Scope, DependencySet> {
 
     private DependencySet resolveScopedDependencies(HierarchicalProperties properties, ArtifactRetriever retriever, List<Repository> repositories, Scope[] resolvedScopes, Scope[] transitiveScopes, DependencySet excluded) {
         var resolution = new VersionResolution(properties);
-        var dependencies = new DependencySet();
+        var roots = new ArrayList<Dependency>();
         for (var scope : resolvedScopes) {
             var scoped_dependencies = get(scope);
             if (scoped_dependencies != null) {
-                for (var dependency : scoped_dependencies) {
-                    dependencies.addAll(new DependencyResolver(resolution, retriever, repositories, dependency).getAllDependencies(transitiveScopes));
-                }
+                roots.addAll(scoped_dependencies);
             }
         }
+        var dependencies = DependencyResolver.resolveAllDependencies(resolution, retriever, repositories, roots, transitiveScopes);
         if (excluded != null) {
             dependencies.removeAll(excluded);
         }

src/main/java/rife/bld/dependencies/DependencySet.java

@@ -8,9 +8,6 @@ import rife.bld.dependencies.exceptions.DependencyTransferException;
 
 import java.io.File;
 import java.util.*;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 import java.util.function.Supplier;
 
 /**
@@ -195,39 +192,8 @@ public class DependencySet extends AbstractSet<Dependency> implements Set<Depend
 
     private static List<RepositoryArtifact> executeTransfers(List<Supplier<List<RepositoryArtifact>>> transfers, int transferParallelism) {
         var result = new ArrayList<RepositoryArtifact>();
-
-        var parallelism = Math.min(transfers.size(), transferParallelism);
-        if (parallelism <= 1) {
-            for (var transfer : transfers) {
-                result.addAll(transfer.get());
-            }
-            return result;
-        }
-
-        var executor = Executors.newFixedThreadPool(parallelism);
-        try {
-            var futures = new ArrayList<Future<List<RepositoryArtifact>>>();
-            for (var transfer : transfers) {
-                futures.add(executor.submit(transfer::get));
-            }
-            for (var future : futures) {
-                try {
-                    result.addAll(future.get());
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                    throw new IllegalStateException("Artifact transfer was interrupted", e);
-                } catch (ExecutionException e) {
-                    if (e.getCause() instanceof RuntimeException runtime) {
-                        throw runtime;
-                    }
-                    if (e.getCause() instanceof Error error) {
-                        throw error;
-                    }
-                    throw new IllegalStateException(e.getCause());
-                }
-            }
-        } finally {
-            executor.shutdownNow();
+        for (var artifacts : ParallelExecution.execute(transfers, transferParallelism)) {
+            result.addAll(artifacts);
         }
         return result;
     }
@@ -260,11 +226,7 @@ public class DependencySet extends AbstractSet<Dependency> implements Set<Depend
      * @since 2.0
      */
     public String generateTransitiveDependencyTree(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories, Scope... scopes) {
-        var compile_dependencies = new DependencySet();
-        for (var dependency : this) {
-            compile_dependencies.addAll(new DependencyResolver(resolution, retriever, repositories, dependency).getAllDependencies(scopes));
-        }
-        return compile_dependencies.generateDependencyTree();
+        return DependencyResolver.resolveAllDependencies(resolution, retriever, repositories, this, scopes).generateDependencyTree();
     }
 
     /**

src/main/java/rife/bld/dependencies/ParallelExecution.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Supplier;
+
+/**
+ * Executes tasks in parallel while preserving the order of their results
+ * and the sequential semantics of failures: the first task that fails in
+ * order will have its exception rethrown.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.3.1
+ */
+final class ParallelExecution {
+    private ParallelExecution() {
+    }
+
+    static <T> List<T> execute(List<Supplier<T>> tasks, int parallelism) {
+        var result = new ArrayList<T>(tasks.size());
+
+        parallelism = Math.min(tasks.size(), parallelism);
+        if (parallelism <= 1) {
+            for (var task : tasks) {
+                result.add(task.get());
+            }
+            return result;
+        }
+
+        var executor = Executors.newFixedThreadPool(parallelism);
+        try {
+            var futures = new ArrayList<Future<T>>(tasks.size());
+            for (var task : tasks) {
+                futures.add(executor.submit(task::get));
+            }
+            for (var future : futures) {
+                try {
+                    result.add(future.get());
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    throw new IllegalStateException("Parallel execution was interrupted", e);
+                } catch (ExecutionException e) {
+                    if (e.getCause() instanceof RuntimeException runtime) {
+                        throw runtime;
+                    }
+                    if (e.getCause() instanceof Error error) {
+                        throw error;
+                    }
+                    throw new IllegalStateException(e.getCause());
+                }
+            }
+        } finally {
+            executor.shutdownNow();
+        }
+        return result;
+    }
+}

src/main/java/rife/bld/dependencies/PomPrefetcher.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com)
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ */
+package rife.bld.dependencies;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Speculatively retrieves POMs in parallel so that they are already cached
+ * by the artifact retriever when the sequential dependency resolution
+ * processes them, without influencing the resolution semantics.
+ * <p>
+ * A single instance can be shared by multiple concurrent resolutions, the
+ * POM of each unique dependency will only be prefetched once.
+ *
+ * @author Geert Bevin (gbevin[remove] at uwyn dot com)
+ * @since 2.3.1
+ */
+class PomPrefetcher {
+    private final VersionResolution resolution_;
+    private final ArtifactRetriever retriever_;
+    private final List<Repository> repositories_;
+    private final ExecutorService executor_;
+    private final Set<Dependency> submitted_ = ConcurrentHashMap.newKeySet();
+
+    /**
+     * Creates a prefetcher when it can be beneficial.
+     *
+     * @return the prefetcher; or {@code null} when the retriever doesn't
+     * cache the retrieved POMs or when the resolution parallelism disables it
+     * @since 2.3.1
+     */
+    static PomPrefetcher create(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories) {
+        // prefetching only has benefits when the retrieved POMs are
+        // cached for the sequential resolution that follows
+        if (!retriever.isCaching() || resolution.resolutionParallelism() <= 1) {
+            return null;
+        }
+        return new PomPrefetcher(resolution, retriever, repositories);
+    }
+
+    private PomPrefetcher(VersionResolution resolution, ArtifactRetriever retriever, List<Repository> repositories) {
+        resolution_ = resolution;
+        retriever_ = retriever;
+        repositories_ = repositories;
+        executor_ = Executors.newFixedThreadPool(resolution.resolutionParallelism());
+    }
+
+    void prefetch(Collection<PomDependency> candidates) {
+        for (var candidate : candidates) {
+            var dependency = resolution_.overrideDependency(candidate.convertToDependency());
+            if (submitted_.add(dependency)) {
+                executor_.submit(() -> {
+                    try {
+                        new DependencyResolver(resolution_, retriever_, repositories_, dependency).getMavenPom(dependency);
+                    } catch (Throwable e) {
+                        // failures are ignored since they will resurface
+                        // with the proper context when the dependency is
+                        // resolved in order
+                    }
+                });
+            }
+        }
+    }
+
+    void shutdown() {
+        executor_.shutdownNow();
+    }
+}

src/main/java/rife/bld/dependencies/VersionResolution.java

@@ -49,8 +49,18 @@ public class VersionResolution {
     public static final String PROPERTY_TRANSFER_PARALLELISM = "bld.transferParallelism";
     private static final int DEFAULT_TRANSFER_PARALLELISM = 6;
 
+    /**
+     * The property key that determines how many POMs are speculatively
+     * retrieved in parallel during transitive dependency resolution,
+     * {@code 1} disables the parallel retrieval.
+     * @since 2.3.1
+     */
+    public static final String PROPERTY_RESOLUTION_PARALLELISM = "bld.resolutionParallelism";
+    private static final int DEFAULT_RESOLUTION_PARALLELISM = 6;
+
     private final Map<String, Version> versionOverrides_ = new HashMap<>();
     private final int transferParallelism_;
+    private final int resolutionParallelism_;
 
     /**
      * Returns a dummy {@code VersionResolution} instance that doesn't override anything.
@@ -73,7 +83,6 @@ public class VersionResolution {
      * @since 2.0
      */
     public VersionResolution(HierarchicalProperties properties) {
-        var transfer_parallelism = DEFAULT_TRANSFER_PARALLELISM;
         if (properties != null) {
             for (var name : properties.getNames()) {
                 if (name.startsWith(PROPERTY_OVERRIDE_PREFIX)) {
@@ -88,17 +97,23 @@ public class VersionResolution {
                     }
                 }
             }
+        }
+        transferParallelism_ = parseParallelism(properties, PROPERTY_TRANSFER_PARALLELISM, DEFAULT_TRANSFER_PARALLELISM);
+        resolutionParallelism_ = parseParallelism(properties, PROPERTY_RESOLUTION_PARALLELISM, DEFAULT_RESOLUTION_PARALLELISM);
+    }
 
-            var parallelism = properties.getValueString(PROPERTY_TRANSFER_PARALLELISM);
+    private static int parseParallelism(HierarchicalProperties properties, String property, int defaultValue) {
+        if (properties != null) {
+            var parallelism = properties.getValueString(property);
             if (parallelism != null && !parallelism.isBlank()) {
                 try {
-                    transfer_parallelism = Math.max(1, Integer.parseInt(parallelism.trim()));
+                    return Math.max(1, Integer.parseInt(parallelism.trim()));
                 } catch (NumberFormatException e) {
-                    Logger.getLogger("rife.bld").warning("Unable to parse the " + PROPERTY_TRANSFER_PARALLELISM + " property as an integer: '" + parallelism + "', using " + transfer_parallelism + " instead");
+                    Logger.getLogger("rife.bld").warning("Unable to parse the " + property + " property as an integer: '" + parallelism + "', using " + defaultValue + " instead");
                 }
             }
         }
-        transferParallelism_ = transfer_parallelism;
+        return defaultValue;
     }
 
     /**
@@ -158,4 +173,16 @@ public class VersionResolution {
     public int transferParallelism() {
         return transferParallelism_;
     }
+
+    /**
+     * Returns the number of POMs that are speculatively retrieved in parallel
+     * during transitive dependency resolution, {@code 1} means the parallel
+     * retrieval is disabled.
+     *
+     * @return the number of parallel POM retrievals
+     * @since 2.3.1
+     */
+    public int resolutionParallelism() {
+        return resolutionParallelism_;
+    }
 }

src/test/java/rife/bld/dependencies/TestDependencyResolver.java

@@ -4,15 +4,20 @@
  */
 package rife.bld.dependencies;
 
+import com.sun.net.httpserver.HttpServer;
 import org.junit.jupiter.api.Test;
 import rife.ioc.HierarchicalProperties;
 import rife.tools.FileUtils;
 import rife.tools.StringUtils;
 
 import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.jupiter.api.Assertions.*;
 import static rife.bld.dependencies.Dependency.CLASSIFIER_JAVADOC;
@@ -2341,4 +2346,98 @@ public class TestDependencyResolver {
             FileUtils.deleteDirectory(tmp2);
         }
     }
+
+    @Test
+    void testGetAllDependenciesParallelPomPrefetching() throws Exception {
+        var max_concurrent_retrievals = new AtomicInteger();
+        var server = createPomServer(max_concurrent_retrievals);
+        server.start();
+        try {
+            var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/"));
+            var root = new Dependency("com.example", "root", new VersionNumber(1, 0, 0));
+
+            // the caching retriever enables parallel POM prefetching
+            var resolver = new DependencyResolver(VersionResolution.dummy(), ArtifactRetriever.cachingInstance(), repositories, root);
+            assertEquals(StringUtils.convertLineSeparator("""
+                com.example:root:1.0.0
+                com.example:child1:1.0.0
+                com.example:child2:1.0.0
+                com.example:child3:1.0.0
+                com.example:child4:1.0.0
+                com.example:child5:1.0.0
+                com.example:child6:1.0.0
+                com.example:shared:1.0.0"""), StringUtils.join(resolver.getAllDependencies(compile), System.lineSeparator()));
+            assertTrue(max_concurrent_retrievals.get() > 1, "expected concurrent POM retrievals, max was " + max_concurrent_retrievals.get());
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    @Test
+    void testGetAllDependenciesSequentialWithoutCachingRetriever() throws Exception {
+        var max_concurrent_retrievals = new AtomicInteger();
+        var server = createPomServer(max_concurrent_retrievals);
+        server.start();
+        try {
+            var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/"));
+            var root = new Dependency("com.example", "root", new VersionNumber(1, 0, 0));
+
+            // the uncached retriever disables prefetching, retrievals stay sequential
+            var resolver = new DependencyResolver(VersionResolution.dummy(), ArtifactRetriever.instance(), repositories, root);
+            assertEquals(8, resolver.getAllDependencies(compile).size());
+            assertEquals(1, max_concurrent_retrievals.get(), "expected sequential POM retrievals, max was " + max_concurrent_retrievals.get());
+        } finally {
+            server.stop(0);
+        }
+    }
+
+    private static HttpServer createPomServer(AtomicInteger maxConcurrentRetrievals)
+    throws IOException {
+        var graph = new HashMap<String, List<String>>();
+        graph.put("root", List.of("child1", "child2", "child3", "child4", "child5", "child6"));
+        graph.put("child1", List.of("shared"));
+
+        var active_retrievals = new AtomicInteger();
+        var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+        server.createContext("/", exchange -> {
+            var active = active_retrievals.incrementAndGet();
+            maxConcurrentRetrievals.accumulateAndGet(active, Math::max);
+            try {
+                // delay the response so that parallel retrievals overlap
+                Thread.sleep(100);
+
+                // serve the POM of the artifact in the request path
+                var segments = exchange.getRequestURI().getPath().split("/");
+                var artifact = segments[segments.length - 3];
+                var body = buildPom(artifact, graph.getOrDefault(artifact, List.of())).getBytes();
+                exchange.sendResponseHeaders(200, body.length);
+                exchange.getResponseBody().write(body);
+                exchange.close();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            } finally {
+                active_retrievals.decrementAndGet();
+            }
+        });
+        server.setExecutor(Executors.newCachedThreadPool());
+        return server;
+    }
+
+    private static String buildPom(String artifact, List<String> children) {
+        var dependencies = new StringBuilder();
+        for (var child : children) {
+            dependencies.append("<dependency><groupId>com.example</groupId><artifactId>")
+                .append(child)
+                .append("</artifactId><version>1.0.0</version></dependency>");
+        }
+        return """
+            <?xml version="1.0" encoding="UTF-8"?>
+            <project>
+                <modelVersion>4.0.0</modelVersion>
+                <groupId>com.example</groupId>
+                <artifactId>%s</artifactId>
+                <version>1.0.0</version>
+                <dependencies>%s</dependencies>
+            </project>""".formatted(artifact, dependencies);
+    }
 }