diff --git a/src/main/java/rife/bld/dependencies/ArtifactRetriever.java b/src/main/java/rife/bld/dependencies/ArtifactRetriever.java index aa69070..f477900 100644 --- a/src/main/java/rife/bld/dependencies/ArtifactRetriever.java +++ b/src/main/java/rife/bld/dependencies/ArtifactRetriever.java @@ -45,6 +45,10 @@ public abstract class ArtifactRetriever { void cache(RepositoryArtifact artifact, String content) { } + + boolean isCaching() { + return false; + } }; /** @@ -76,6 +80,9 @@ public abstract class ArtifactRetriever { artifactCache.put(artifact, content); } + boolean isCaching() { + return true; + } }; } @@ -86,6 +93,8 @@ public abstract class ArtifactRetriever { abstract void cache(RepositoryArtifact artifact, String content); + abstract boolean isCaching(); + /** * Reads the contents of an artifact as a string. * diff --git a/src/main/java/rife/bld/dependencies/DependencyResolver.java b/src/main/java/rife/bld/dependencies/DependencyResolver.java index b671f90..d9c0c1c 100644 --- a/src/main/java/rife/bld/dependencies/DependencyResolver.java +++ b/src/main/java/rife/bld/dependencies/DependencyResolver.java @@ -9,6 +9,7 @@ import rife.tools.exceptions.FileUtilsErrorException; import java.io.*; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; import static rife.bld.dependencies.Dependency.*; @@ -121,6 +122,17 @@ public class DependencyResolver { * @since 1.5 */ public DependencySet getAllDependencies(Scope... scopes) { + var prefetcher = PomPrefetcher.create(resolution_, retriever_, repositories_); + try { + return getAllDependencies(prefetcher, scopes); + } finally { + if (prefetcher != null) { + prefetcher.shutdown(); + } + } + } + + DependencySet getAllDependencies(PomPrefetcher prefetcher, Scope... scopes) { var result = new DependencySet(); var overridden = resolution_.overrideDependency(dependency_); result.add(overridden); @@ -138,6 +150,11 @@ public class DependencyResolver { next_dependencies.removeIf(it -> matchesExclusions(exclusion_context, it)); // add all next dependencies to the queue dependency_queue.addAll(next_dependencies); + // speculatively retrieve the POMs of the queued dependencies in + // parallel so that they are cached when they're processed in order + if (prefetcher != null) { + prefetcher.prefetch(next_dependencies); + } // unless we find a next set of dependencies to add, stop resolving parent = null; @@ -163,6 +180,46 @@ public class DependencyResolver { return result; } + /** + * Resolves the transitive dependencies of multiple root dependencies, + * merging the results in the order of the provided roots. + *

+ * The roots are resolved in parallel according to the resolution + * parallelism, while a shared {@code PomPrefetcher} speculatively warms + * the retriever cache across all of them. The merged result is identical + * to resolving each root sequentially. + * + * @param resolution the version resolution state that can be cached + * @param retriever the retriever to use to get artifacts + * @param repositories the repositories to use for the resolution + * @param roots the root dependencies to resolve + * @param scopes the scopes to return the transitive dependencies for + * @return the merged transitive dependencies of all the roots + * @since 2.3.1 + */ + static DependencySet resolveAllDependencies(VersionResolution resolution, ArtifactRetriever retriever, List repositories, Collection roots, Scope... scopes) { + var result = new DependencySet(); + if (roots.isEmpty()) { + return result; + } + + var prefetcher = PomPrefetcher.create(resolution, retriever, repositories); + try { + var resolutions = new ArrayList>(roots.size()); + for (var root : roots) { + resolutions.add(() -> new DependencyResolver(resolution, retriever, repositories, root).getAllDependencies(prefetcher, scopes)); + } + for (var dependencies : ParallelExecution.execute(resolutions, resolution.resolutionParallelism())) { + result.addAll(dependencies); + } + } finally { + if (prefetcher != null) { + prefetcher.shutdown(); + } + } + return result; + } + private boolean matchesExclusions(Dependency context, PomDependency checked) { while (context != null) { if (context.exclusions() != null) { diff --git a/src/main/java/rife/bld/dependencies/DependencyScopes.java b/src/main/java/rife/bld/dependencies/DependencyScopes.java index d261858..7f0c8f2 100644 --- a/src/main/java/rife/bld/dependencies/DependencyScopes.java +++ b/src/main/java/rife/bld/dependencies/DependencyScopes.java @@ -6,6 +6,7 @@ package rife.bld.dependencies; import rife.ioc.HierarchicalProperties; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -147,15 +148,14 @@ public class DependencyScopes extends LinkedHashMap { private DependencySet resolveScopedDependencies(HierarchicalProperties properties, ArtifactRetriever retriever, List repositories, Scope[] resolvedScopes, Scope[] transitiveScopes, DependencySet excluded) { var resolution = new VersionResolution(properties); - var dependencies = new DependencySet(); + var roots = new ArrayList(); for (var scope : resolvedScopes) { var scoped_dependencies = get(scope); if (scoped_dependencies != null) { - for (var dependency : scoped_dependencies) { - dependencies.addAll(new DependencyResolver(resolution, retriever, repositories, dependency).getAllDependencies(transitiveScopes)); - } + roots.addAll(scoped_dependencies); } } + var dependencies = DependencyResolver.resolveAllDependencies(resolution, retriever, repositories, roots, transitiveScopes); if (excluded != null) { dependencies.removeAll(excluded); } diff --git a/src/main/java/rife/bld/dependencies/DependencySet.java b/src/main/java/rife/bld/dependencies/DependencySet.java index 813817d..a62bd10 100644 --- a/src/main/java/rife/bld/dependencies/DependencySet.java +++ b/src/main/java/rife/bld/dependencies/DependencySet.java @@ -8,9 +8,6 @@ import rife.bld.dependencies.exceptions.DependencyTransferException; import java.io.File; import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.function.Supplier; /** @@ -195,39 +192,8 @@ public class DependencySet extends AbstractSet implements Set executeTransfers(List>> transfers, int transferParallelism) { var result = new ArrayList(); - - var parallelism = Math.min(transfers.size(), transferParallelism); - if (parallelism <= 1) { - for (var transfer : transfers) { - result.addAll(transfer.get()); - } - return result; - } - - var executor = Executors.newFixedThreadPool(parallelism); - try { - var futures = new ArrayList>>(); - for (var transfer : transfers) { - futures.add(executor.submit(transfer::get)); - } - for (var future : futures) { - try { - result.addAll(future.get()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Artifact transfer was interrupted", e); - } catch (ExecutionException e) { - if (e.getCause() instanceof RuntimeException runtime) { - throw runtime; - } - if (e.getCause() instanceof Error error) { - throw error; - } - throw new IllegalStateException(e.getCause()); - } - } - } finally { - executor.shutdownNow(); + for (var artifacts : ParallelExecution.execute(transfers, transferParallelism)) { + result.addAll(artifacts); } return result; } @@ -260,11 +226,7 @@ public class DependencySet extends AbstractSet implements Set repositories, Scope... scopes) { - var compile_dependencies = new DependencySet(); - for (var dependency : this) { - compile_dependencies.addAll(new DependencyResolver(resolution, retriever, repositories, dependency).getAllDependencies(scopes)); - } - return compile_dependencies.generateDependencyTree(); + return DependencyResolver.resolveAllDependencies(resolution, retriever, repositories, this, scopes).generateDependencyTree(); } /** diff --git a/src/main/java/rife/bld/dependencies/ParallelExecution.java b/src/main/java/rife/bld/dependencies/ParallelExecution.java new file mode 100644 index 0000000..86dd7ca --- /dev/null +++ b/src/main/java/rife/bld/dependencies/ParallelExecution.java @@ -0,0 +1,64 @@ +/* + * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com) + * Licensed under the Apache License, Version 2.0 (the "License") + */ +package rife.bld.dependencies; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Supplier; + +/** + * Executes tasks in parallel while preserving the order of their results + * and the sequential semantics of failures: the first task that fails in + * order will have its exception rethrown. + * + * @author Geert Bevin (gbevin[remove] at uwyn dot com) + * @since 2.3.1 + */ +final class ParallelExecution { + private ParallelExecution() { + } + + static List execute(List> tasks, int parallelism) { + var result = new ArrayList(tasks.size()); + + parallelism = Math.min(tasks.size(), parallelism); + if (parallelism <= 1) { + for (var task : tasks) { + result.add(task.get()); + } + return result; + } + + var executor = Executors.newFixedThreadPool(parallelism); + try { + var futures = new ArrayList>(tasks.size()); + for (var task : tasks) { + futures.add(executor.submit(task::get)); + } + for (var future : futures) { + try { + result.add(future.get()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Parallel execution was interrupted", e); + } catch (ExecutionException e) { + if (e.getCause() instanceof RuntimeException runtime) { + throw runtime; + } + if (e.getCause() instanceof Error error) { + throw error; + } + throw new IllegalStateException(e.getCause()); + } + } + } finally { + executor.shutdownNow(); + } + return result; + } +} diff --git a/src/main/java/rife/bld/dependencies/PomPrefetcher.java b/src/main/java/rife/bld/dependencies/PomPrefetcher.java new file mode 100644 index 0000000..fabbc45 --- /dev/null +++ b/src/main/java/rife/bld/dependencies/PomPrefetcher.java @@ -0,0 +1,75 @@ +/* + * Copyright 2001-2026 Geert Bevin (gbevin[remove] at uwyn dot com) + * Licensed under the Apache License, Version 2.0 (the "License") + */ +package rife.bld.dependencies; + +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Speculatively retrieves POMs in parallel so that they are already cached + * by the artifact retriever when the sequential dependency resolution + * processes them, without influencing the resolution semantics. + *

+ * A single instance can be shared by multiple concurrent resolutions, the + * POM of each unique dependency will only be prefetched once. + * + * @author Geert Bevin (gbevin[remove] at uwyn dot com) + * @since 2.3.1 + */ +class PomPrefetcher { + private final VersionResolution resolution_; + private final ArtifactRetriever retriever_; + private final List repositories_; + private final ExecutorService executor_; + private final Set submitted_ = ConcurrentHashMap.newKeySet(); + + /** + * Creates a prefetcher when it can be beneficial. + * + * @return the prefetcher; or {@code null} when the retriever doesn't + * cache the retrieved POMs or when the resolution parallelism disables it + * @since 2.3.1 + */ + static PomPrefetcher create(VersionResolution resolution, ArtifactRetriever retriever, List repositories) { + // prefetching only has benefits when the retrieved POMs are + // cached for the sequential resolution that follows + if (!retriever.isCaching() || resolution.resolutionParallelism() <= 1) { + return null; + } + return new PomPrefetcher(resolution, retriever, repositories); + } + + private PomPrefetcher(VersionResolution resolution, ArtifactRetriever retriever, List repositories) { + resolution_ = resolution; + retriever_ = retriever; + repositories_ = repositories; + executor_ = Executors.newFixedThreadPool(resolution.resolutionParallelism()); + } + + void prefetch(Collection candidates) { + for (var candidate : candidates) { + var dependency = resolution_.overrideDependency(candidate.convertToDependency()); + if (submitted_.add(dependency)) { + executor_.submit(() -> { + try { + new DependencyResolver(resolution_, retriever_, repositories_, dependency).getMavenPom(dependency); + } catch (Throwable e) { + // failures are ignored since they will resurface + // with the proper context when the dependency is + // resolved in order + } + }); + } + } + } + + void shutdown() { + executor_.shutdownNow(); + } +} diff --git a/src/main/java/rife/bld/dependencies/VersionResolution.java b/src/main/java/rife/bld/dependencies/VersionResolution.java index 7741c4e..8dc2de0 100644 --- a/src/main/java/rife/bld/dependencies/VersionResolution.java +++ b/src/main/java/rife/bld/dependencies/VersionResolution.java @@ -49,8 +49,18 @@ public class VersionResolution { public static final String PROPERTY_TRANSFER_PARALLELISM = "bld.transferParallelism"; private static final int DEFAULT_TRANSFER_PARALLELISM = 6; + /** + * The property key that determines how many POMs are speculatively + * retrieved in parallel during transitive dependency resolution, + * {@code 1} disables the parallel retrieval. + * @since 2.3.1 + */ + public static final String PROPERTY_RESOLUTION_PARALLELISM = "bld.resolutionParallelism"; + private static final int DEFAULT_RESOLUTION_PARALLELISM = 6; + private final Map versionOverrides_ = new HashMap<>(); private final int transferParallelism_; + private final int resolutionParallelism_; /** * Returns a dummy {@code VersionResolution} instance that doesn't override anything. @@ -73,7 +83,6 @@ public class VersionResolution { * @since 2.0 */ public VersionResolution(HierarchicalProperties properties) { - var transfer_parallelism = DEFAULT_TRANSFER_PARALLELISM; if (properties != null) { for (var name : properties.getNames()) { if (name.startsWith(PROPERTY_OVERRIDE_PREFIX)) { @@ -88,17 +97,23 @@ public class VersionResolution { } } } + } + transferParallelism_ = parseParallelism(properties, PROPERTY_TRANSFER_PARALLELISM, DEFAULT_TRANSFER_PARALLELISM); + resolutionParallelism_ = parseParallelism(properties, PROPERTY_RESOLUTION_PARALLELISM, DEFAULT_RESOLUTION_PARALLELISM); + } - var parallelism = properties.getValueString(PROPERTY_TRANSFER_PARALLELISM); + private static int parseParallelism(HierarchicalProperties properties, String property, int defaultValue) { + if (properties != null) { + var parallelism = properties.getValueString(property); if (parallelism != null && !parallelism.isBlank()) { try { - transfer_parallelism = Math.max(1, Integer.parseInt(parallelism.trim())); + return Math.max(1, Integer.parseInt(parallelism.trim())); } catch (NumberFormatException e) { - Logger.getLogger("rife.bld").warning("Unable to parse the " + PROPERTY_TRANSFER_PARALLELISM + " property as an integer: '" + parallelism + "', using " + transfer_parallelism + " instead"); + Logger.getLogger("rife.bld").warning("Unable to parse the " + property + " property as an integer: '" + parallelism + "', using " + defaultValue + " instead"); } } } - transferParallelism_ = transfer_parallelism; + return defaultValue; } /** @@ -158,4 +173,16 @@ public class VersionResolution { public int transferParallelism() { return transferParallelism_; } + + /** + * Returns the number of POMs that are speculatively retrieved in parallel + * during transitive dependency resolution, {@code 1} means the parallel + * retrieval is disabled. + * + * @return the number of parallel POM retrievals + * @since 2.3.1 + */ + public int resolutionParallelism() { + return resolutionParallelism_; + } } diff --git a/src/test/java/rife/bld/dependencies/TestDependencyResolver.java b/src/test/java/rife/bld/dependencies/TestDependencyResolver.java index f8eb44d..7939498 100644 --- a/src/test/java/rife/bld/dependencies/TestDependencyResolver.java +++ b/src/test/java/rife/bld/dependencies/TestDependencyResolver.java @@ -4,15 +4,20 @@ */ package rife.bld.dependencies; +import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Test; import rife.ioc.HierarchicalProperties; import rife.tools.FileUtils; import rife.tools.StringUtils; import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.*; import static rife.bld.dependencies.Dependency.CLASSIFIER_JAVADOC; @@ -2341,4 +2346,98 @@ public class TestDependencyResolver { FileUtils.deleteDirectory(tmp2); } } + + @Test + void testGetAllDependenciesParallelPomPrefetching() throws Exception { + var max_concurrent_retrievals = new AtomicInteger(); + var server = createPomServer(max_concurrent_retrievals); + server.start(); + try { + var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/")); + var root = new Dependency("com.example", "root", new VersionNumber(1, 0, 0)); + + // the caching retriever enables parallel POM prefetching + var resolver = new DependencyResolver(VersionResolution.dummy(), ArtifactRetriever.cachingInstance(), repositories, root); + assertEquals(StringUtils.convertLineSeparator(""" + com.example:root:1.0.0 + com.example:child1:1.0.0 + com.example:child2:1.0.0 + com.example:child3:1.0.0 + com.example:child4:1.0.0 + com.example:child5:1.0.0 + com.example:child6:1.0.0 + com.example:shared:1.0.0"""), StringUtils.join(resolver.getAllDependencies(compile), System.lineSeparator())); + assertTrue(max_concurrent_retrievals.get() > 1, "expected concurrent POM retrievals, max was " + max_concurrent_retrievals.get()); + } finally { + server.stop(0); + } + } + + @Test + void testGetAllDependenciesSequentialWithoutCachingRetriever() throws Exception { + var max_concurrent_retrievals = new AtomicInteger(); + var server = createPomServer(max_concurrent_retrievals); + server.start(); + try { + var repositories = List.of(new Repository("http://localhost:" + server.getAddress().getPort() + "/")); + var root = new Dependency("com.example", "root", new VersionNumber(1, 0, 0)); + + // the uncached retriever disables prefetching, retrievals stay sequential + var resolver = new DependencyResolver(VersionResolution.dummy(), ArtifactRetriever.instance(), repositories, root); + assertEquals(8, resolver.getAllDependencies(compile).size()); + assertEquals(1, max_concurrent_retrievals.get(), "expected sequential POM retrievals, max was " + max_concurrent_retrievals.get()); + } finally { + server.stop(0); + } + } + + private static HttpServer createPomServer(AtomicInteger maxConcurrentRetrievals) + throws IOException { + var graph = new HashMap>(); + graph.put("root", List.of("child1", "child2", "child3", "child4", "child5", "child6")); + graph.put("child1", List.of("shared")); + + var active_retrievals = new AtomicInteger(); + var server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + server.createContext("/", exchange -> { + var active = active_retrievals.incrementAndGet(); + maxConcurrentRetrievals.accumulateAndGet(active, Math::max); + try { + // delay the response so that parallel retrievals overlap + Thread.sleep(100); + + // serve the POM of the artifact in the request path + var segments = exchange.getRequestURI().getPath().split("/"); + var artifact = segments[segments.length - 3]; + var body = buildPom(artifact, graph.getOrDefault(artifact, List.of())).getBytes(); + exchange.sendResponseHeaders(200, body.length); + exchange.getResponseBody().write(body); + exchange.close(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + active_retrievals.decrementAndGet(); + } + }); + server.setExecutor(Executors.newCachedThreadPool()); + return server; + } + + private static String buildPom(String artifact, List children) { + var dependencies = new StringBuilder(); + for (var child : children) { + dependencies.append("com.example") + .append(child) + .append("1.0.0"); + } + return """ + + + 4.0.0 + com.example + %s + 1.0.0 + %s + """.formatted(artifact, dependencies); + } }