From 05edd0c8715a3bc7d4e779cad77ce22ab1fa5ee4 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Tue, 28 Jul 2015 17:02:44 -0400 Subject: [PATCH 01/24] Completely redesign the melting pot Read the docs. Maybe it is helpful for testing now. Example usage: $ melting-pot.sh net.imagej:imagej-common:0.15.1 \ -r http://maven.imagej.net/content/groups/public \ -c org.scijava:scijava-common:2.44.3-SNAPSHOT \ -i 'org.scijava:*,net.imagej:*,net.imglib2:*,io.scif:*' \ -e net.imglib2:imglib2-roi -v Make sure you 'mvn install' scijava-common 2.44.3-SNAPSHOT first. --- melting-pot.sh | 450 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 417 insertions(+), 33 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index f7b904e..eecad86 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -1,35 +1,419 @@ #!/bin/sh -# melting-pot.sh - A script to build the entire SciJava software stack -# from the latest code on the respective master branches. - -rm -rf melting-pot -mkdir melting-pot -cd melting-pot - -echo '' > pom.xml -echo '' >> pom.xml -echo ' 4.0.0' >> pom.xml -echo >> pom.xml -echo ' org.scijava' >> pom.xml -echo ' melting-pot' >> pom.xml -echo ' 0.0.0-SNAPSHOT' >> pom.xml -echo ' pom' >> pom.xml -echo >> pom.xml -echo ' SciJava Uber Build' >> pom.xml -echo >> pom.xml -echo ' ' >> pom.xml - -for repo in $(sj-hierarchy.pl -g) -do - git clone $repo --depth 1 - module=${repo##*/} - module=${module%.git} - echo " $module" >> pom.xml -done - -echo ' ' >> pom.xml -echo -echo '' >> pom.xml - -mvn -Pdev.scijava,dev.imglib2,dev.scifio,dev.imagej validate +# ============================================================================ +# melting-pot.sh +# ============================================================================ +# Tests all components of a project affected by changes in its dependencies. +# +# First, an anecdote illustrating the problem this script solves: +# +# Suppose you have a large application, org:app:1.0.0, with many dependencies: +# org:foo:1.2.3, org:bar:3.4.5, and many others. +# +# Now suppose you make some changes to foo, and want to know whether deploying +# them (i.e., releasing a new foo and updating app to depend on that release) +# will break the app. So you manually update your local copy of app to depend +# on org:foo:1.3.0-SNAPSHOT, and run the build (including tests, of course). +# +# The build passes, but this alone is insufficient: org:bar:3.4.5 also depends +# on org:foo:1.2.3, so you manually update bar to use org:foo:1.3.0-SNAPSHOT, +# then build bar to verify that it also is not broken by the update. +# +# This process quickly becomes very tedious when there are dozens of +# components of app which all depend on foo. +# +# And more importantly, testing each component individually in this manner is +# still insufficient to determine whether all of them will truly work together +# at runtime, where only a single version of each component is deployed. +# +# For example: suppose org:bar:3.4.5 depends on org:lib:8.0.0, while +# org:foo:1.2.3 depends on org:lib:7.0.0. The relevant facts are: +# +# * Your new foo (org:foo:1.3.0-SNAPSHOT) builds against lib 7, and portions +# of it rely on lib-7-specific API. +# +# * The bar component pinned to foo 1.3.0-SNAPSHOT builds against lib 8; it +# compiles with passing tests because bar only invokes portions of the foo +# API which do not require lib-7-specific API. +# +# In this scenario, it is lib 8 that is actually deployed at runtime with the +# app, so parts of foo will be broken, even though both foo and bar build with +# passing tests individually. +# +# This "melting pot" build seeks to overcome many of these issues by unifying +# all components of the app into a single multi-module build, with all +# versions uniformly pinned to the ones that will actually be deployed at +# runtime. +# +# This goal is achieved by synthesizing a multi-module build including all +# affected components (or optionally, all components period) of the specific +# project, and then executing a Maven build with uniformly overridden versions +# of all components to the ones resolved for the project itself. +# +# IMPORTANT IMPLEMENTATION DETAIL! The override works by setting a version +# property for each component of the form "artifactId.version"; it is assumed +# that all components declare their dependencies using version properties of +# this form. E.g.: +# +# +# com.google.guava +# guava +# ${guava.version} +# +# +# Using dependencyManagement is fine too, as long as it then uses this pattern +# to declare the versions as properties, which can be overridden. +# +# Any dependency which does not declare a version property matching this +# assumption will not be properly overridden in the melting pot! +# +# Author: Curtis Rueden +# Dependencies: git, mvn, xmllint +# ============================================================================ + +# -- Functions -- + +stderr() { + >&2 echo "$@" +} + +debug() { + test "$verbose" && + stderr "[DEBUG] $@" +} + +error() { + stderr "[ERROR] $@" +} + +die() { + code="$1" + shift + error $@ + exit "$code" +} + +unknownArg() { + error "Unknown option: $@" + usage=1 +} + +parseArguments() { + while [ $# -ge 1 ] + do + case "$1" in + -c|--changes) + changes="$2" + shift + ;; + -i|--includes) + includes="$2" + shift + ;; + -e|--excludes) + excludes="$2" + shift + ;; + -r|--remoteRepos) + remoteRepos="$2" + shift + ;; + -l|--localRepo) + repoBase="$2" + shift + ;; + -o|--outputDir) + outputDir="$2" + shift + ;; + -v|--verbose) + verbose=1 + ;; + -f|--force) + force=1 + ;; + -h|--help) + usage=1 + ;; + -*) + unknownArg "$1" + ;; + *) + test -z "$project" && project="$1" || + unknownArg "$1" + ;; + esac + shift + done + + test -z "$project" && error "No project specified!" && usage=1 + + if [ "$usage" ] + then + echo "Usage: $(basename "$0") [-c ] \\ + [-i ] [-e ] [-r ] [-l ] [-o ] [-vfh] + + + The project to build, including dependencies, with consistent versions. +-c, --changes + Comma-separated list of GAVs to inject into the project, replacing + normal versions. E.g.: \"com.mycompany:myartifact:1.2.3-SNAPSHOT\" +-i, --includes + Comma-separated list of GAs (no version; wildcards OK for G or A) to + include in the build. All by default. E.g.: \"mystuff:*,myotherstuff:*\" +-e, --excludes + Comma-separated list of GAs (no version; wildcards OK for G or A) to + exclude from the build. E.g.: \"mystuff:extraneous,mystuff:irrelevant\" +-r, --remoteRepos + Comma-separated list of additional remote Maven repositories to check + for artifacts, in the format id::[layout]::url or just url. +-l, --localRepos + Overrides the directory of the Maven local repository cache. +-o, --outputDir + Overrides the output directory. The default is \"melting-pot\". +-v, --verbose + Enable verbose/debugging output. +-f, --force + Wipe out the output directory if it already exists. +-h, --help + Display this usage information." + exit 1 + fi + + # Assign default parameter values. + test "$outputDir" || outputDir="melting-pot" + test "$repoBase" || repoBase="$HOME/.m2/repository" +} + +createDir() { + test -z "$force" -a -e "$1" && + die 2 "Directory already exists: $1" + + rm -rf "$1" + mkdir -p "$1" + cd "$1" +} + +groupId() { + echo "${1%%:*}" +} + +artifactId() { + result="${1#*:}" # strip groupId + echo "${result%%:*}" +} + +version() { + result="${1#*:}" # strip groupId + case "$result" in + *:*) + result="${result#*:}" # strip artifactId + case "$result" in + *:*:*:*) + # G:A:P:C:V:S + result="${result#*:}" # strip packaging + result="${result#*:}" # strip classifier + ;; + *:*:*) + # G:A:P:V:S + result="${result#*:}" # strip packaging + ;; + *) + # G:A:V or G:A:V:? + ;; + esac + echo "${result%%:*}" + ;; + esac +} + +# Converts the given GAV into a path in the local repository cache. +repoPath() { + gPath="$(echo "$(groupId "$1")" | tr :. /)" + aPath="$(artifactId "$1")" + vPath="$(version "$1")" + echo "$repoBase/$gPath/$aPath/$vPath" +} + +# Gets the path to the given GAV's POM file in the local repository cache. +pomPath() { + pomFile="$(artifactId "$1")-$(version "$1").pom" + echo "$(repoPath "$1")/$pomFile" +} + +# Fetches the POM for the given GAV into the local repository cache. +downloadPOM() { + mvn dependency:get \ + -DrepoUrl="$remoteRepos" \ + -DgroupId="$(groupId "$1")" \ + -DartifactId="$(artifactId "$1")" \ + -Dversion="$(version "$1")" \ + -Dpackaging=pom +} + +# Gets the POM path for the given GAV, ensuring it exists locally. +pom() { + pomPath="$(pomPath "$1")" + test -f "$pomPath" || downloadPOM "$1" + echo "$pomPath" +} + +# Gets the SCM URL for the given GAV. +scmURL() { + scmXPath="//*[local-name()='project']/*[local-name()='scm']/*[local-name()='connection']" + xmllint --xpath "$scmXPath" "$(pom "$1")" | sed -E 's/.*>scm:git:(.*)<.*/\1/' +} + +# Gets the SCM tag for the given GAV. +scmTag() { + echo "$(artifactId "$1")-$(version "$1")" +} + +# Fetches the source code for the given GAV. Returns the directory. +retrieveSource() { + scmURL="$(scmURL "$1")" + scmTag="$(scmTag "$1")" + dir="$(groupId "$1")/$(artifactId "$1")" + git clone "$scmURL" --branch "$scmTag" --depth 1 "$dir" 2> /dev/null + echo "$dir" +} + +# Gets the list of dependencies for the project in the CWD. +deps() { + mvn dependency:list | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' +} + +# Checks whether the given GA(V) matches the specified filter pattern. +gaMatch() { + ga="$1" + filter="$2" + g="$(groupId "$ga")" + a="$(artifactId "$ga")" + fg="$(groupId "$filter")" + fa="$(artifactId "$filter")" + test "$fg" = "$g" -o "$fg" = "*" || return + test "$fa" = "$a" -o "$fa" = "*" || return + echo 1 +} + +# Determines whether the given GA(V) version is being overridden. +isChanged() { + local IFS="," + + for change in $changes + do + test "$(gaMatch "$1" "$change")" && echo 1 && return + done +} + +# Determines whether the given GA(V) meets the inclusion criteria. +isIncluded() { + # do not include the changed artifacts we are testing against + test "$(isChanged "$1")" && return + + local IFS="," + + # ensure GA is not excluded + for exclude in $excludes + do + test "$(gaMatch "$1" "$exclude")" && return + done + + # ensure GA is included + test -z "$includes" && echo 1 && return + for include in $includes + do + test "$(gaMatch "$1" "$include")" && echo 1 && return + done +} + +# Generates an aggregator POM for all modules in the current directory. +generatePOM() { + echo '' > pom.xml + echo '> pom.xml + echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml + echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml + echo ' http://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml + echo ' 4.0.0' >> pom.xml + echo >> pom.xml + echo ' melting-pot' >> pom.xml + echo ' melting-pot' >> pom.xml + echo ' 0.0.0-SNAPSHOT' >> pom.xml + echo ' pom' >> pom.xml + echo >> pom.xml + echo ' Melting Pot' >> pom.xml + echo >> pom.xml + echo ' ' >> pom.xml + for dir in */* + do + test -d "$dir" && + echo " $dir" >> pom.xml + done + echo ' ' >> pom.xml + echo '' >> pom.xml +} + +# Creates and tests an appropriate multi-module reactor for the given project. +# All relevant dependencies which match the inclusion criteria are linked into +# the multi-module build, with each changed GAV overridding the originally +# specified version for the corresponding GA. +meltDown() { + # Fetch the project source code. + debug "$1: fetching project source" + dir="$(retrieveSource "$1")" + + # Get the project dependencies. + debug "$1: determining project dependencies" + cd "$dir" + deps="$(deps)" + cd - > /dev/null + + args="-Denforcer.skip" + + # Process the dependencies. + debug "$1: processing project dependencies" + for dep in $deps + do + g="$(groupId "$dep")" + a="$(artifactId "$dep")" + v="$(version "$dep")" + gav="$g:$a:$v" + + test -z "$(isChanged "$gav")" && + args="$args -D$a.version=$v" + + if [ "$(isIncluded "$gav")" ] + then + debug "$1: $a: fetching component source" + dir="$(retrieveSource "$gav")" + fi + done + + # Override versions of changed GAVs. + debug "$1: processing changed components" + local TLS=, + for gav in $changes + do + a="$(artifactId "$gav")" + v="$(version "$gav")" + args="$args -D$a.version=$v" + done + unset TLS + + # Generate the aggregator POM. + debug "Generating aggregator POM" + generatePOM + + # Build everything. + debug "Building the project!" + # NB: All code is fresh; no need to clean. + mvn $args test + + debug "$1: complete" +} + +# -- Main -- + +parseArguments $@ +createDir "$outputDir" +meltDown "$project" From 9ab53065670c72259fece33dd07db71b6c2c24f8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 29 Jul 2015 18:35:17 -0400 Subject: [PATCH 02/24] melting-pot: add an option to skip the build This still prepares the melting pot according to the given criteria, but does not actually call "mvn test" at the end. It will be very useful for regression testing with cram. --- melting-pot.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index eecad86..72d1b40 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -132,6 +132,9 @@ parseArguments() { -f|--force) force=1 ;; + -s|--skipBuild) + skipBuild=1 + ;; -h|--help) usage=1 ;; @@ -151,7 +154,7 @@ parseArguments() { if [ "$usage" ] then echo "Usage: $(basename "$0") [-c ] \\ - [-i ] [-e ] [-r ] [-l ] [-o ] [-vfh] + [-i ] [-e ] [-r ] [-l ] [-o ] [-vfsh] The project to build, including dependencies, with consistent versions. @@ -175,6 +178,8 @@ parseArguments() { Enable verbose/debugging output. -f, --force Wipe out the output directory if it already exists. +-s, --skipBuild + Skips the final build step. Useful for automated testing. -h, --help Display this usage information." exit 1 @@ -405,9 +410,15 @@ meltDown() { generatePOM # Build everything. - debug "Building the project!" - # NB: All code is fresh; no need to clean. - mvn $args test + if [ "$skipBuild" ] + then + debug "Skipping the build; the command would have been:" + debug "mvn $args test" + else + debug "Building the project!" + # NB: All code is fresh; no need to clean. + mvn $args test + fi debug "$1: complete" } From bd0d53897400137d01e6f4810e1be7b4e754276d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 29 Jul 2015 18:51:22 -0400 Subject: [PATCH 03/24] Add a couple of cram tests for the melting pot It's a relatively complex beast, so let's really make sure it works. --- tests/melting-pot-out-dir.t | 5 +++++ tests/melting-pot-simple.t | 20 ++++++++++++++++++++ tests/readme.txt | 5 +++++ 3 files changed, 30 insertions(+) create mode 100644 tests/melting-pot-out-dir.t create mode 100644 tests/melting-pot-simple.t create mode 100644 tests/readme.txt diff --git a/tests/melting-pot-out-dir.t b/tests/melting-pot-out-dir.t new file mode 100644 index 0000000..2a236ae --- /dev/null +++ b/tests/melting-pot-out-dir.t @@ -0,0 +1,5 @@ +Script should fail if output directory already exists: + + $ mkdir melting-pot && sh "$TESTDIR/../melting-pot.sh" foo:bar + [ERROR] Directory already exists: melting-pot + [2] diff --git a/tests/melting-pot-simple.t b/tests/melting-pot-simple.t new file mode 100644 index 0000000..9ab0525 --- /dev/null +++ b/tests/melting-pot-simple.t @@ -0,0 +1,20 @@ +Down-the-middle test of a relatively simply project: + + $ sh "$TESTDIR/../melting-pot.sh" net.imagej:imagej-common:0.15.1 -r http://maven.imagej.net/content/groups/public -c org.scijava:scijava-common:2.44.2 -i 'org.scijava:*,net.imagej:*,net.imglib2:*,io.scif:*' -e net.imglib2:imglib2-roi -v -f -s + [DEBUG] net.imagej:imagej-common:0.15.1: fetching project source + [DEBUG] net.imagej:imagej-common:0.15.1: determining project dependencies + [DEBUG] net.imagej:imagej-common:0.15.1: processing project dependencies + [DEBUG] net.imagej:imagej-common:0.15.1: imglib2: fetching component source + [DEBUG] net.imagej:imagej-common:0.15.1: processing changed components + [DEBUG] Generating aggregator POM + [DEBUG] Skipping the build; the command would have been: + [DEBUG] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test + [DEBUG] net.imagej:imagej-common:0.15.1: complete + + $ find melting-pot -maxdepth 2 + melting-pot + melting-pot/net.imagej + melting-pot/net.imagej/imagej-common + melting-pot/net.imglib2 + melting-pot/net.imglib2/imglib2 + melting-pot/pom.xml diff --git a/tests/readme.txt b/tests/readme.txt new file mode 100644 index 0000000..91db61b --- /dev/null +++ b/tests/readme.txt @@ -0,0 +1,5 @@ +This directory houses automated tests for use with cram. + + https://bitheap.org/cram/ + +To run them, type "cram tests" from the toplevel directory. From 2ec1263615093eeab84cce94e02fbe62f4273b83 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 29 Jul 2015 19:10:50 -0400 Subject: [PATCH 04/24] melting-pot: make local vars actually local This helps clarify the intent of each variable, as well as potentially avoiding certain classes of bugs relating to those variable names used across multiple functions. --- melting-pot.sh | 58 ++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 72d1b40..06e5297 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -87,7 +87,7 @@ error() { } die() { - code="$1" + local code="$1" shift error $@ exit "$code" @@ -204,12 +204,12 @@ groupId() { } artifactId() { - result="${1#*:}" # strip groupId + local result="${1#*:}" # strip groupId echo "${result%%:*}" } version() { - result="${1#*:}" # strip groupId + local result="${1#*:}" # strip groupId case "$result" in *:*) result="${result#*:}" # strip artifactId @@ -234,15 +234,15 @@ version() { # Converts the given GAV into a path in the local repository cache. repoPath() { - gPath="$(echo "$(groupId "$1")" | tr :. /)" - aPath="$(artifactId "$1")" - vPath="$(version "$1")" + local gPath="$(echo "$(groupId "$1")" | tr :. /)" + local aPath="$(artifactId "$1")" + local vPath="$(version "$1")" echo "$repoBase/$gPath/$aPath/$vPath" } # Gets the path to the given GAV's POM file in the local repository cache. pomPath() { - pomFile="$(artifactId "$1")-$(version "$1").pom" + local pomFile="$(artifactId "$1")-$(version "$1").pom" echo "$(repoPath "$1")/$pomFile" } @@ -258,14 +258,14 @@ downloadPOM() { # Gets the POM path for the given GAV, ensuring it exists locally. pom() { - pomPath="$(pomPath "$1")" + local pomPath="$(pomPath "$1")" test -f "$pomPath" || downloadPOM "$1" echo "$pomPath" } # Gets the SCM URL for the given GAV. scmURL() { - scmXPath="//*[local-name()='project']/*[local-name()='scm']/*[local-name()='connection']" + local scmXPath="//*[local-name()='project']/*[local-name()='scm']/*[local-name()='connection']" xmllint --xpath "$scmXPath" "$(pom "$1")" | sed -E 's/.*>scm:git:(.*)<.*/\1/' } @@ -276,9 +276,9 @@ scmTag() { # Fetches the source code for the given GAV. Returns the directory. retrieveSource() { - scmURL="$(scmURL "$1")" - scmTag="$(scmTag "$1")" - dir="$(groupId "$1")/$(artifactId "$1")" + local scmURL="$(scmURL "$1")" + local scmTag="$(scmTag "$1")" + local dir="$(groupId "$1")/$(artifactId "$1")" git clone "$scmURL" --branch "$scmTag" --depth 1 "$dir" 2> /dev/null echo "$dir" } @@ -290,12 +290,12 @@ deps() { # Checks whether the given GA(V) matches the specified filter pattern. gaMatch() { - ga="$1" - filter="$2" - g="$(groupId "$ga")" - a="$(artifactId "$ga")" - fg="$(groupId "$filter")" - fa="$(artifactId "$filter")" + local ga="$1" + local filter="$2" + local g="$(groupId "$ga")" + local a="$(artifactId "$ga")" + local fg="$(groupId "$filter")" + local fa="$(artifactId "$filter")" test "$fg" = "$g" -o "$fg" = "*" || return test "$fa" = "$a" -o "$fa" = "*" || return echo 1 @@ -305,6 +305,7 @@ gaMatch() { isChanged() { local IFS="," + local change for change in $changes do test "$(gaMatch "$1" "$change")" && echo 1 && return @@ -319,6 +320,7 @@ isIncluded() { local IFS="," # ensure GA is not excluded + local exclude for exclude in $excludes do test "$(gaMatch "$1" "$exclude")" && return @@ -326,6 +328,7 @@ isIncluded() { # ensure GA is included test -z "$includes" && echo 1 && return + local include for include in $includes do test "$(gaMatch "$1" "$include")" && echo 1 && return @@ -349,6 +352,7 @@ generatePOM() { echo ' Melting Pot' >> pom.xml echo >> pom.xml echo ' ' >> pom.xml + local dir for dir in */* do test -d "$dir" && @@ -365,24 +369,25 @@ generatePOM() { meltDown() { # Fetch the project source code. debug "$1: fetching project source" - dir="$(retrieveSource "$1")" + local dir="$(retrieveSource "$1")" # Get the project dependencies. debug "$1: determining project dependencies" cd "$dir" - deps="$(deps)" + local deps="$(deps)" cd - > /dev/null args="-Denforcer.skip" # Process the dependencies. debug "$1: processing project dependencies" + local dep for dep in $deps do - g="$(groupId "$dep")" - a="$(artifactId "$dep")" - v="$(version "$dep")" - gav="$g:$a:$v" + local g="$(groupId "$dep")" + local a="$(artifactId "$dep")" + local v="$(version "$dep")" + local gav="$g:$a:$v" test -z "$(isChanged "$gav")" && args="$args -D$a.version=$v" @@ -397,10 +402,11 @@ meltDown() { # Override versions of changed GAVs. debug "$1: processing changed components" local TLS=, + local gav for gav in $changes do - a="$(artifactId "$gav")" - v="$(version "$gav")" + local a="$(artifactId "$gav")" + local v="$(version "$gav")" args="$args -D$a.version=$v" done unset TLS From 522f493f7eb5f23d6974f326606ccb918cc3f2c0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 29 Jul 2015 19:19:33 -0400 Subject: [PATCH 05/24] melting-pot: improve elegance of deps function Better to change the directory (and back) from within the function. --- melting-pot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 06e5297..1473f76 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -285,7 +285,9 @@ retrieveSource() { # Gets the list of dependencies for the project in the CWD. deps() { + cd "$1" mvn dependency:list | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' + cd - > /dev/null } # Checks whether the given GA(V) matches the specified filter pattern. @@ -373,9 +375,7 @@ meltDown() { # Get the project dependencies. debug "$1: determining project dependencies" - cd "$dir" - local deps="$(deps)" - cd - > /dev/null + local deps="$(deps "$dir")" args="-Denforcer.skip" From c46b1000691f2d84091ad4d31d08bcfa8c80274d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 29 Jul 2015 18:14:49 -0400 Subject: [PATCH 06/24] melting-pot: add flag to prune unaffected deps The default behavior of the melting pot is to create a multi-module build of all project components which match the inclusion criteria (i.e., the includes and excludes), minus any changed components specified by the '--changes' flag. However, when the set of changes is small, and the main interest is in testing breakages relating to those changes, then it is nice to limit the build to only those components which depend on the changed components (either directly or transitively). Hence, the new '--prune' flag does exactly that! Be warned that even if a component does not explicitly depend on a changed component, it might still be affected by changes in that component. One example is components which use dependency injection at runtime, using resources discovered on the classpath (e.g., plugins). --- melting-pot.sh | 38 +++++++++++++++++++++++++++++++++++++- tests/melting-pot-prune.t | 25 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 tests/melting-pot-prune.t diff --git a/melting-pot.sh b/melting-pot.sh index 1473f76..35849b3 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -126,6 +126,9 @@ parseArguments() { outputDir="$2" shift ;; + -p|--prune) + prune=1 + ;; -v|--verbose) verbose=1 ;; @@ -154,7 +157,7 @@ parseArguments() { if [ "$usage" ] then echo "Usage: $(basename "$0") [-c ] \\ - [-i ] [-e ] [-r ] [-l ] [-o ] [-vfsh] + [-i ] [-e ] [-r ] [-l ] [-o ] [-pvfsh] The project to build, including dependencies, with consistent versions. @@ -174,6 +177,10 @@ parseArguments() { Overrides the directory of the Maven local repository cache. -o, --outputDir Overrides the output directory. The default is \"melting-pot\". +-p, --prune + Build only the components which themselves depend on a changed + artifact. This will make the build much faster, at the expense of + not fully testing runtime compatibility across all components. -v, --verbose Enable verbose/debugging output. -f, --force @@ -337,6 +344,32 @@ isIncluded() { done } +# Deletes components which do not depend on a changed GAV. +pruneReactor() { + local dir + for dir in */* + do + debug "Checking relevance of component $dir" + local deps="$(deps "$dir")" + + # Determine whether the component depends on a changed GAV. + local keep + unset keep + local dep + for dep in $deps + do + test "$(isChanged "$dep")" && keep=1 && break + done + + # If the component is irrelevant, prune it. + if [ -z "$keep" ] + then + debug "Pruning irrelevant component: $dir" + rm -rf "$dir" + fi + done +} + # Generates an aggregator POM for all modules in the current directory. generatePOM() { echo '' > pom.xml @@ -411,6 +444,9 @@ meltDown() { done unset TLS + # Prune the build, if applicable. + test "$prune" && pruneReactor + # Generate the aggregator POM. debug "Generating aggregator POM" generatePOM diff --git a/tests/melting-pot-prune.t b/tests/melting-pot-prune.t new file mode 100644 index 0000000..9ea2c96 --- /dev/null +++ b/tests/melting-pot-prune.t @@ -0,0 +1,25 @@ +Test that the '--prune' flag works as intended: + + $ sh "$TESTDIR/../melting-pot.sh" net.imagej:imagej-common:0.15.1 -r http://maven.imagej.net/content/groups/public -c org.scijava:scijava-common:2.44.2 -i 'org.scijava:*,net.imagej:*,net.imglib2:*' -p -v -f -s + [DEBUG] net.imagej:imagej-common:0.15.1: fetching project source + [DEBUG] net.imagej:imagej-common:0.15.1: determining project dependencies + [DEBUG] net.imagej:imagej-common:0.15.1: processing project dependencies + [DEBUG] net.imagej:imagej-common:0.15.1: imglib2-roi: fetching component source + [DEBUG] net.imagej:imagej-common:0.15.1: imglib2: fetching component source + [DEBUG] net.imagej:imagej-common:0.15.1: processing changed components + [DEBUG] Checking relevance of component net.imagej/imagej-common + [DEBUG] Checking relevance of component net.imglib2/imglib2 + [DEBUG] Pruning irrelevant component: net.imglib2/imglib2 + [DEBUG] Checking relevance of component net.imglib2/imglib2-roi + [DEBUG] Pruning irrelevant component: net.imglib2/imglib2-roi + [DEBUG] Generating aggregator POM + [DEBUG] Skipping the build; the command would have been: + [DEBUG] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test + [DEBUG] net.imagej:imagej-common:0.15.1: complete + + $ find melting-pot -maxdepth 2 + melting-pot + melting-pot/net.imagej + melting-pot/net.imagej/imagej-common + melting-pot/net.imglib2 + melting-pot/pom.xml From 1f869e8fdb8d99cf3c3b960766b6867a220e7bcd Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 07:41:01 -0400 Subject: [PATCH 07/24] melting-pot: do not complain if -h is given When no project is given, but -h is passed, just show the help, instead of also complaining about a missing project. --- melting-pot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 35849b3..719f9a8 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -152,7 +152,8 @@ parseArguments() { shift done - test -z "$project" && error "No project specified!" && usage=1 + test -z "$project" -a -z "$usage" && + error "No project specified!" && usage=1 if [ "$usage" ] then From ef639bd3fcc14826d5bb4c6ebc9c22314cebe280 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 07:42:06 -0400 Subject: [PATCH 08/24] melting-pot: make args variable local It is only used in one function. --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 719f9a8..c38054f 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -411,7 +411,7 @@ meltDown() { debug "$1: determining project dependencies" local deps="$(deps "$dir")" - args="-Denforcer.skip" + local args="-Denforcer.skip" # Process the dependencies. debug "$1: processing project dependencies" From 3059bcf6b9e56612b2c9477a0646d6c48fbe6ffa Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 07:52:28 -0400 Subject: [PATCH 09/24] melting-pot: add option to override the branch For projects like sc.fiji:fiji:2.0.0-SNAPSHOT, there is no tag or branch called 'fiji-2.0.0-SNAPSHOT' because it is a snapshot. The -b option allows to specify the branch to use explicitly; e.g., for the fiji GAV above, it should be master. So now the following invocation is much closer to working: melting-pot.sh sc.fiji:fiji:2.0.0-SNAPSHOT -b master \ -i 'org.scijava:*,net.imagej:*,net.imglib2:*,io.scif:*,sc.fiji:*' \ -r http://maven.imagej.net/content/groups/public -v -f --- melting-pot.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index c38054f..7c579c3 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -102,6 +102,10 @@ parseArguments() { while [ $# -ge 1 ] do case "$1" in + -b|--branch) + branch="$2" + shift + ;; -c|--changes) changes="$2" shift @@ -157,11 +161,14 @@ parseArguments() { if [ "$usage" ] then - echo "Usage: $(basename "$0") [-c ] \\ + echo "Usage: $(basename "$0") [-b ] [-c ] \\ [-i ] [-e ] [-r ] [-l ] [-o ] [-pvfsh] The project to build, including dependencies, with consistent versions. +-b, --branch + Override the branch/tag of the project to build. By default, + the branch used will be the tag named \"artifactId-version\". -c, --changes Comma-separated list of GAVs to inject into the project, replacing normal versions. E.g.: \"com.mycompany:myartifact:1.2.3-SNAPSHOT\" @@ -285,9 +292,10 @@ scmTag() { # Fetches the source code for the given GAV. Returns the directory. retrieveSource() { local scmURL="$(scmURL "$1")" - local scmTag="$(scmTag "$1")" + local scmBranch + test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" local dir="$(groupId "$1")/$(artifactId "$1")" - git clone "$scmURL" --branch "$scmTag" --depth 1 "$dir" 2> /dev/null + git clone "$scmURL" --branch "$scmBranch" --depth 1 "$dir" 2> /dev/null echo "$dir" } @@ -405,7 +413,7 @@ generatePOM() { meltDown() { # Fetch the project source code. debug "$1: fetching project source" - local dir="$(retrieveSource "$1")" + local dir="$(retrieveSource "$1" "$branch")" # Get the project dependencies. debug "$1: determining project dependencies" From 553b33ee51b9cef9082ff49211924e8ef6278b86 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 11:34:09 -0400 Subject: [PATCH 10/24] melting-pot: tweak while condition I like "> 0" aesthetically more than ">= 1". --- melting-pot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 7c579c3..91bcf7e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -99,7 +99,7 @@ unknownArg() { } parseArguments() { - while [ $# -ge 1 ] + while [ $# -gt 0 ] do case "$1" in -b|--branch) From 2cff749828eb92694d9db1482ae5c7f45d642cf7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 11:34:42 -0400 Subject: [PATCH 11/24] melting-pot: generalize xpath logic We will want to extract other information from POMs safely, so let's provide a more general-purpose xpath function using xmllint. --- melting-pot.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 91bcf7e..e05a883 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -278,10 +278,24 @@ pom() { echo "$pomPath" } +# For the given XML file on disk ($1), gets the value of the +# specified XPath expression of the form "//$2/$3/$4/...". +xpath() { + local xmlFile="$1" + shift + local xpath="/" + while [ $# -gt 0 ] + do + # NB: Ignore namespace issues; see: http://stackoverflow.com/a/8266075 + xpath="$xpath/*[local-name()='$1']" + shift + done + xmllint --xpath "$xpath" "$xmlFile" | sed -E 's/^[^>]*>(.*)<[^<]*$/\1/' +} + # Gets the SCM URL for the given GAV. scmURL() { - local scmXPath="//*[local-name()='project']/*[local-name()='scm']/*[local-name()='connection']" - xmllint --xpath "$scmXPath" "$(pom "$1")" | sed -E 's/.*>scm:git:(.*)<.*/\1/' + xpath "$(pom "$1")" project scm connection | sed -E 's/.*>scm:git:(.*)<.*/\1/' } # Gets the SCM tag for the given GAV. From 32ee1b45227184eec305ae54f531f65844daf56c Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 12:02:51 -0400 Subject: [PATCH 12/24] melting-pot: generalize pom value extraction This lets us recursively extract values from POMs specified as GAVs. --- melting-pot.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index e05a883..13aa984 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -293,9 +293,31 @@ xpath() { xmllint --xpath "$xpath" "$xmlFile" | sed -E 's/^[^>]*>(.*)<[^<]*$/\1/' } +# For the given GAV ($1), recursively gets the value of the +# specified XPath expression of the form "//$2/$3/$4/...". +pomValue() { + local pomPath="$(pom "$1")" + shift + local value="$(xpath "$pomPath" $@)" + if [ "$value" ] + then + echo "$value" + else + # Path not found in POM; look in the parent POM. + local pg="$(xpath "$pomPath" project parent groupId)" + if [ "$pg" ] + then + # There is a parent POM declaration in this POM. + local pa="$(xpath "$pomPath" project parent artifactId)" + local pv="$(xpath "$pomPath" project parent version)" + pomValue "$pg:$pa:$pv" $@ + fi + fi +} + # Gets the SCM URL for the given GAV. scmURL() { - xpath "$(pom "$1")" project scm connection | sed -E 's/.*>scm:git:(.*)<.*/\1/' + pomValue "$1" project scm connection | sed -E 's/^scm:git://' } # Gets the SCM tag for the given GAV. From 8fb9a2a75cddb00c115c1ecd7055f479d5b4de2f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 12:20:25 -0400 Subject: [PATCH 13/24] melting-pot: eat "XPath set is empty" messages We will try again with the parent POM, if possible. --- melting-pot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 13aa984..b710870 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -290,7 +290,8 @@ xpath() { xpath="$xpath/*[local-name()='$1']" shift done - xmllint --xpath "$xpath" "$xmlFile" | sed -E 's/^[^>]*>(.*)<[^<]*$/\1/' + xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | + sed -E 's/^[^>]*>(.*)<[^<]*$/\1/' } # For the given GAV ($1), recursively gets the value of the From e65644a43c1ea2bddf3e684027733fb45cd47b04 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 13:38:06 -0400 Subject: [PATCH 14/24] melting-pot: rename verbose output 'debug'->'info' This will make room for an even more verbose '--debug' option. --- melting-pot.sh | 28 ++++++++++++++-------------- tests/melting-pot-prune.t | 30 +++++++++++++++--------------- tests/melting-pot-simple.t | 18 +++++++++--------- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index b710870..9f2caad 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -77,9 +77,9 @@ stderr() { >&2 echo "$@" } -debug() { +info() { test "$verbose" && - stderr "[DEBUG] $@" + stderr "[INFO] $@" } error() { @@ -395,7 +395,7 @@ pruneReactor() { local dir for dir in */* do - debug "Checking relevance of component $dir" + info "Checking relevance of component $dir" local deps="$(deps "$dir")" # Determine whether the component depends on a changed GAV. @@ -410,7 +410,7 @@ pruneReactor() { # If the component is irrelevant, prune it. if [ -z "$keep" ] then - debug "Pruning irrelevant component: $dir" + info "Pruning irrelevant component: $dir" rm -rf "$dir" fi done @@ -449,17 +449,17 @@ generatePOM() { # specified version for the corresponding GA. meltDown() { # Fetch the project source code. - debug "$1: fetching project source" + info "$1: fetching project source" local dir="$(retrieveSource "$1" "$branch")" # Get the project dependencies. - debug "$1: determining project dependencies" + info "$1: determining project dependencies" local deps="$(deps "$dir")" local args="-Denforcer.skip" # Process the dependencies. - debug "$1: processing project dependencies" + info "$1: processing project dependencies" local dep for dep in $deps do @@ -473,13 +473,13 @@ meltDown() { if [ "$(isIncluded "$gav")" ] then - debug "$1: $a: fetching component source" + info "$1: $a: fetching component source" dir="$(retrieveSource "$gav")" fi done # Override versions of changed GAVs. - debug "$1: processing changed components" + info "$1: processing changed components" local TLS=, local gav for gav in $changes @@ -494,21 +494,21 @@ meltDown() { test "$prune" && pruneReactor # Generate the aggregator POM. - debug "Generating aggregator POM" + info "Generating aggregator POM" generatePOM # Build everything. if [ "$skipBuild" ] then - debug "Skipping the build; the command would have been:" - debug "mvn $args test" + info "Skipping the build; the command would have been:" + info "mvn $args test" else - debug "Building the project!" + info "Building the project!" # NB: All code is fresh; no need to clean. mvn $args test fi - debug "$1: complete" + info "$1: complete" } # -- Main -- diff --git a/tests/melting-pot-prune.t b/tests/melting-pot-prune.t index 9ea2c96..215e28d 100644 --- a/tests/melting-pot-prune.t +++ b/tests/melting-pot-prune.t @@ -1,21 +1,21 @@ Test that the '--prune' flag works as intended: $ sh "$TESTDIR/../melting-pot.sh" net.imagej:imagej-common:0.15.1 -r http://maven.imagej.net/content/groups/public -c org.scijava:scijava-common:2.44.2 -i 'org.scijava:*,net.imagej:*,net.imglib2:*' -p -v -f -s - [DEBUG] net.imagej:imagej-common:0.15.1: fetching project source - [DEBUG] net.imagej:imagej-common:0.15.1: determining project dependencies - [DEBUG] net.imagej:imagej-common:0.15.1: processing project dependencies - [DEBUG] net.imagej:imagej-common:0.15.1: imglib2-roi: fetching component source - [DEBUG] net.imagej:imagej-common:0.15.1: imglib2: fetching component source - [DEBUG] net.imagej:imagej-common:0.15.1: processing changed components - [DEBUG] Checking relevance of component net.imagej/imagej-common - [DEBUG] Checking relevance of component net.imglib2/imglib2 - [DEBUG] Pruning irrelevant component: net.imglib2/imglib2 - [DEBUG] Checking relevance of component net.imglib2/imglib2-roi - [DEBUG] Pruning irrelevant component: net.imglib2/imglib2-roi - [DEBUG] Generating aggregator POM - [DEBUG] Skipping the build; the command would have been: - [DEBUG] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test - [DEBUG] net.imagej:imagej-common:0.15.1: complete + [INFO] net.imagej:imagej-common:0.15.1: fetching project source + [INFO] net.imagej:imagej-common:0.15.1: determining project dependencies + [INFO] net.imagej:imagej-common:0.15.1: processing project dependencies + [INFO] net.imagej:imagej-common:0.15.1: imglib2-roi: fetching component source + [INFO] net.imagej:imagej-common:0.15.1: imglib2: fetching component source + [INFO] net.imagej:imagej-common:0.15.1: processing changed components + [INFO] Checking relevance of component net.imagej/imagej-common + [INFO] Checking relevance of component net.imglib2/imglib2 + [INFO] Pruning irrelevant component: net.imglib2/imglib2 + [INFO] Checking relevance of component net.imglib2/imglib2-roi + [INFO] Pruning irrelevant component: net.imglib2/imglib2-roi + [INFO] Generating aggregator POM + [INFO] Skipping the build; the command would have been: + [INFO] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test + [INFO] net.imagej:imagej-common:0.15.1: complete $ find melting-pot -maxdepth 2 melting-pot diff --git a/tests/melting-pot-simple.t b/tests/melting-pot-simple.t index 9ab0525..ee1d659 100644 --- a/tests/melting-pot-simple.t +++ b/tests/melting-pot-simple.t @@ -1,15 +1,15 @@ Down-the-middle test of a relatively simply project: $ sh "$TESTDIR/../melting-pot.sh" net.imagej:imagej-common:0.15.1 -r http://maven.imagej.net/content/groups/public -c org.scijava:scijava-common:2.44.2 -i 'org.scijava:*,net.imagej:*,net.imglib2:*,io.scif:*' -e net.imglib2:imglib2-roi -v -f -s - [DEBUG] net.imagej:imagej-common:0.15.1: fetching project source - [DEBUG] net.imagej:imagej-common:0.15.1: determining project dependencies - [DEBUG] net.imagej:imagej-common:0.15.1: processing project dependencies - [DEBUG] net.imagej:imagej-common:0.15.1: imglib2: fetching component source - [DEBUG] net.imagej:imagej-common:0.15.1: processing changed components - [DEBUG] Generating aggregator POM - [DEBUG] Skipping the build; the command would have been: - [DEBUG] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test - [DEBUG] net.imagej:imagej-common:0.15.1: complete + [INFO] net.imagej:imagej-common:0.15.1: fetching project source + [INFO] net.imagej:imagej-common:0.15.1: determining project dependencies + [INFO] net.imagej:imagej-common:0.15.1: processing project dependencies + [INFO] net.imagej:imagej-common:0.15.1: imglib2: fetching component source + [INFO] net.imagej:imagej-common:0.15.1: processing changed components + [INFO] Generating aggregator POM + [INFO] Skipping the build; the command would have been: + [INFO] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test + [INFO] net.imagej:imagej-common:0.15.1: complete $ find melting-pot -maxdepth 2 melting-pot From a405d6e8cf5fa4fc0fdbf43e6721fc0935289759 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 14:23:05 -0400 Subject: [PATCH 15/24] melting-pot: eliminate die function --- melting-pot.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 9f2caad..7bf6bdb 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -86,13 +86,6 @@ error() { stderr "[ERROR] $@" } -die() { - local code="$1" - shift - error $@ - exit "$code" -} - unknownArg() { error "Unknown option: $@" usage=1 @@ -207,7 +200,7 @@ parseArguments() { createDir() { test -z "$force" -a -e "$1" && - die 2 "Directory already exists: $1" + error "Directory already exists: $1" && exit 2 rm -rf "$1" mkdir -p "$1" From e6207c2680c1c9ad153f53a9637123b6f340884d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 14:23:40 -0400 Subject: [PATCH 16/24] melting-pot: fail if project source is not fetched --- melting-pot.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/melting-pot.sh b/melting-pot.sh index 7bf6bdb..636de20 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -444,6 +444,7 @@ meltDown() { # Fetch the project source code. info "$1: fetching project source" local dir="$(retrieveSource "$1" "$branch")" + test ! -d "$dir" && error "Could not fetch project source" && exit 3 # Get the project dependencies. info "$1: determining project dependencies" From cb2d382f652825da183f9b54b2b400f4ba24fa76 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 14:24:14 -0400 Subject: [PATCH 17/24] melting-pot: add a debugging flag --- melting-pot.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/melting-pot.sh b/melting-pot.sh index 636de20..24aa300 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -77,6 +77,11 @@ stderr() { >&2 echo "$@" } +debug() { + test "$debug" && + stderr "+ $@" +} + info() { test "$verbose" && stderr "[INFO] $@" @@ -129,6 +134,9 @@ parseArguments() { -v|--verbose) verbose=1 ;; + -d|--debug) + debug=1 + ;; -f|--force) force=1 ;; @@ -325,6 +333,7 @@ retrieveSource() { local scmBranch test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" local dir="$(groupId "$1")/$(artifactId "$1")" + debug "git clone \"$scmURL\" --branch \"$scmBranch\" --depth 1 \"$dir\"" git clone "$scmURL" --branch "$scmBranch" --depth 1 "$dir" 2> /dev/null echo "$dir" } @@ -332,6 +341,7 @@ retrieveSource() { # Gets the list of dependencies for the project in the CWD. deps() { cd "$1" + debug "mvn dependency:list" mvn dependency:list | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' cd - > /dev/null } @@ -499,6 +509,7 @@ meltDown() { else info "Building the project!" # NB: All code is fresh; no need to clean. + debug "mvn $args test" mvn $args test fi From db4875e105cb35f9138de1e3545fa60a80e381e7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 15:03:52 -0400 Subject: [PATCH 18/24] melting-pot: test the recursive SCM retrieval See 32ee1b45227184eec305ae54f531f65844daf56c. --- tests/melting-pot-multi.t | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/melting-pot-multi.t diff --git a/tests/melting-pot-multi.t b/tests/melting-pot-multi.t new file mode 100644 index 0000000..edb24e5 --- /dev/null +++ b/tests/melting-pot-multi.t @@ -0,0 +1,19 @@ +Test that recursive SCM retrieval works: + + $ sh "$TESTDIR/../melting-pot.sh" sc.fiji:TrakEM2_:1.0f -r http://maven.imagej.net/content/groups/public -i 'sc.fiji:TrakEM2_' -v -s -d -f + [INFO] sc.fiji:TrakEM2_:1.0f: fetching project source + + git clone "git://github.com/trakem2/TrakEM2" --branch "TrakEM2_-1.0f" --depth 1 "sc.fiji/TrakEM2_" + [INFO] sc.fiji:TrakEM2_:1.0f: determining project dependencies + + mvn dependency:list + [INFO] sc.fiji:TrakEM2_:1.0f: processing project dependencies + [INFO] sc.fiji:TrakEM2_:1.0f: processing changed components + [INFO] Generating aggregator POM + [INFO] Skipping the build; the command would have been: + [INFO] mvn -Denforcer.skip -Dnone.version= -Dmpicbg.version=1.0.1 -Djunit.version=4.11 -Dij.version=1.49p -Dhamcrest-core.version=1.3 -Dtools.version=1.4.2 -Djama.version=1.0.3 -Dmpicbg.version=1.0.1 -Dlegacy-imglib1.version=1.1.2-DEPRECATED -DFiji_Plugins.version=3.0.0 -Dlogback-core.version=1.1.1 -DVIB_.version=2.0.2 -Dmpicbg-trakem2.version=1.2.2 -Djoda-time.version=2.3 -D3D_Viewer.version=3.0.1 -Djama.version=1.0.3 -Dimglib2.version=2.2.1 -DLasso_and_Blow_Tool.version=2.0.1 -DSkeletonize3D_.version=1.0.1 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dvecmath.version=1.5.2 -Dj3d-core-utils.version=1.5.2 -Dformats-common.version=5.0.7 -Dimagej-common.version=0.12.2 -Dgentyref.version=1.1.0 -Djai-codec.version=1.1.3 -Djgoodies-forms.version=1.7.2 -Dperf4j.version=0.9.13 -Dmpicbg_.version=1.0.1 -Dbatik.version=1.8 -Dnative-lib-loader.version=2.0.2 -Djai_imageio.version=5.0.7 -DVIB-lib.version=2.0.1 -Djgoodies-common.version=1.7.0 -Djfreechart.version=1.0.19 -Dij.version=1.49p -DbUnwarpJ_.version=2.6.2 -Dslf4j-api.version=1.7.6 -Dspecification.version=5.0.7 -Dformats-api.version=5.0.7 -Dpal-optimization.version=2.0.0 -DVectorString.version=1.0.2 -Dpostgresql.version=8.2-507.jdbc3 -Djcommon.version=1.0.23 -Dlogback-classic.version=1.1.1 -Dome-xml.version=5.0.7 -Dtools.version=1.4.2 -Dj3d-core.version=1.5.2 -DAnalyzeSkeleton_.version=2.0.4 -Djavassist.version=3.16.1-GA -DSimple_Neurite_Tracer.version=2.0.3 -Dformats-bsd.version=5.0.7 -Dfiji-lib.version=2.1.0 -Dimglib2-ij.version=2.0.0-beta-30 -Dscijava-common.version=2.39.0 -Dturbojpeg.version=5.0.7 -Dcommons-math3.version=3.4.1 -Dij1-patcher.version=0.12.0 -Dimglib2-roi.version=0.3.0 -Djython-shaded.version=2.5.3 -Dkryo.version=2.21 -Djai-core.version=1.1.3 -Dmines-jtk.version=20100113 -Dtrove4j.version=3.0.3 -Dlevel_sets.version=1.0.1 test + [INFO] sc.fiji:TrakEM2_:1.0f: complete + + $ find melting-pot -maxdepth 2 + melting-pot + melting-pot/pom.xml + melting-pot/sc.fiji + melting-pot/sc.fiji/TrakEM2_ From e09487dff9be6cc6230a45980e013b44de7ddaec Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 15:21:28 -0400 Subject: [PATCH 19/24] melting-pot: cherry-pick child modules properly If the SCM link was a multi-module project, let's cherry-pick the relevant child module to include in the melting pot. This makes some assumptions: * The child module will be named the same as its artifactId. * It will be exactly one directory lower than the repository toplevel. Our main use case, TrakEM2, does conform to these expectations, though. --- melting-pot.sh | 15 ++++++++++++++- tests/melting-pot-multi.t | 5 ++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 24aa300..2dbd3bb 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -419,6 +419,12 @@ pruneReactor() { done } +# Tests if the given directory contains the appropriate source code. +isProject() { + local a="$(xpath "$1/pom.xml" project artifactId)" + test "$a" = "$(basename "$1")" && echo 1 +} + # Generates an aggregator POM for all modules in the current directory. generatePOM() { echo '' > pom.xml @@ -439,8 +445,15 @@ generatePOM() { local dir for dir in */* do - test -d "$dir" && + if [ "$(isProject "$dir")" ] + then echo " $dir" >> pom.xml + else + # Check for a child component of a multi-module project. + local childDir="$dir/$(basename "$dir")" + test "$(isProject "$childDir")" && + echo " $childDir" >> pom.xml + fi done echo ' ' >> pom.xml echo '' >> pom.xml diff --git a/tests/melting-pot-multi.t b/tests/melting-pot-multi.t index edb24e5..7c48e92 100644 --- a/tests/melting-pot-multi.t +++ b/tests/melting-pot-multi.t @@ -1,4 +1,4 @@ -Test that recursive SCM retrieval works: +Test that recursive SCM retrieval and multi-module projects work: $ sh "$TESTDIR/../melting-pot.sh" sc.fiji:TrakEM2_:1.0f -r http://maven.imagej.net/content/groups/public -i 'sc.fiji:TrakEM2_' -v -s -d -f [INFO] sc.fiji:TrakEM2_:1.0f: fetching project source @@ -17,3 +17,6 @@ Test that recursive SCM retrieval works: melting-pot/pom.xml melting-pot/sc.fiji melting-pot/sc.fiji/TrakEM2_ + + $ grep 'sc.fiji/TrakEM2_/TrakEM2_' melting-pot/pom.xml + \t\tsc.fiji/TrakEM2_/TrakEM2_ (esc) From 8b6dc8b7d0c3e0df0b634edf627606977e829080 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 16:31:30 -0400 Subject: [PATCH 20/24] melting-pot: explicitly check for prerequisites That way, things won't explode in weird ways. --- melting-pot.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/melting-pot.sh b/melting-pot.sh index 2dbd3bb..1888700 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -68,7 +68,6 @@ # assumption will not be properly overridden in the melting pot! # # Author: Curtis Rueden -# Dependencies: git, mvn, xmllint # ============================================================================ # -- Functions -- @@ -96,6 +95,19 @@ unknownArg() { usage=1 } +checkPrereqs() { + while [ $# -gt 0 ] + do + which $1 > /dev/null 2> /dev/null + test $? -ne 0 && echo "Missing prerequisite: $1" && exit 255 + shift + done +} + +verifyPrereqs() { + checkPrereqs git mvn xmllint +} + parseArguments() { while [ $# -gt 0 ] do @@ -531,6 +543,7 @@ meltDown() { # -- Main -- +verifyPrereqs parseArguments $@ createDir "$outputDir" meltDown "$project" From fef756abe76a17f0f5913e7a6064defe87016f6f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 17:30:05 -0400 Subject: [PATCH 21/24] melting-pot: add xmllint commands to debug output --- melting-pot.sh | 1 + tests/melting-pot-multi.t | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/melting-pot.sh b/melting-pot.sh index 1888700..62c870f 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -303,6 +303,7 @@ xpath() { xpath="$xpath/*[local-name()='$1']" shift done + debug "xmllint --xpath \"$xpath\" \"$xmlFile\"" xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | sed -E 's/^[^>]*>(.*)<[^<]*$/\1/' } diff --git a/tests/melting-pot-multi.t b/tests/melting-pot-multi.t index 7c48e92..0e1f127 100644 --- a/tests/melting-pot-multi.t +++ b/tests/melting-pot-multi.t @@ -2,12 +2,19 @@ Test that recursive SCM retrieval and multi-module projects work: $ sh "$TESTDIR/../melting-pot.sh" sc.fiji:TrakEM2_:1.0f -r http://maven.imagej.net/content/groups/public -i 'sc.fiji:TrakEM2_' -v -s -d -f [INFO] sc.fiji:TrakEM2_:1.0f: fetching project source + \+ xmllint --xpath ".*'project'.*'scm'.*'connection'.*" ".*/sc/fiji/TrakEM2_/1.0f/TrakEM2_-1.0f.pom" (re) + \+ xmllint --xpath ".*'project'.*'parent'.*'groupId'.*" ".*/sc/fiji/TrakEM2_/1.0f/TrakEM2_-1.0f.pom" (re) + \+ xmllint --xpath ".*'project'.*'parent'.*'artifactId'.*" ".*/sc/fiji/TrakEM2_/1.0f/TrakEM2_-1.0f.pom" (re) + \+ xmllint --xpath ".*'project'.*'parent'.*'version'.*" ".*/sc/fiji/TrakEM2_/1.0f/TrakEM2_-1.0f.pom" (re) + \+ xmllint --xpath ".*'project'.*'scm'.*'connection'.*" ".*/sc/fiji/pom-trakem2/1.3.2/pom-trakem2-1.3.2.pom" (re) + git clone "git://github.com/trakem2/TrakEM2" --branch "TrakEM2_-1.0f" --depth 1 "sc.fiji/TrakEM2_" [INFO] sc.fiji:TrakEM2_:1.0f: determining project dependencies + mvn dependency:list [INFO] sc.fiji:TrakEM2_:1.0f: processing project dependencies [INFO] sc.fiji:TrakEM2_:1.0f: processing changed components [INFO] Generating aggregator POM + + xmllint --xpath "//*[local-name()='project']/*[local-name()='artifactId']" "sc.fiji/TrakEM2_/pom.xml" + + xmllint --xpath "//*[local-name()='project']/*[local-name()='artifactId']" "sc.fiji/TrakEM2_/TrakEM2_/pom.xml" [INFO] Skipping the build; the command would have been: [INFO] mvn -Denforcer.skip -Dnone.version= -Dmpicbg.version=1.0.1 -Djunit.version=4.11 -Dij.version=1.49p -Dhamcrest-core.version=1.3 -Dtools.version=1.4.2 -Djama.version=1.0.3 -Dmpicbg.version=1.0.1 -Dlegacy-imglib1.version=1.1.2-DEPRECATED -DFiji_Plugins.version=3.0.0 -Dlogback-core.version=1.1.1 -DVIB_.version=2.0.2 -Dmpicbg-trakem2.version=1.2.2 -Djoda-time.version=2.3 -D3D_Viewer.version=3.0.1 -Djama.version=1.0.3 -Dimglib2.version=2.2.1 -DLasso_and_Blow_Tool.version=2.0.1 -DSkeletonize3D_.version=1.0.1 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dvecmath.version=1.5.2 -Dj3d-core-utils.version=1.5.2 -Dformats-common.version=5.0.7 -Dimagej-common.version=0.12.2 -Dgentyref.version=1.1.0 -Djai-codec.version=1.1.3 -Djgoodies-forms.version=1.7.2 -Dperf4j.version=0.9.13 -Dmpicbg_.version=1.0.1 -Dbatik.version=1.8 -Dnative-lib-loader.version=2.0.2 -Djai_imageio.version=5.0.7 -DVIB-lib.version=2.0.1 -Djgoodies-common.version=1.7.0 -Djfreechart.version=1.0.19 -Dij.version=1.49p -DbUnwarpJ_.version=2.6.2 -Dslf4j-api.version=1.7.6 -Dspecification.version=5.0.7 -Dformats-api.version=5.0.7 -Dpal-optimization.version=2.0.0 -DVectorString.version=1.0.2 -Dpostgresql.version=8.2-507.jdbc3 -Djcommon.version=1.0.23 -Dlogback-classic.version=1.1.1 -Dome-xml.version=5.0.7 -Dtools.version=1.4.2 -Dj3d-core.version=1.5.2 -DAnalyzeSkeleton_.version=2.0.4 -Djavassist.version=3.16.1-GA -DSimple_Neurite_Tracer.version=2.0.3 -Dformats-bsd.version=5.0.7 -Dfiji-lib.version=2.1.0 -Dimglib2-ij.version=2.0.0-beta-30 -Dscijava-common.version=2.39.0 -Dturbojpeg.version=5.0.7 -Dcommons-math3.version=3.4.1 -Dij1-patcher.version=0.12.0 -Dimglib2-roi.version=0.3.0 -Djython-shaded.version=2.5.3 -Dkryo.version=2.21 -Djai-core.version=1.1.3 -Dmines-jtk.version=20100113 -Dtrove4j.version=3.0.3 -Dlevel_sets.version=1.0.1 test [INFO] sc.fiji:TrakEM2_:1.0f: complete From 3e8116fbdb068348c28d1525d0971946f46962b7 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 17:31:23 -0400 Subject: [PATCH 22/24] melting-pot: sort the find output, for consistency The order of files returned is dependent on the file system and/or the version of find. Sorting avoids the issue. --- tests/melting-pot-multi.t | 2 +- tests/melting-pot-prune.t | 2 +- tests/melting-pot-simple.t | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/melting-pot-multi.t b/tests/melting-pot-multi.t index 0e1f127..99794a0 100644 --- a/tests/melting-pot-multi.t +++ b/tests/melting-pot-multi.t @@ -19,7 +19,7 @@ Test that recursive SCM retrieval and multi-module projects work: [INFO] mvn -Denforcer.skip -Dnone.version= -Dmpicbg.version=1.0.1 -Djunit.version=4.11 -Dij.version=1.49p -Dhamcrest-core.version=1.3 -Dtools.version=1.4.2 -Djama.version=1.0.3 -Dmpicbg.version=1.0.1 -Dlegacy-imglib1.version=1.1.2-DEPRECATED -DFiji_Plugins.version=3.0.0 -Dlogback-core.version=1.1.1 -DVIB_.version=2.0.2 -Dmpicbg-trakem2.version=1.2.2 -Djoda-time.version=2.3 -D3D_Viewer.version=3.0.1 -Djama.version=1.0.3 -Dimglib2.version=2.2.1 -DLasso_and_Blow_Tool.version=2.0.1 -DSkeletonize3D_.version=1.0.1 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dvecmath.version=1.5.2 -Dj3d-core-utils.version=1.5.2 -Dformats-common.version=5.0.7 -Dimagej-common.version=0.12.2 -Dgentyref.version=1.1.0 -Djai-codec.version=1.1.3 -Djgoodies-forms.version=1.7.2 -Dperf4j.version=0.9.13 -Dmpicbg_.version=1.0.1 -Dbatik.version=1.8 -Dnative-lib-loader.version=2.0.2 -Djai_imageio.version=5.0.7 -DVIB-lib.version=2.0.1 -Djgoodies-common.version=1.7.0 -Djfreechart.version=1.0.19 -Dij.version=1.49p -DbUnwarpJ_.version=2.6.2 -Dslf4j-api.version=1.7.6 -Dspecification.version=5.0.7 -Dformats-api.version=5.0.7 -Dpal-optimization.version=2.0.0 -DVectorString.version=1.0.2 -Dpostgresql.version=8.2-507.jdbc3 -Djcommon.version=1.0.23 -Dlogback-classic.version=1.1.1 -Dome-xml.version=5.0.7 -Dtools.version=1.4.2 -Dj3d-core.version=1.5.2 -DAnalyzeSkeleton_.version=2.0.4 -Djavassist.version=3.16.1-GA -DSimple_Neurite_Tracer.version=2.0.3 -Dformats-bsd.version=5.0.7 -Dfiji-lib.version=2.1.0 -Dimglib2-ij.version=2.0.0-beta-30 -Dscijava-common.version=2.39.0 -Dturbojpeg.version=5.0.7 -Dcommons-math3.version=3.4.1 -Dij1-patcher.version=0.12.0 -Dimglib2-roi.version=0.3.0 -Djython-shaded.version=2.5.3 -Dkryo.version=2.21 -Djai-core.version=1.1.3 -Dmines-jtk.version=20100113 -Dtrove4j.version=3.0.3 -Dlevel_sets.version=1.0.1 test [INFO] sc.fiji:TrakEM2_:1.0f: complete - $ find melting-pot -maxdepth 2 + $ find melting-pot -maxdepth 2 | sort melting-pot melting-pot/pom.xml melting-pot/sc.fiji diff --git a/tests/melting-pot-prune.t b/tests/melting-pot-prune.t index 215e28d..292801c 100644 --- a/tests/melting-pot-prune.t +++ b/tests/melting-pot-prune.t @@ -17,7 +17,7 @@ Test that the '--prune' flag works as intended: [INFO] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test [INFO] net.imagej:imagej-common:0.15.1: complete - $ find melting-pot -maxdepth 2 + $ find melting-pot -maxdepth 2 | sort melting-pot melting-pot/net.imagej melting-pot/net.imagej/imagej-common diff --git a/tests/melting-pot-simple.t b/tests/melting-pot-simple.t index ee1d659..1509c3e 100644 --- a/tests/melting-pot-simple.t +++ b/tests/melting-pot-simple.t @@ -11,7 +11,7 @@ Down-the-middle test of a relatively simply project: [INFO] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test [INFO] net.imagej:imagej-common:0.15.1: complete - $ find melting-pot -maxdepth 2 + $ find melting-pot -maxdepth 2 | sort melting-pot melting-pot/net.imagej melting-pot/net.imagej/imagej-common From 92f2ce34ada3c235c52abfafae47ffdf8ef172c5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 18:13:19 -0400 Subject: [PATCH 23/24] melting-pot: order components consistently This makes regression testing easier. --- melting-pot.sh | 3 ++- tests/melting-pot-multi.t | 2 +- tests/melting-pot-prune.t | 2 +- tests/melting-pot-simple.t | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index 62c870f..d9dc86e 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -355,7 +355,8 @@ retrieveSource() { deps() { cd "$1" debug "mvn dependency:list" - mvn dependency:list | grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' + mvn dependency:list | + grep '^\[INFO\] [^ ]' | sed 's/\[INFO\] //' | sort cd - > /dev/null } diff --git a/tests/melting-pot-multi.t b/tests/melting-pot-multi.t index 99794a0..0c87511 100644 --- a/tests/melting-pot-multi.t +++ b/tests/melting-pot-multi.t @@ -16,7 +16,7 @@ Test that recursive SCM retrieval and multi-module projects work: + xmllint --xpath "//*[local-name()='project']/*[local-name()='artifactId']" "sc.fiji/TrakEM2_/pom.xml" + xmllint --xpath "//*[local-name()='project']/*[local-name()='artifactId']" "sc.fiji/TrakEM2_/TrakEM2_/pom.xml" [INFO] Skipping the build; the command would have been: - [INFO] mvn -Denforcer.skip -Dnone.version= -Dmpicbg.version=1.0.1 -Djunit.version=4.11 -Dij.version=1.49p -Dhamcrest-core.version=1.3 -Dtools.version=1.4.2 -Djama.version=1.0.3 -Dmpicbg.version=1.0.1 -Dlegacy-imglib1.version=1.1.2-DEPRECATED -DFiji_Plugins.version=3.0.0 -Dlogback-core.version=1.1.1 -DVIB_.version=2.0.2 -Dmpicbg-trakem2.version=1.2.2 -Djoda-time.version=2.3 -D3D_Viewer.version=3.0.1 -Djama.version=1.0.3 -Dimglib2.version=2.2.1 -DLasso_and_Blow_Tool.version=2.0.1 -DSkeletonize3D_.version=1.0.1 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dvecmath.version=1.5.2 -Dj3d-core-utils.version=1.5.2 -Dformats-common.version=5.0.7 -Dimagej-common.version=0.12.2 -Dgentyref.version=1.1.0 -Djai-codec.version=1.1.3 -Djgoodies-forms.version=1.7.2 -Dperf4j.version=0.9.13 -Dmpicbg_.version=1.0.1 -Dbatik.version=1.8 -Dnative-lib-loader.version=2.0.2 -Djai_imageio.version=5.0.7 -DVIB-lib.version=2.0.1 -Djgoodies-common.version=1.7.0 -Djfreechart.version=1.0.19 -Dij.version=1.49p -DbUnwarpJ_.version=2.6.2 -Dslf4j-api.version=1.7.6 -Dspecification.version=5.0.7 -Dformats-api.version=5.0.7 -Dpal-optimization.version=2.0.0 -DVectorString.version=1.0.2 -Dpostgresql.version=8.2-507.jdbc3 -Djcommon.version=1.0.23 -Dlogback-classic.version=1.1.1 -Dome-xml.version=5.0.7 -Dtools.version=1.4.2 -Dj3d-core.version=1.5.2 -DAnalyzeSkeleton_.version=2.0.4 -Djavassist.version=3.16.1-GA -DSimple_Neurite_Tracer.version=2.0.3 -Dformats-bsd.version=5.0.7 -Dfiji-lib.version=2.1.0 -Dimglib2-ij.version=2.0.0-beta-30 -Dscijava-common.version=2.39.0 -Dturbojpeg.version=5.0.7 -Dcommons-math3.version=3.4.1 -Dij1-patcher.version=0.12.0 -Dimglib2-roi.version=0.3.0 -Djython-shaded.version=2.5.3 -Dkryo.version=2.21 -Djai-core.version=1.1.3 -Dmines-jtk.version=20100113 -Dtrove4j.version=3.0.3 -Dlevel_sets.version=1.0.1 test + [INFO] mvn -Denforcer.skip -Dbatik.version=1.8 -Dlogback-classic.version=1.1.1 -Dlogback-core.version=1.1.1 -Dkryo.version=2.21 -Dgentyref.version=1.1.0 -Djgoodies-common.version=1.7.0 -Djgoodies-forms.version=1.7.2 -Djai-codec.version=1.1.3 -Dtools.version=1.4.2 -Dtools.version=1.4.2 -Dmines-jtk.version=20100113 -Dudunits.version=4.3.18 -Djama.version=1.0.3 -Djama.version=1.0.3 -Dj3d-core-utils.version=1.5.2 -Dj3d-core.version=1.5.2 -Dvecmath.version=1.5.2 -Djai-core.version=1.1.3 -Djoda-time.version=2.3 -Djunit.version=4.11 -Dmpicbg.version=1.0.1 -Dmpicbg.version=1.0.1 -Dmpicbg_.version=1.0.1 -Dij1-patcher.version=0.12.0 -Dij.version=1.49p -Dij.version=1.49p -Dimagej-common.version=0.12.2 -Dimglib2-ij.version=2.0.0-beta-30 -Dimglib2-roi.version=0.3.0 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dnone.version= -Dformats-api.version=5.0.7 -Dformats-bsd.version=5.0.7 -Dformats-common.version=5.0.7 -Djai_imageio.version=5.0.7 -Dome-xml.version=5.0.7 -Dspecification.version=5.0.7 -Dturbojpeg.version=5.0.7 -Dcommons-math3.version=3.4.1 -Deventbus.version=1.4 -Dhamcrest-core.version=1.3 -Djavassist.version=3.16.1-GA -Djcommon.version=1.0.23 -Djfreechart.version=1.0.19 -Dperf4j.version=0.9.13 -Djython-shaded.version=2.5.3 -Dnative-lib-loader.version=2.0.2 -Dscijava-common.version=2.39.0 -Dslf4j-api.version=1.7.6 -Dpostgresql.version=8.2-507.jdbc3 -D3D_Viewer.version=3.0.1 -DAnalyzeSkeleton_.version=2.0.4 -DFiji_Plugins.version=3.0.0 -DLasso_and_Blow_Tool.version=2.0.1 -DSimple_Neurite_Tracer.version=2.0.3 -DSkeletonize3D_.version=1.0.1 -DVIB-lib.version=2.0.1 -DVIB_.version=2.0.2 -DVectorString.version=1.0.2 -DbUnwarpJ_.version=2.6.2 -Dfiji-lib.version=2.1.0 -Dlegacy-imglib1.version=1.1.2-DEPRECATED -Dlevel_sets.version=1.0.1 -Dmpicbg-trakem2.version=1.2.2 -Dpal-optimization.version=2.0.0 test [INFO] sc.fiji:TrakEM2_:1.0f: complete $ find melting-pot -maxdepth 2 | sort diff --git a/tests/melting-pot-prune.t b/tests/melting-pot-prune.t index 292801c..e1c6d6b 100644 --- a/tests/melting-pot-prune.t +++ b/tests/melting-pot-prune.t @@ -14,7 +14,7 @@ Test that the '--prune' flag works as intended: [INFO] Pruning irrelevant component: net.imglib2/imglib2-roi [INFO] Generating aggregator POM [INFO] Skipping the build; the command would have been: - [INFO] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test + [INFO] mvn -Denforcer.skip -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Djunit.version=4.11 -Dimglib2-roi.version=0.3.0 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Deventbus.version=1.4 -Dhamcrest-core.version=1.3 -Dscijava-common.version=2.44.2 test [INFO] net.imagej:imagej-common:0.15.1: complete $ find melting-pot -maxdepth 2 | sort diff --git a/tests/melting-pot-simple.t b/tests/melting-pot-simple.t index 1509c3e..6e817b6 100644 --- a/tests/melting-pot-simple.t +++ b/tests/melting-pot-simple.t @@ -8,7 +8,7 @@ Down-the-middle test of a relatively simply project: [INFO] net.imagej:imagej-common:0.15.1: processing changed components [INFO] Generating aggregator POM [INFO] Skipping the build; the command would have been: - [INFO] mvn -Denforcer.skip -Dimglib2-roi.version=0.3.0 -Djunit.version=4.11 -Dhamcrest-core.version=1.3 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Deventbus.version=1.4 -Dscijava-common.version=2.44.2 test + [INFO] mvn -Denforcer.skip -Dgentyref.version=1.1.0 -Dudunits.version=4.3.18 -Djunit.version=4.11 -Dimglib2-roi.version=0.3.0 -Dimglib2.version=2.2.1 -Dtrove4j.version=3.0.3 -Deventbus.version=1.4 -Dhamcrest-core.version=1.3 -Dscijava-common.version=2.44.2 test [INFO] net.imagej:imagej-common:0.15.1: complete $ find melting-pot -maxdepth 2 | sort From b67c0c828cf8e46e7158ee2cab4218c85beb85be Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 30 Jul 2015 17:36:56 -0400 Subject: [PATCH 24/24] melting-pot: add dependency:get to debug output Unfortunately, this makes the debugging output nondeterministic... so the cram tests might fail the first time they are run. But only the first time. Pragmatically, I am OK with it. --- melting-pot.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/melting-pot.sh b/melting-pot.sh index d9dc86e..5998306 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -276,12 +276,21 @@ pomPath() { # Fetches the POM for the given GAV into the local repository cache. downloadPOM() { + local g="$(groupId "$1")" + local a="$(artifactId "$1")" + local v="$(version "$1")" + debug "mvn dependency:get \\ + -DrepoUrl=\"$remoteRepos\" \\ + -DgroupId=\"$g\" \\ + -DartifactId=\"$a\" \\ + -Dversion=\"$v\" \\ + -Dpackaging=pom" mvn dependency:get \ -DrepoUrl="$remoteRepos" \ - -DgroupId="$(groupId "$1")" \ - -DartifactId="$(artifactId "$1")" \ - -Dversion="$(version "$1")" \ - -Dpackaging=pom + -DgroupId="$g" \ + -DartifactId="$a" \ + -Dversion="$v" \ + -Dpackaging=pom > /dev/null } # Gets the POM path for the given GAV, ensuring it exists locally.