diff --git a/ci-build.sh b/ci-build.sh new file mode 100755 index 0000000..93a5392 --- /dev/null +++ b/ci-build.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +# +# ci-build.sh - A script to build and/or release SciJava-based projects +# automatically using a continuous integration +# service. +# +# Required environment variables: +# BUILD_OS - the operating system running the current build (e.g. macOS) +# BUILD_REPOSITORY - the repository slug (org/repo) running the current build + +dir="$(dirname "$0")" + +success=0 +checkSuccess() { + # Log non-zero exit code. + test $1 -eq 0 || echo "==> FAILED: EXIT CODE $1" 1>&2 + + # Record the first non-zero exit code. + test $success -eq 0 && success=$1 +} + +# Build Maven projects. +if [ -f pom.xml ]; then + echo ::group::"= Maven build =" + echo + echo "== Configuring Maven ==" + + # NB: Suppress "Downloading/Downloaded" messages. + # See: https://stackoverflow.com/a/35653426/1207769 + export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + + # Populate the settings.xml configuration. + mkdir -p "$HOME/.m2" + settingsFile="$HOME/.m2/settings.xml" + customSettings=.ci/settings.xml + if [ -f "$customSettings" ]; then + cp "$customSettings" "$settingsFile" + else + cat >"$settingsFile" < + + + scijava.releases + $MAVEN_USER + $MAVEN_PASS + + + scijava.snapshots + $MAVEN_USER + $MAVEN_PASS + + + sonatype-nexus-releases + scijava-ci + $OSSRH_PASS + + +EOL + cat >>"$settingsFile" < + + gpg + + + $HOME/.gnupg + + + + $GPG_KEY_NAME + $GPG_PASSPHRASE + + + + +EOL + fi + + # Determine whether deploying will be possible. + deployOK= + ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) + + if [ $? -ne 0 ]; then + echo "No deploy -- could not extract ciManagement URL" + echo "Output of failed attempt follows:" + echo "$ciURL" + else + ciRepo=${ciURL##*/} + ciPrefix=${ciURL%/*} + ciOrg=${ciPrefix##*/} + if [ ! "$SIGNING_ASC" ] || [ ! "$GPG_KEY_NAME" ] || [ ! "$GPG_PASSPHRASE" ] || [ ! "$MAVEN_PASS" ] || [ ! "$OSSRH_PASS" ]; then + echo "No deploy -- secure environment variables not available" + elif [ "$BUILD_REPOSITORY" != "$ciOrg/$ciRepo" ]; then + echo "No deploy -- repository fork: $BUILD_REPOSITORY != $ciOrg/$ciRepo" + else + echo "All checks passed for artifact deployment" + deployOK=1 + fi + fi + + # Install GPG on OSX/macOS + if [ $BUILD_OS = 'macOS' ]; then + HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 + fi + + # Import the GPG signing key. + keyFile=.ci/signingkey.asc + if [ "$deployOK" ]; then + echo "== Importing GPG keypair ==" + mkdir -p .ci + echo "$SIGNING_ASC" > "$keyFile" + ls -la "$keyFile" + gpg --batch --fast-import "$keyFile" + checkSuccess $? + fi + + # Run the build. + BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' + if [ "$deployOK" ]; then + echo + echo "== Building and deploying master SNAPSHOT ==" + mvn -Pdeploy-to-scijava $BUILD_ARGS deploy + checkSuccess $? + elif [ "$deployOK" -a -f release.properties ]; then + echo + echo "== Cutting and deploying release version ==" + mvn -B $BUILD_ARGS release:perform + checkSuccess $? + echo "== Invalidating SciJava Maven repository cache ==" + curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && + gav=$(sh maven-helper.sh gav-from-pom pom.xml) && + ga=${gav%:*} && + echo "--> Artifact to invalidate = $ga" && + echo "machine maven.scijava.org" >"$HOME/.netrc" && + echo " login $MAVEN_USER" >>"$HOME/.netrc" && + echo " password $MAVEN_PASS" >>"$HOME/.netrc" && + sh maven-helper.sh invalidate-cache "$ga" + checkSuccess $? + else + echo + echo "== Building the artifact locally only ==" + mvn $BUILD_ARGS install javadoc:javadoc + checkSuccess $? + fi + echo ::endgroup:: +fi + +# Configure conda environment, if one is needed. +if [ -f environment.yml ]; then + echo ::group::"= Conda setup =" + + condaDir=$HOME/miniconda + condaSh=$condaDir/etc/profile.d/conda.sh + if [ ! -f "$condaSh" ]; then + echo + echo "== Installing conda ==" + python_version=${python -V} + python_version=${python_version:7:3} # get python version in the format like '2.7' + if [ "${python_version}" = "2.7" ]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + fi + rm -rf "$condaDir" + bash miniconda.sh -b -p "$condaDir" + checkSuccess $? + fi + + echo + echo "== Updating conda ==" + . "$condaSh" && + conda config --set always_yes yes --set changeps1 no && + conda update -q conda && + conda info -a + checkSuccess $? + + echo + echo "== Configuring environment ==" + condaEnv=ci-scijava + test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create + conda env "$condaAction" -n "$condaEnv" -f environment.yml && + conda activate "$condaEnv" + checkSuccess $? + + echo ::endgroup:: +fi + +# Execute Jupyter notebooks. +if which jupyter >/dev/null 2>/dev/null; then + echo ::group::"= Jupyter notebooks =" + # NB: This part is fiddly. We want to loop over files even with spaces, + # so we use the "find ... -print0 | while read $'\0' ..." idiom. + # However, that runs the piped expression in a subshell, which means + # that any updates to the success variable will not persist outside + # the loop. So we suppress all stdout inside the loop, echoing only + # the final value of success upon completion, and then capture the + # echoed value back into the parent shell's success variable. + success=$(find . -name '*.ipynb' -print0 | { + while read -d $'\0' nbf; do + echo 1>&2 + echo "== $nbf ==" 1>&2 + jupyter nbconvert --execute --stdout "$nbf" >/dev/null + checkSuccess $? + done + echo $success + }) + echo ::endgroup:: +fi + +exit $success diff --git a/github-action-ci.sh b/github-action-ci.sh new file mode 100755 index 0000000..e368959 --- /dev/null +++ b/github-action-ci.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# +# github-action-ci.sh - A script to set up ci-related environment variables from GitHub Actions +# + +echo "BUILD_REPOSITORY=${GITHUB_REPOSITORY}" +echo "BUILD_OS=${RUNNER_OS}" + +echo "BUILD_REPOSITORY=${GITHUB_REPOSITORY}" >> $GITHUB_ENV +echo "BUILD_OS=${RUNNER_OS}" >> $GITHUB_ENV diff --git a/github-actionify.sh b/github-actionify.sh new file mode 100755 index 0000000..37fc436 --- /dev/null +++ b/github-actionify.sh @@ -0,0 +1,320 @@ +#!/bin/sh + +# github-actionify.sh +# +# Script for enabling or updating GitHub Action builds for a given repository. + +# Environment variables: +# $EXEC - an optional prefix for bash commands (for example, if $EXEC=sudo, then the commands will be run as super user access) +# $@ - all positional parameters +# $0, $1, ... - specific positional parameters for each method + +#set -e + +dir="$(dirname "$0")" + +gitactionDir=.github +gitactionConfigRoot=/workflows/.gitaction.yml +gitactionConfig=$gitactionDir$gitactionConfigRoot +gitactionPRConfig=$gitactionDir/workflows/.gitaction-pr.yml +gitactionSetupScript=$gitactionDir/setup.sh +gitactionBuildScript=$gitactionDir/build.sh +gitactionSettingsFile=$gitactionDir/settings.xml +gitactionNotifyScript=$gitactionDir/notify.sh +credentialsDir=$HOME/.scijava/credentials +varsFile=$credentialsDir/vars +pomMinVersion='17.1.1' +tmpFile=gitaction.tmp + +info() { echo "- $@"; } +warn() { echo "[WARNING] $@" 1>&2; } +err() { echo "[ERROR] $@" 1>&2; } +die() { err "$@"; exit 1; } + +check() { + for tool in $@ + do + which "$tool" >/dev/null || + die "The '$tool' utility is required but not found" + done +} + +var() { + grep "^$1=" "$varsFile" || + die "$1 not found in $varsFile" +} + +update() { + file=$1 + msg=$2 + exe=$3 + test "$msg" || msg="GitHub Action: update $file" + if [ -e "$file" ] + then + if diff -q "$file" "$tmpFile" >/dev/null + then + info "$file is already OK" + else + info "Updating $file" + $EXEC rm -rf "$file" + $EXEC mv -f "$tmpFile" "$file" + fi + else + info "Creating $file" + $EXEC mkdir -p "$(dirname "$file")" + $EXEC mv "$tmpFile" "$file" + fi + rm -rf "$tmpFile" + $EXEC git add "$file" + if [ -n "$exe" ] + then + info "Adding execute permission to $file" + $EXEC git update-index --chmod=+x "$file" + fi + $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "$msg" +} + +process() { + cd "$1" + + # -- Git sanity checks -- + + repoSlug=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="scm"]/*[local-name()="connection"]' pom.xml|sed 's_.*github.com[:/]\(.*\)<.*_\1_') + test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' + case "$repoSlug" in + *.git) + die "GitHub repository slug ('$repoSlug') ends in '.git'; please fix the POM" + ;; + esac + git fetch >/dev/null + git diff-index --quiet HEAD -- || die "Dirty working copy" + currentBranch=$(git rev-parse --abbrev-ref HEAD) + upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) + remote=${upstreamBranch%/*} + defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') + test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" + git merge --ff --ff-only 'HEAD@{u}' >/dev/null || + die "Cannot fast forward (local diverging?)" +# test "$(git rev-parse HEAD)" = "$(git rev-parse 'HEAD@{u}')" || +# die "Mismatch with upstream branch (local ahead?)" + + # -- POM sanity checks -- + + parent=$(xmllint --xpath '//*[local-name()="project"]/*[local-name()="parent"]/*[local-name()="artifactId"]' pom.xml|sed 's/[^>]*>//'|sed 's/<.*//') + if [ -z "$SKIP_PARENT_CHECK" ] + then + test "$parent" = "pom-scijava" || + die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." + fi + + # Change pom.xml from Travis CI to GitHub Action + domain="github.com" + sed -i 's/Travis CI/GitHub Actions/g' pom.xml + sed -i "s/travis-ci.*/github.com\/$repoSlug\/actions\/workflows\/\.gitaction\.yml<\/url>/g" pom.xml + + # -- GitHub Action sanity checks -- + + test -e "$gitactionDir" -a ! -d "$gitactionDir" && die "$gitactionDir is not a directory" + test -e "$gitactionConfig" -a ! -f "$gitactionConfig" && die "$gitactionConfig is not a regular file" + test -e "$gitactionPRConfig" -a ! -f "$gitactionPRConfig" && die "$gitactionPRConfig is not a regular file" + test -e "$gitactionConfig" && warn "$gitactionConfig already exists" + test -e "$gitactionBuildScript" && warn "$gitactionBuildScript already exists" + test -e "$gitactionSetupScript" && warn "$gitactionSetupScript already exists" + + # -- Do things -- + + # Add/update the main GitHun Actions configuration file. + cat >"$tmpFile" <"$tmpFile" <"$tmpFile" <"$tmpFile" <]*>//'|sed 's/<.*//') + # HACK: Using a lexicographic comparison here is imperfect. + if [ "$version" \< "$pomMinVersion" ] + then + info 'Upgrading pom-scijava version' + sed "s|^ $version$| $pomMinVersion|" pom.xml >"$tmpFile" + update pom.xml "POM: update pom-scijava parent to $pomMinVersion" + else + info "Version of pom-scijava ($version) is OK" + fi + fi + + # ensure section is present + releaseProfile=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//') + if [ "$releaseProfile" ] + then + test "$releaseProfile" = 'deploy-to-scijava' || + warn "Unknown release profile: $releaseProfile" + else + info 'Adding property' + cp pom.xml "$tmpFile" + perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tdeploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" + update pom.xml 'POM: deploy releases to the SciJava repository' + fi + + # update the README + # https://docs.github.com/en/actions/managing-workflow-runs/adding-a-workflow-status-badge + if grep -q "travis-ci.*svg" README.md >/dev/null 2>&1 + then + info "Updating README.md GitHub Action badge" + sed "s/travis-ci.*/${domain//\//\\/}\/${repoSlug//\//\\/}\/actions${gitactionConfigRoot//\//\\/}\/badge\.svg\)\]\(https:\/\/$domain\/${repoSlug//\//\\/}\/actions${gitactionConfigRoot//\//\\/}\)/g" README.md >"$tmpFile" + update README.md 'GitHub Action: fix README.md badge link' + else + info "Adding GitHub Action badge to README.md" + echo "[![SciJava CI](https://$domain/$repoSlug/actions/$gitactionConfig/badge.svg)](https://$domain/$repoSlug/actions/$gitactionConfig)/g" README.md >"$tmpFile" + echo >>"$tmpFile" + test -f README.md && cat README.md >>"$tmpFile" + update README.md 'GitHub Action: add badge to README.md' + fi +} + +echo "Note that CI deployment requires additional configuration. Please contact a SciJava administrator for more information." + +# call check method to verify prerequisites +check git sed cut perl xmllint + +# parse arguments +EXEC=: +SKIP_PARENT_CHECK= +while test $# -gt 0 +do + case "$1" in + -f) EXEC=;; + -p) SKIP_PARENT_CHECK=true;; + --) break;; + -*) echo "Ignoring unknown option: $1" >&2; break;; + *) break;; + esac + shift +done + +test "$EXEC" && warn "Simulation only. Run with -f flag to go for real." + +# process arguments +if [ $# -gt 0 ] +then + for d in $@ + do ( + echo "[$d]" + process "$d" + ) done +else + process . +fi diff --git a/github-javadoc.sh b/github-javadoc.sh new file mode 100644 index 0000000..b680924 --- /dev/null +++ b/github-javadoc.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +# +# github-javadoc.sh - A script to build the javadocs of a SciJava-based project. +# + +# The following repositories are known to use this script: +# +# bonej-org/bonej-javadoc +# fiji/fiji-javadoc +# imagej/imagej-javadoc +# imglib/imglib2-javadoc +# scifio/scifio-javadoc +# scijava/java3d-javadoc +# scijava/scijava-javadoc +# slim-curve/slim-javadoc +# uw-loci/loci-javadoc + +# Wait for a launched background command to complete, emitting +# an occasional message to avoid long periods without output. +# Return the same exit code as the launched command. +keep_alive() { + pid="$1" + if [ "$pid" = "" ] + then + echo "[ERROR] No PID given" + return + fi + i=0 + while kill -0 "$pid" 2>/dev/null; do + i=$((i+1)) + m=$((i/60)) + s=$((i%60)) + test $s -eq 0 && echo "[$m minutes elapsed]" + sleep 1 + done + wait "$pid" +} + +ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) +ciRepo=${ciURL##*/} +ciPrefix=${ciURL%/*} +ciOrg=${ciPrefix##*/} +gitBranch=$(git branch --show-current) # get current branch name +curl -o pull-request.txt https://api.github.com/repos/$ciOrg/$ciRepo/pulls >/dev/null 2>&1 # Check for pull requests +curl -o secure-env.txt https://api.github.com/orgs/$ciOrg/$ciRepo/secrets >/dev/null 2>&1 # Check for secure env var +if [ grep -q "documentation_url" secure-env.txt \ + -a ! grep -q "url" pull-request.txt \ + -a "$gitBranch" = master ] +then + project=$1 + openssl_key=$2 + openssl_iv=$3 + + # Populate the settings.xml configuration. + mkdir -p "$HOME/.m2" + settingsFile="$HOME/.m2/settings.xml" + customSettings=.github/settings.xml + if [ -f "$customSettings" ] + then + cp "$customSettings" "$settingsFile" + else + # NB: Use maven.scijava.org as sole mirror if defined in . + test -f pom.xml && grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && + cat >"$settingsFile" < + + + scijava-mirror + SciJava mirror + https://maven.scijava.org/content/groups/public/ + * + + + +EOL + fi + + # Emit some details useful for debugging. + # NB: We run once with -q to suppress the download messages, + # then again without it to emit the desired dependency tree. + mvn -B -q dependency:tree && + mvn -B dependency:tree && + + echo && + echo "== Generating javadoc ==" && + + # Build the javadocs. + (mvn -B -q -Pbuild-javadoc) & + keep_alive $! && + test -d target/apidocs && + # Strip out date stamps, to avoid spurious changes being committed. + sed -i'' -e '/\(