56f910f7db
* Reconciliation tests PoC * removed debug statement * Added more reconciliation tests * Fixed typo * Fixed reconciliation test * test disabling error handling, troubleshooting * Attempt to fix flaky integration tests * restored error handling, fixed flaky test * Attempt to fix flaly test * Attempt to fix flaky tests
1016 lines
29 KiB
Bash
Executable File
1016 lines
29 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
##########################################################
|
|
# OpenRefine Control System #
|
|
##########################################################
|
|
|
|
# -------------- utility functions ----------------------
|
|
|
|
fail () {
|
|
cat <<EOF
|
|
ERROR: $1
|
|
Type '$0 -h' for usage information.
|
|
EOF
|
|
exit 1
|
|
}
|
|
|
|
error() {
|
|
echo "Error: $1"
|
|
exit 1
|
|
}
|
|
|
|
warn() {
|
|
echo "Warning: $1"
|
|
exit 0
|
|
}
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $0 [options] <action>
|
|
where [options] include:
|
|
|
|
-h print this message and exit
|
|
|
|
-p <port> the port that OpenRefine will listen to
|
|
default: 3333
|
|
|
|
-i <interface> the network interface OpenRefine should bind to
|
|
default: 127.0.0.1
|
|
|
|
-H <host> the expected value for the Host header (set to * to disable checks)
|
|
default: <interface>
|
|
|
|
-w <path> path to the webapp
|
|
default: main/webapp
|
|
|
|
-d <path> path to the data directory
|
|
default: OS dependent
|
|
|
|
-m <memory> max memory heap size to use
|
|
default: 1024M
|
|
Values less than 256M not allowed
|
|
|
|
-v <level> verbosity level [from low to high: error,warn,info,debug,trace]
|
|
default: info
|
|
|
|
-x <name=value> additional configuration parameters to pass to OpenRefine
|
|
default: [none]
|
|
|
|
--debug enable JVM debugging (on port 8000)
|
|
|
|
--jmx enable JMX monitoring (for jconsole and jvisualvm)
|
|
|
|
and <action> is one of
|
|
|
|
build ............................... Build OpenRefine
|
|
run ................................. Run OpenRefine [default]
|
|
|
|
test ................................ Run all OpenRefine tests
|
|
server_test ......................... Run only the server tests
|
|
ui_test <browser> <id> <key> ........ Run only the UI tests (If passing a project Id and a Record Key, tests will be recorded in Cypress.io Dashboard)
|
|
extensions_test ..................... Run only the extensions tests
|
|
|
|
broker .............................. Run OpenRefine Broker
|
|
|
|
broker_appengine_run <id> <ver> ..... Run OpenRefine Broker for Google App Engine in local server
|
|
broker_appengine_upload <id> <ver> .. Upload OpenRefine to Google App Engine
|
|
|
|
findbugs ............................ Run Findbugs against OpenRefine
|
|
pmd ................................. Run PMD against OpenRefine
|
|
cpd ................................. Run Copy/Paste Detection against OpenRefine
|
|
jslint .............................. Run JSlint against OpenRefine
|
|
|
|
whitespace <extension> .............. Normalize whitespace in files with the given extension
|
|
|
|
mac_dist <version> .................. Make MacOSX binary distribution
|
|
windows_dist <version> .............. Make Windows binary distribution
|
|
linux_dist <version> ................ Make Linux binary distribution
|
|
dist <version> ...................... Make all distributions
|
|
|
|
clean ............................... Clean compiled classes
|
|
distclean ........................... Remove all generated files
|
|
|
|
EOF
|
|
exit 1
|
|
}
|
|
|
|
add_option() {
|
|
if [ ! -z "$*" ] ; then
|
|
OPTS+=("$@")
|
|
fi
|
|
}
|
|
|
|
load_configs() {
|
|
TEMP_CONFIG=$(mktemp -t refine.XXXXXXX)
|
|
if [ "${TEMP_CONFIG}" = "" ] ; then
|
|
error "Could not create temporary file to load configurations"
|
|
fi
|
|
cat $1 | egrep "^[A-Z]" | sed 's/^\([^=]*\)=\(.*\)$/export \1=(\2)/' > ${TEMP_CONFIG}
|
|
. ${TEMP_CONFIG}
|
|
rm ${TEMP_CONFIG}
|
|
}
|
|
|
|
check_macosx() {
|
|
if [ "$OS" != "macosx" ] ; then
|
|
error "This action can only run on MacOSX"
|
|
fi
|
|
}
|
|
|
|
check_downloaders() {
|
|
CURL="`which curl 2> /dev/null`"
|
|
WGET="`which wget 2> /dev/null`"
|
|
|
|
if [ -z "$CURL" ] && [ -z "$WGET" ] ; then
|
|
error "We need either 'curl' or 'wget' present in PATH to download external dependencies."
|
|
fi
|
|
}
|
|
|
|
check_unzip() {
|
|
UNZIP="`which unzip 2> /dev/null`"
|
|
|
|
if [ -z "$UNZIP" ] ; then
|
|
error "We need 'unzip' present in PATH to expand external dependencies."
|
|
fi
|
|
}
|
|
|
|
check_python() {
|
|
PYTHON="`which python 2> /dev/null`"
|
|
if [ -z "$PYTHON" ] ; then
|
|
error "This action requires you to have 'python' installed and present in your PATH. You can download it for free at http://www.python.org/"
|
|
fi
|
|
PYTHON_VERSION="`$PYTHON --version 2>&1 | cut -f 2 -d ' ' | cut -f 1,2 -d .`"
|
|
if [ "$PYTHON_VERSION" != "2.6" ] && [ "$PYTHON_VERSION" != "2.7" ]; then
|
|
error "This action requires Python version 2.6.x. or 2.7.x. You can download it for free at http://www.python.org/"
|
|
fi
|
|
}
|
|
|
|
check_pywin32() {
|
|
PYWIN32="`$PYTHON -c 'import win32api' 2>&1`"
|
|
|
|
if [ ! -z "$PYWIN32" ] ; then
|
|
error "This action requires you to have 'pywin32' windows extensions for Python installed on your machine. You can download it for free at http://sourceforge.net/projects/pywin32/"
|
|
fi
|
|
}
|
|
|
|
check_running() {
|
|
check_downloaders
|
|
URL="http://${REFINE_HOST_INTERNAL}:${REFINE_PORT}/"
|
|
CHECK_STR="<title>OpenRefine</title>"
|
|
|
|
if [ "$CURL" ] ; then
|
|
curl -s -S -f $URL > /dev/null 2>&1
|
|
CURL_RETURN=$?
|
|
if [ $CURL_RETURN -eq "7" ] || [ $CURL_RETURN -eq "22" ] ; then
|
|
NOT_RUNNING="1"
|
|
fi
|
|
elif [ "$WGET" ] ; then
|
|
wget -O - $URL > /dev/null 2>&1
|
|
if [ "$?" = "4" ] ; then
|
|
NOT_RUNNING="1"
|
|
fi
|
|
fi
|
|
|
|
if [ -z "${NOT_RUNNING}" ] ; then
|
|
if [ "$CURL" ] ; then
|
|
RUNNING=`curl -s $URL | grep "$CHECK_STR"`
|
|
elif [ "$WGET" ] ; then
|
|
RUNNING=`wget -q -O - $URL | grep "$CHECK_STR"`
|
|
fi
|
|
|
|
if [ -z "${RUNNING}" ] ; then
|
|
error "Something is already running on $URL but doesn't seem to be OpenRefine. Maybe a proxy issue?"
|
|
fi
|
|
else
|
|
RUNNING=""
|
|
fi
|
|
}
|
|
|
|
get_version() {
|
|
VERSION="$1"
|
|
|
|
if [ -z "$VERSION" ] ; then
|
|
fail "Must specify a version number"
|
|
fi
|
|
|
|
NUM_VERSION=`echo $VERSION | sed -E 's/-.*//g'`
|
|
|
|
if [ "${NUM_VERSION}" = "" ] ; then
|
|
fail "${VERSION} is not a valid version number"
|
|
fi
|
|
|
|
if [ "`echo "${NUM_VERSION}" | egrep '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'`" = "${NUM_VERSION}" ] ; then
|
|
FULL_VERSION="${NUM_VERSION}"
|
|
elif [ "`echo "${NUM_VERSION}" | egrep '^[0-9]+\.[0-9]+\.[0-9]+$'`" = "${NUM_VERSION}" ] ; then
|
|
FULL_VERSION="${NUM_VERSION}.0"
|
|
elif [ "`echo "${NUM_VERSION}" | egrep ''^[0-9]+\.[0-9]+$''`" = "${NUM_VERSION}" ] ; then
|
|
FULL_VERSION="${NUM_VERSION}.0.0"
|
|
elif [ "`echo "${NUM_VERSION}" | egrep '^[0-9]+$'`" = "${NUM_VERSION}" ] ; then
|
|
FULL_VERSION="${NUM_VERSION}.0.0.0"
|
|
else
|
|
fail "${VERSION} is not a valid version number"
|
|
fi
|
|
}
|
|
|
|
get_revision() {
|
|
if [ -d ".svn" ] ; then
|
|
INFO=`svn info`
|
|
REVISION=`echo $INFO | sed s/^$VERSION-//`
|
|
elif [ -d ".git" ] ; then
|
|
INFO=`git describe`
|
|
REVISION=`echo $INFO`
|
|
REVISION=${REVISION:4}
|
|
else
|
|
error "cannot obtain revision, exiting!"
|
|
fi
|
|
}
|
|
|
|
download() {
|
|
URL=$1
|
|
DEST=$2
|
|
|
|
check_downloaders
|
|
|
|
if [ "$CURL" ] ; then
|
|
curl -L -o $DEST $URL || exit "Error while downloading $URL"
|
|
elif [ "$WGET" ] ; then
|
|
wget -O $DEST $URL || error "Error while downloading $URL"
|
|
fi
|
|
}
|
|
|
|
tool_download() {
|
|
URL=$1
|
|
FILE=$2
|
|
DIR=$3
|
|
|
|
cd $REFINE_TOOLS_DIR
|
|
if [ ! -f "$FILE" ] ; then
|
|
download $URL $FILE
|
|
fi
|
|
if [ ! -d "$DIR" ] ; then
|
|
if [ -z "`echo $FILE | sed 's@.*.tar.gz$@@' | sed 's@.*.tgz$@@'`" ] ; then
|
|
tar xzf $FILE || error "Error while expanding $FILE"
|
|
fi
|
|
if [ -z "`echo $FILE | sed 's@.*.zip$@@'`" ] ; then
|
|
check_unzip
|
|
$UNZIP -q $FILE || error "Error while expanding $FILE"
|
|
fi
|
|
fi
|
|
cd ..
|
|
}
|
|
|
|
load_data() {
|
|
FILE=$1
|
|
NAME=$2
|
|
URL="http://${REFINE_HOST_INTERNAL}:${REFINE_PORT}/command/core/create-project-from-upload"
|
|
CURL="`which curl 2> /dev/null`"
|
|
|
|
if [ -z "$CURL" ] ; then
|
|
error "We need 'curl' present in PATH to upload data to OpenRefine."
|
|
else
|
|
curl -s -F "project-file=@$FILE" -F "project-name=$NAME" $URL > /dev/null || error "Error while uploading $FILE to OpenRefine"
|
|
echo "Loaded $FILE as $NAME"
|
|
fi
|
|
}
|
|
|
|
display() {
|
|
FILE=$1
|
|
if [ "$OS" = "macosx" ] ; then
|
|
open $FILE
|
|
elif [ "$OS" = "linux" ] ; then
|
|
gnome-open $FILE
|
|
else
|
|
notepad $FILE
|
|
fi
|
|
|
|
}
|
|
# ----------------------------------------------------------------------------------------------
|
|
|
|
build_prepare() {
|
|
if [ ! -d $REFINE_BUILD_DIR ] ; then
|
|
mkdir $REFINE_BUILD_DIR || error "Error while making directory $REFINE_BUILD_DIR"
|
|
fi
|
|
}
|
|
|
|
dist_prepare() {
|
|
if [ ! -d $REFINE_DIST_DIR ] ; then
|
|
mkdir $REFINE_DIST_DIR || error "Error while making directory $REFINE_DIST_DIR"
|
|
fi
|
|
}
|
|
|
|
tools_prepare() {
|
|
if [ ! -d $REFINE_TOOLS_DIR ] ; then
|
|
mkdir $REFINE_TOOLS_DIR || error "Error while making directory $REFINE_TOOLS_DIR"
|
|
fi
|
|
}
|
|
|
|
mvn_prepare() {
|
|
tools_prepare
|
|
|
|
MVN_VERSION="3.6.3"
|
|
MVN_URL="https://www-us.apache.org/dist/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz"
|
|
MVN_FILE=`echo $MVN_URL | sed 's|.*/||'`
|
|
MVN_DIR="apache-maven-${MVN_VERSION}"
|
|
|
|
MVN="`which mvn 2> /dev/null`"
|
|
|
|
if [ -z "$MVN" ] ; then
|
|
if [ -z "$MVN_HOME" ] ; then
|
|
cd $REFINE_TOOLS_DIR
|
|
if [ ! -f "$MVN_FILE" ] ; then
|
|
echo "\nCould not find Maven locally, starting download for Maven ..."
|
|
download $MVN_URL $MVN_FILE
|
|
fi
|
|
if [ ! -d "$MVN_DIR" ] ; then
|
|
tar xzf $MVN_FILE -C . || error "Error while expanding $MVN_FILE"
|
|
fi
|
|
export MVN_HOME="`pwd`/$MVN_DIR"
|
|
if [ "$OS" = "windows" ] ; then
|
|
export MVN_HOME=`cygpath --unix "$MVN_HOME"`
|
|
fi
|
|
cd ..
|
|
fi
|
|
MVN="$MVN_HOME/bin/mvn"
|
|
fi
|
|
}
|
|
|
|
appengine_prepare() {
|
|
if [ -z "$APPENGINE_HOME" ] ; then
|
|
error "You have to have the APPENGINE_HOME environment variable set and pointing to the local installation of the Google AppEngine SDK."
|
|
elif [ ! -f "$APPENGINE_HOME/bin/appcfg.sh" ] ; then
|
|
error "Environment variable APPENGINE_HOME is set to '$APPENGINE_HOME' which doesn't point to a valid Google App Engine SDK."
|
|
fi
|
|
APPENGINE="$APPENGINE_HOME/bin/appcfg.sh"
|
|
APPENGINE_LOCAL="$APPENGINE_HOME/bin/dev_appserver.sh"
|
|
|
|
ANT_PARAMS="$ANT_PARAMS -Dappengine.sdk.dir=$APPENGINE_HOME"
|
|
}
|
|
|
|
virtualenv_prepare() {
|
|
check_python
|
|
|
|
VIRTUALENV_DIR="virtualenv-1.9.1"
|
|
VIRTUALENV_FILE="${VIRTUALENV_DIR}.tar.gz"
|
|
VIRTUALENV_URL="http://pypi.python.org/packages/source/v/virtualenv/${VIRTUALENV_FILE}"
|
|
|
|
tool_download $VIRTUALENV_URL $VIRTUALENV_FILE $VIRTUALENV_DIR
|
|
|
|
PYTHON_LOCAL="$REFINE_TOOLS_DIR/python"
|
|
if [ "$OS" = "windows" ] ; then
|
|
PYTHON_LOCAL="${PYTHON_LOCAL}_win"
|
|
fi
|
|
|
|
if [ ! -d "$PYTHON_LOCAL" ] ; then
|
|
$PYTHON $REFINE_TOOLS_DIR/$VIRTUALENV_DIR/virtualenv.py $PYTHON_LOCAL
|
|
fi
|
|
|
|
PYTHON_HOME="`pwd`/$PYTHON_LOCAL"
|
|
|
|
if [ "$OS" = "windows" ] ; then
|
|
PYTHON="$PYTHON_HOME/Scripts/python.exe"
|
|
PYTHON_INSTALL="$PYTHON_HOME/Scripts/easy_install.exe"
|
|
else
|
|
PYTHON="$PYTHON_HOME/bin/python"
|
|
PYTHON_INSTALL="$PYTHON_HOME/bin/easy_install"
|
|
fi
|
|
}
|
|
|
|
# ----------------------------------------------------------------------------------------------
|
|
|
|
mvn() {
|
|
mvn_prepare
|
|
|
|
"$MVN" $MVN_PARAMS -Dversion="$VERSION" -Dfull_version="$FULL_VERSION" $1 || error "Error while running maven task '$1'"
|
|
}
|
|
|
|
# ----------------------------------------------------------------------------------------------
|
|
|
|
dist() {
|
|
get_version $1
|
|
mvn_prepare
|
|
"$MVN" versions:set -DnewVersion="$VERSION"
|
|
"$MVN" package
|
|
}
|
|
|
|
windows_dist() {
|
|
get_version $1
|
|
mvn_prepare
|
|
"$MVN" versions:set -DnewVersion="$VERSION"
|
|
"$MVN" package -P windows
|
|
}
|
|
|
|
linux_dist() {
|
|
get_version $1
|
|
mvn_prepare
|
|
"$MVN" versions:set -DnewVersion="$VERSION"
|
|
"$MVN" package -P linux
|
|
}
|
|
|
|
# Kept just in case someone wants to follow this workflow on a mac,
|
|
# but no longer needed as "mvn package" does it directly on both mac and linux.
|
|
mac_dist() {
|
|
check_macosx
|
|
dist_prepare
|
|
get_version $1
|
|
get_revision
|
|
|
|
appbundler_prepare
|
|
|
|
ANT_PARAMS="-Dappbundler.dir=${REFINE_TOOLS_DIR}/${APPBUNDLER_DIR}"
|
|
ant mac
|
|
|
|
mkdir -p "$REFINE_BUILD_DIR/mac/.background"
|
|
cp graphics/dmg_background/dmg_background.png "$REFINE_BUILD_DIR/mac/.background/dmg_background.png"
|
|
|
|
SIZE=350
|
|
|
|
if [ -f "$REFINE_BUILD_DIR/temp_refine.dmg" ] ; then
|
|
rm "$REFINE_BUILD_DIR/temp_refine.dmg"
|
|
fi
|
|
|
|
# Sign the bundle with a self-signed cert so OS X doesn't frustrate users by making app invisible
|
|
codesign --deep -s "OpenRefine Code Signing" "$REFINE_BUILD_DIR/mac/OpenRefine.app"
|
|
spctl --assess --type execute --verbose=4 "$REFINE_BUILD_DIR/mac/OpenRefine.app"
|
|
|
|
TITLE="OpenRefine $VERSION"
|
|
echo "Building MacOSX DMG for $TITLE"
|
|
hdiutil create -srcfolder "$REFINE_BUILD_DIR/mac" -volname "$TITLE" -fs HFS+ -fsargs "-c c=64,a=16,e=16" -format UDRW -size ${SIZE}m "$REFINE_BUILD_DIR/temp_refine.dmg" || error "can't create empty DMG"
|
|
DEVICE=`hdiutil attach -readwrite -noverify -noautoopen "$REFINE_BUILD_DIR/temp_refine.dmg" | egrep '^/dev/' | sed -e "s/^\/dev\///g" -e 1q | awk '{print $1}'`
|
|
echo $DEVICE
|
|
hdiutil attach "$REFINE_BUILD_DIR/temp_refine.dmg" || error "Can't attach temp DMG"
|
|
|
|
echo '
|
|
tell application "Finder"
|
|
tell disk "'$TITLE'"
|
|
open
|
|
set current view of container window to icon view
|
|
set toolbar visible of container window to false
|
|
set statusbar visible of container window to false
|
|
set the bounds of container window to {200, 100, 760, 460}
|
|
set theViewOptions to the icon view options of container window
|
|
set arrangement of theViewOptions to not arranged
|
|
set icon size of theViewOptions to 100
|
|
set background picture of theViewOptions to file ".background:dmg_background.png"
|
|
make new alias file at container window to POSIX file "/Applications" with properties {name:"Applications"}
|
|
set position of item "OpenRefine" of container window to {170, 175}
|
|
set position of item "Applications" of container window to {380, 175}
|
|
close
|
|
open
|
|
update without registering applications
|
|
delay 5
|
|
eject
|
|
end tell
|
|
end tell
|
|
' | osascript || error "Error running applescript"
|
|
|
|
sync
|
|
sync
|
|
sleep 3
|
|
hdiutil detach $DEVICE
|
|
|
|
if [ -f "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg" ] ; then
|
|
rm "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg"
|
|
fi
|
|
|
|
hdiutil convert "$REFINE_BUILD_DIR/temp_refine.dmg" -format UDZO -imagekey zlib-level=9 -o "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg" || error "Error compressing DMG"
|
|
hdiutil internet-enable -yes "$REFINE_DIST_DIR/openrefine-mac-$VERSION.dmg" || error "Error internet-enabling DMG"
|
|
|
|
rm -f "$REFINE_BUILD_DIR/temp_refine.dmg"
|
|
}
|
|
|
|
test() {
|
|
mvn_prepare
|
|
$MVN test
|
|
}
|
|
|
|
ui_test() {
|
|
download http://okfnlabs.org/reconcile-csv/dist/reconcile-csv-0.1.2.jar ./tools/reconcile-csv-0.1.2.jar
|
|
RECONCILE_SERVER_CMD="$JAVA -Xmx2g -jar ./tools/reconcile-csv-0.1.2.jar ./main/tests/cypress/cypress/fixtures/csv-reconcile-species.csv scientific_name taxon_id"
|
|
echo "Starting reconcile-csv-0.1.2 ..."
|
|
$RECONCILE_SERVER_CMD 2>&1 &
|
|
RECONCILE_SERVER_PID="$!"
|
|
|
|
|
|
|
|
get_revision
|
|
|
|
BROWSER="$1"
|
|
CYPRESS_PROJECT_ID="$2"
|
|
CYPRESS_RECORD_KEY="$3"
|
|
CYPRESS_RECORD=0
|
|
|
|
if [ -z "$BROWSER" ] ; then
|
|
BROWSER="electron"
|
|
fi
|
|
|
|
if [ ! -z "$CYPRESS_PROJECT_ID" ] && [ ! -z "$CYPRESS_RECORD_KEY" ] ; then
|
|
CYPRESS_RECORD=1
|
|
echo "Tests will be recorded in Cypress Dashboard"
|
|
elif [ ! -z "$CYPRESS_PROJECT_ID" ] && [ -z "$CYPRESS_RECORD_KEY" ] ; then
|
|
fail "Found a Cypress project id but no record key"
|
|
fi
|
|
|
|
REFINE_DATA_DIR="${TMPDIR:=/tmp}/openrefine-tests"
|
|
|
|
add_option "-Drefine.headless=true"
|
|
add_option "-Drefine.autoreload=false"
|
|
add_option "-Dbutterfly.autoreload=false"
|
|
|
|
run fork > /dev/null
|
|
|
|
echo "Waiting for OpenRefine to load..."
|
|
sleep 5
|
|
check_running
|
|
if [ -z "$RUNNING" ] ; then
|
|
sleep 10
|
|
fi
|
|
echo "... proceed with the tests."
|
|
echo ""
|
|
|
|
echo "Starting Cypress..."
|
|
CYPRESS_RUN_CMD="yarn --cwd ./main/tests/cypress run cypress run --browser $BROWSER --headless --quiet --reporter list --env OPENREFINE_URL=http://$REFINE_HOST_INTERNAL:$REFINE_PORT"
|
|
if [ "$CYPRESS_RECORD" = "1" ] ; then
|
|
# if tests are recorded, project id is added to env vars, and --record flag is added to the cmd-line
|
|
export CYPRESS_PROJECT_ID=$CYPRESS_PROJECT_ID
|
|
CYPRESS_RUN_CMD="$CYPRESS_RUN_CMD --record --key $CYPRESS_RECORD_KEY --tag $BROWSER,$REVISION"
|
|
fi
|
|
export MOZ_FORCE_DISABLE_E10S=1
|
|
echo $CYPRESS_RUN_CMD
|
|
$CYPRESS_RUN_CMD
|
|
|
|
if [ "$?" = "0" ] ; then
|
|
UI_TEST_SUCCESS="1"
|
|
else
|
|
UI_TEST_SUCCESS="0"
|
|
fi
|
|
|
|
if [ "$CYPRESS_RECORD" = "1" ] ; then
|
|
echo "You can review tests on Cypress.io: https://dashboard.cypress.io/projects/$CYPRESS_PROJECT_ID/runs"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Killing OpenRefine"
|
|
/bin/kill -9 $REFINE_PID
|
|
echo "Killing Reconciliation Server"
|
|
/bin/kill -9 $RECONCILE_SERVER_PID
|
|
echo "Cleaning up"
|
|
rm -rf "$REFINE_DATA_DIR"
|
|
|
|
if [ "$UI_TEST_SUCCESS" = "0" ] ; then
|
|
error "The UI test suite failed."
|
|
fi
|
|
}
|
|
|
|
server_test() {
|
|
mvn_prepare
|
|
$MVN test -f main
|
|
}
|
|
|
|
extensions_test() {
|
|
mvn_prepare
|
|
$MVN test -f extensions
|
|
}
|
|
|
|
run() {
|
|
FORK=$1
|
|
|
|
check_running
|
|
|
|
if [ "$RUNNING" ] ; then
|
|
warn "OpenRefine is already running."
|
|
fi
|
|
|
|
if [ ! -d $REFINE_CLASSES_DIR ] ; then
|
|
IS_JAR=`ls $REFINE_LIB_DIR | grep openrefine`
|
|
if [ -z "$IS_JAR" ] ; then
|
|
mvn_prepare
|
|
$MVN process-resources
|
|
$MVN compile test-compile
|
|
echo ""
|
|
fi
|
|
fi
|
|
|
|
if [ -d $REFINE_CLASSES_DIR ] ; then
|
|
add_option "-Drefine.autoreload=true" "-Dbutterfly.autoreload=true"
|
|
fi
|
|
|
|
if [ "$OS" = "macosx" ] ; then
|
|
add_option '-Xdock:icon=graphics/icon/openrefine.icns'
|
|
fi
|
|
|
|
if [ "$REFINE_DATA_DIR" ] ; then
|
|
add_option "-Drefine.data_dir=$REFINE_DATA_DIR"
|
|
fi
|
|
|
|
if [ "$REFINE_WEBAPP" ] ; then
|
|
add_option "-Drefine.webapp=$REFINE_WEBAPP"
|
|
fi
|
|
|
|
if [ "$REFINE_PORT" ] ; then
|
|
add_option "-Drefine.port=$REFINE_PORT"
|
|
fi
|
|
|
|
if [ "$REFINE_INTERFACE" ] ; then
|
|
add_option "-Drefine.interface=$REFINE_INTERFACE"
|
|
fi
|
|
|
|
if [ "$REFINE_HOST" ] ; then
|
|
add_option "-Drefine.host=$REFINE_HOST"
|
|
fi
|
|
|
|
if [ "$REFINE_AUTOSAVE_PERIOD" ] ; then
|
|
add_option "-Drefine.autosave=$REFINE_AUTOSAVE_PERIOD"
|
|
fi
|
|
|
|
CLASSPATH="$REFINE_CLASSES_DIR${SEP}$REFINE_LIB_DIR/*"
|
|
|
|
RUN_CMD=("$JAVA" -cp "$CLASSPATH" "${OPTS[@]}" "com.google.refine.Refine")
|
|
echo "${RUN_CMD[@]}"
|
|
|
|
echo "Starting OpenRefine at 'http://${REFINE_HOST_INTERNAL}:${REFINE_PORT}/'"
|
|
echo ""
|
|
|
|
if [ -z "$FORK" ] ; then
|
|
exec "${RUN_CMD[@]}"
|
|
else
|
|
"${RUN_CMD[@]}" &
|
|
REFINE_PID="$!"
|
|
fi
|
|
}
|
|
|
|
broker_build() {
|
|
build_prepare
|
|
get_revision
|
|
# TODO migrate to Maven
|
|
$MVN prepare_broker
|
|
}
|
|
|
|
broker_run() {
|
|
FORK=$1
|
|
|
|
if [ ! -d "broker/core/WEB-INF/lib" ] ; then
|
|
broker_build
|
|
echo ""
|
|
fi
|
|
|
|
if [ -d $REFINE_CLASSES_DIR ] ; then
|
|
add_option "-Drefine.autoreload=true" "-Dbutterfly.autoreload=true"
|
|
add_option "-Drefine.development=true"
|
|
fi
|
|
|
|
add_option "-Drefine.webapp=broker/core"
|
|
add_option "-Drefine.headless=true"
|
|
|
|
add_option "-Drefine.port=$REFINE_PORT"
|
|
add_option "-Drefine.host=0.0.0.0"
|
|
|
|
LIB_PATHS=`cat server/classpath.txt`
|
|
CLASSPATH="$REFINE_CLASSES_DIR${SEP}$LIB_PATHS"
|
|
|
|
RUN_CMD="$JAVA -cp $CLASSPATH $OPTS com.google.refine.Refine"
|
|
|
|
#echo "$RUN_CMD"
|
|
#echo ""
|
|
|
|
echo "Starting OpenRefine Broker at 'http://0.0.0.0:${REFINE_PORT}/'"
|
|
echo ""
|
|
|
|
if [ -z "$FORK" ] ; then
|
|
exec $RUN_CMD
|
|
else
|
|
$RUN_CMD &
|
|
REFINE_PID="$!"
|
|
fi
|
|
|
|
}
|
|
|
|
broker_appengine_build() {
|
|
appengine_prepare
|
|
get_revision
|
|
|
|
MVN_PARAMS="-Dappengine.sdk.dir=${APPENGINE_HOME}"
|
|
|
|
if [ "$1" ] ; then
|
|
MVN_PARAMS="$MVN_PARAMS -Dappengine.app_id=$1"
|
|
fi
|
|
|
|
if [ "$2" ] ; then
|
|
MVN_PARAMS="$MVN_PARAMS -Dappengine.version=$2"
|
|
fi
|
|
|
|
# TODO migrate this to Maven
|
|
$MVN prepare_broker_appengine
|
|
}
|
|
|
|
broker_appengine_upload() {
|
|
broker_appengine_build $1 $2
|
|
"$APPENGINE" update "$REFINE_BUILD_DIR/broker/appengine"
|
|
}
|
|
|
|
broker_appengine_run() {
|
|
broker_appengine_build $1 $2
|
|
"$APPENGINE_LOCAL" "$REFINE_BUILD_DIR/broker/appengine"
|
|
}
|
|
|
|
findbugs() {
|
|
findbugs_prepare
|
|
|
|
MVN_PARAMS="-Dfindbugs.dir=${REFINE_TOOLS_DIR}/${FINDBUGS_DIR}"
|
|
ant findbugs
|
|
|
|
display "$REFINE_BUILD_DIR/reports/findbugs.html"
|
|
}
|
|
|
|
pmd() {
|
|
pmd_prepare
|
|
|
|
ANT_PARAMS="-Dpmd.dir=${REFINE_TOOLS_DIR}/${PMD_DIR}"
|
|
ant pmd
|
|
|
|
display "$REFINE_BUILD_DIR/reports/pmd.html"
|
|
}
|
|
|
|
cpd() {
|
|
pmd_prepare
|
|
|
|
ANT_PARAMS="-Dpmd.dir=${REFINE_TOOLS_DIR}/${PMD_DIR}"
|
|
ant cpd
|
|
|
|
display "$REFINE_BUILD_DIR/reports/cpd.txt"
|
|
}
|
|
|
|
jslint() {
|
|
jslint_prepare
|
|
|
|
ANT_PARAMS="-Djslint.dir=${REFINE_TOOLS_DIR}/${JSLINT_DIR}"
|
|
ant jslint
|
|
|
|
display "$REFINE_BUILD_DIR/reports/jslint.txt"
|
|
}
|
|
|
|
whitespace() {
|
|
[ $# -gt 0 ] || usage
|
|
|
|
for i in `find . -name *.$1`; do
|
|
# expand tabs to spaces
|
|
expand -t 4 < $i > $i.1
|
|
|
|
# convert DOS to UNIX newlines
|
|
tr -d '\r' < $i.1 > $i.2
|
|
|
|
rm $i $i.1
|
|
mv $i.2 $i
|
|
done
|
|
}
|
|
|
|
checkJavaMajorVersion() {
|
|
java_ver=$("$JAVA" -version 2>&1 | grep version | cut -d ' ' -f 3 | tr -d \")
|
|
# Java 6, 7, 8 starts with 1.x
|
|
if [ ${java_ver:0:2} == "1." ] ; then
|
|
major=`echo ${java_ver} | sed -E 's/1\.([0-9])[0-9_.]{2,6}/\1/g'`
|
|
else
|
|
# Java 9+ starts with x using semver versioning
|
|
major=`echo ${java_ver} | sed -E 's/([0-9]+)(-ea|(\.[0-9]+)*)/\1/g'`
|
|
fi
|
|
if (( ${major} < 8 )); then
|
|
error "OpenRefine requires Java version 8 or later. If you have multiple versions of Java installed, please set the environment variable JAVA_HOME to the correct version."
|
|
fi
|
|
if (( ${major} > 12 )); then
|
|
echo "WARNING: OpenRefine is not tested and not recommended for use with Java versions greater than 12."
|
|
fi
|
|
}
|
|
|
|
# -------------------------- script -----------------------------
|
|
|
|
# ----- Normalize the current directory -------------------------
|
|
|
|
cd `dirname $0`
|
|
|
|
# ----- Default values ------------------------------------------
|
|
|
|
OPTS=()
|
|
|
|
# ---- OS-specific support --------------------------------------
|
|
|
|
SYSTEM=`uname`
|
|
|
|
case "$SYSTEM" in
|
|
CYGWIN*) OS="windows" ;;
|
|
Darwin*) OS="macosx" ;;
|
|
Linux*) OS="linux" ;;
|
|
*) OS="other" ;;
|
|
esac
|
|
|
|
SEP=":"
|
|
if [ "$OS" = "windows" ] ; then
|
|
SEP=";"
|
|
fi
|
|
|
|
# ----- Load configurations -------------------------------------
|
|
|
|
if [ -f "refine-dev.ini" ]; then
|
|
echo "Using refine-dev.ini for configuration"
|
|
load_configs refine-dev.ini
|
|
else
|
|
echo "Using refine.ini for configuration"
|
|
load_configs refine.ini
|
|
fi
|
|
|
|
# ----- Make sure there is an appropriate java environment is available -------------
|
|
|
|
if [ "$OS" = "macosx" ] ; then
|
|
if [ -z "$JAVA_HOME" ] ; then
|
|
# We need want recent Java because we're bundling JRE - may want to warn and force developer to set JAVA_HOME
|
|
# The /usr/libexec/java_home utility may be tied to the Java prefs app, so could go away when Apple removes it
|
|
export JAVA_HOME=$(/usr/libexec/java_home)
|
|
fi
|
|
fi
|
|
|
|
if [ "$JAVA_HOME" ] ; then
|
|
JAVA="$JAVA_HOME/bin/java"
|
|
else
|
|
JAVA="`which java 2> /dev/null`"
|
|
fi
|
|
|
|
if [ ! -x "$JAVA" ] ; then
|
|
error "Could not find the 'java' executable at '$JAVA', are you sure your JAVA_HOME environment variable is pointing to a proper java installation?"
|
|
fi
|
|
|
|
checkJavaMajorVersion
|
|
|
|
# ----- Parse the command line args ------------------------------------------
|
|
|
|
while [ $# -ne 0 ] ; do
|
|
case "$1" in
|
|
-h) usage;;
|
|
-p) shift; REFINE_PORT="$1"; shift; continue;;
|
|
-H) shift; REFINE_HOST="$1"; shift; continue;;
|
|
-i) shift; REFINE_INTERFACE="$1"; shift; continue;;
|
|
-w) shift; REFINE_WEBAPP="$1"; shift; continue;;
|
|
-d) shift; REFINE_DATA_DIR="$1"; shift; continue;;
|
|
-m)
|
|
shift;
|
|
if [ $1 -le 256 ]
|
|
then
|
|
fail "Value of -m less than 256 not allowed";
|
|
else
|
|
REFINE_MEMORY="$1";
|
|
shift;
|
|
continue;
|
|
fi;;
|
|
-v) shift; REFINE_VERBOSITY="$1"; shift; continue;;
|
|
-x) shift; REFINE_EXTRA_OPTS="$1"; shift; continue;;
|
|
--debug) shift; add_option '-Xdebug' '-Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=n'; continue;;
|
|
--jmx) shift; add_option '-Dcom.sun.management.jmxremote'; continue;;
|
|
-*) fail "Invalid option: $1";;
|
|
*) break;;
|
|
esac
|
|
done
|
|
|
|
if [ $# -ne 0 ] ; then
|
|
ACTION=$1; shift
|
|
fi
|
|
|
|
if [ -z "$ACTION" ] ; then
|
|
ACTION="run"
|
|
fi
|
|
|
|
# ----- Verify and Set Required Environment Variables -------------------------
|
|
|
|
if [ -z "$JAVA_OPTIONS" ] ; then
|
|
JAVA_OPTIONS=""
|
|
fi
|
|
add_option "${JAVA_OPTIONS[@]}"
|
|
|
|
if [ -z "$REFINE_MEMORY" ] ; then
|
|
REFINE_MEMORY="1024M"
|
|
fi
|
|
if [ -z "$REFINE_MIN_MEMORY" ] ; then
|
|
REFINE_MIN_MEMORY="256M"
|
|
fi
|
|
add_option "-Xms$REFINE_MIN_MEMORY" "-Xmx$REFINE_MEMORY" "-Drefine.memory=$REFINE_MEMORY"
|
|
|
|
freeRam=UNKNOWN
|
|
if [ "$OS" = "macosx" ] ; then
|
|
freeRam=$(top -l 1 | grep PhysMem | awk '{print $6}' | tr -d M)
|
|
elif [ "$OS" = "linux" ] ; then
|
|
freeRam=$(free -m | grep -oP '\d+' | head -n 1)
|
|
fi
|
|
|
|
echo You have "$freeRam"M of free memory.
|
|
echo Your current configuration is set to use $REFINE_MEMORY of memory.
|
|
echo OpenRefine can run better when given more memory. Read our FAQ on how to allocate more memory here:
|
|
echo https://github.com/OpenRefine/OpenRefine/wiki/FAQ-Allocate-More-Memory
|
|
|
|
if [ -z "$REFINE_MAX_FORM_CONTENT_SIZE" ] ; then
|
|
REFINE_MAX_FORM_CONTENT_SIZE="1048576"
|
|
fi
|
|
add_option "-Drefine.max_form_content_size=$REFINE_MAX_FORM_CONTENT_SIZE"
|
|
|
|
if [ -z "$REFINE_PORT" ] ; then
|
|
REFINE_PORT="3333"
|
|
fi
|
|
|
|
if [ -z "$REFINE_INTERFACE" ] ; then
|
|
REFINE_INTERFACE="127.0.0.1"
|
|
fi
|
|
|
|
if [ -z "$REFINE_HOST" ] ; then
|
|
if [ "$REFINE_INTERFACE" = "0.0.0.0" ] ; then
|
|
REFINE_HOST='*'
|
|
else
|
|
REFINE_HOST="$REFINE_INTERFACE"
|
|
fi
|
|
fi
|
|
|
|
if [ "$REFINE_HOST" = '*' ] ; then
|
|
echo No host specified while binding to interface 0.0.0.0, guessing localhost.
|
|
REFINE_HOST_INTERNAL="localhost"
|
|
else
|
|
REFINE_HOST_INTERNAL="$REFINE_HOST"
|
|
fi
|
|
|
|
if [ -z "$REFINE_WEBAPP" ] ; then
|
|
REFINE_WEBAPP="main/webapp"
|
|
fi
|
|
|
|
if [ -z "$REFINE_TEST_DIR" ] ; then
|
|
REFINE_TEST_DIR="main/tests"
|
|
fi
|
|
|
|
if [ -z "$REFINE_CLASSES_DIR" ] ; then
|
|
REFINE_CLASSES_DIR="server/classes"
|
|
fi
|
|
|
|
if [ -z "$REFINE_LIB_DIR" ] ; then
|
|
REFINE_LIB_DIR="server/target/lib"
|
|
fi
|
|
|
|
if [ -z "$REFINE_BUILD_DIR" ] ; then
|
|
REFINE_BUILD_DIR="build"
|
|
fi
|
|
|
|
if [ -z "$REFINE_TOOLS_DIR" ] ; then
|
|
REFINE_TOOLS_DIR="tools"
|
|
fi
|
|
|
|
if [ -z "$REFINE_DIST_DIR" ] ; then
|
|
REFINE_DIST_DIR="dist"
|
|
fi
|
|
|
|
if [ -z "$REFINE_VERBOSITY" ] ; then
|
|
REFINE_VERBOSITY="info"
|
|
fi
|
|
add_option "-Drefine.verbosity=$REFINE_VERBOSITY"
|
|
|
|
if [ ! -z "$REFINE_EXTRA_OPTS" ] ; then
|
|
add_option "-D$REFINE_EXTRA_OPTS"
|
|
fi
|
|
|
|
if [ -z "$JYTHONPATH" ] ; then
|
|
JYTHONPATH="$REFINE_WEBAPP/WEB-INF/lib/jython"
|
|
else
|
|
JYTHONPATH="$REFINE_WEBAPP/WEB-INF/lib/jython${SEP}$JYTHONPATH"
|
|
fi
|
|
|
|
add_option "-Dpython.path=$JYTHONPATH"
|
|
add_option "-Dpython.cachedir=$HOME/.local/share/google/refine/cachedir"
|
|
|
|
if [ ! -z "$GDATA_CLIENT_ID" ] ; then
|
|
if [ ! -z "$GDATA_CLIENT_SECRET" ] ; then
|
|
if [ ! -z "$GDATA_API_KEY" ] ; then
|
|
add_option "-Dext.gdata.clientid=$GDATA_CLIENT_ID" "-Dext.gdata.clientsecret=$GDATA_CLIENT_SECRET" "-Dext.gdata.apikey=$GDATA_API_KEY"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ----- Respond to the action given --------------------------------------------
|
|
case "$ACTION" in
|
|
build) build_prepare; mvn process-resources; mvn compile\ test-compile;;
|
|
clean) mvn clean;;
|
|
whitespace) whitespace $1;;
|
|
distclean) mvn distclean;;
|
|
test) test $1;;
|
|
tests) test $1;;
|
|
ui_test) ui_test $1 $2 $3;;
|
|
ui_tests) ui_test $1 $2 $3;;
|
|
server_test) server_test $1;;
|
|
server_tests) server_test $1;;
|
|
extensions_test) extensions_test $1;;
|
|
extensions_tests) extensions_test $1;;
|
|
findbugs) findbugs;;
|
|
pmd) pmd;;
|
|
cpd) cpd;;
|
|
jslint) jslint;;
|
|
run) run;;
|
|
broker) broker_run;;
|
|
broker_appengine_run) broker_appengine_run $1 $2;;
|
|
broker_appengine_upload) broker_appengine_upload $1 $2;;
|
|
mac_dist) mac_dist $1;;
|
|
windows_dist) windows_dist $1;;
|
|
linux_dist) linux_dist $1;;
|
|
dist) dist $1;;
|
|
*) usage; ;;
|
|
esac
|
|
|
|
# ----------- end of file --------------------
|