Migrate from Travis to GitHub Actions (#3378)

* Only keep JRE 8 and 14 in the build matrix.

For #3377.

* Run tests in GitHub Actions on each PR

* Attempt to fix Postgres configuration

* Set explicit password for Postgres on GitHub Actions

* Set explicit password for MySQL and MariaDB

* Fix credentials for postgres and mysql

* Fix duplicate id in GitHub workflow

* Fix creation of test_db on MySQL

* Revert back to GH Action MySQL service

* Populate initial test databases

* Fix syntax of workflow file

* Reorder steps to give more time for MySQL to boot

* Run MySQL database as a service, forward ports to config

* Reformat MySQL options

* Fix YAML syntax

* Add missing 'steps' field

* Fix connection to MySQL and Postgres

* Add back explicit database creation steps

* Force TCP/IP connection for postgres

* Remove explicit creation of test database for postgres

* Fix Postgres and Mariadb configs

* Fix parameter spelling for mariadb

* Display MariaDB test configuration

* Fix more inconsistent parameter names

* Fix more inconsistent parameter names

* Attempt to use Coveralls maven plugin instead of GH action

* Fix workflow file

* Enable submission to coveralls

* Supply coveralls token

* Remove Travis-specific configuration files

* Also update appveyor script after rename of SQL files

* Reintroduce packaging/test_pom.xml used by Appveyor

* Update filenames in appveyor.yml

.github/workflows/pull_request.yml

@@ -0,0 +1,71 @@
+name: Java CI
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: 'postgres'
+          POSTGRES_DB: test_db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      mysql:
+        image: mysql:8
+        ports:
+          - 3306
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 5s
+          --health-timeout 2s
+          --health-retries 3
+
+    steps:
+    - uses: actions/checkout@v2.3.4
+
+    - name: Restore dependency cache
+      uses: actions/cache@v2.1.3
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+            ${{ runner.os }}-maven-
+
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+
+    - name: Configure connections to databases
+      id: configure_db_connections
+      run: cat extensions/database/tests/conf/github_actions_tests.xml | sed -e "s/MYSQL_PORT/${{ job.services.mysql.ports[3306] }}/g" | sed -e "s/POSTGRES_PORT/${{ job.services.postgres.ports[5432] }}/g" > extensions/database/tests/conf/tests.xml
+
+    - name: Populate databases with test data
+      id: populate_databases_with_test_data
+      run: |
+        mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot -e 'CREATE DATABASE test_db;'
+        mysql -u root -h 127.0.0.1 -P ${{ job.services.mysql.ports[3306] }} -proot < extensions/database/tests/conf/test-mysql.sql
+        psql -U postgres test_db -h 127.0.0.1 -p ${{ job.services.postgres.ports[5432] }} < extensions/database/tests/conf/test-pgsql.sql
+      env:
+        PGPASSWORD: postgres
+
+    - name: Build and test with Maven
+      run: mvn jacoco:prepare-agent test
+
+    - name: Submit test coverage to Coveralls
+      run: |
+        mvn prepare-package -DskipTests=true
+        mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}

.github/workflows/snapshot_release.yml

@@ -1,4 +1,4 @@
-name: Java CI
+name: Snapshot release
 
 on:
   push:
@@ -31,8 +31,8 @@ jobs:
     - name: Install genisoimage and jq
       run: sudo apt-get install genisoimage jq
 
-    - name: Build with Maven
+    - name: Build and test with Maven
-      run: ./refine build
+      run: ./refine test
 
     - name: Get the OpenRefine snapshot version
       run: echo ::set-env name=OR_VERSION::$(cat ./main/webapp/WEB-INF/classes/git.properties | jq -r '.["git.commit.id.describe"]')

.travis.yml

@@ -1,85 +0,0 @@
-os: linux
-language: java
-dist: bionic
-
-jobs:
-  include:
-    - dist: trusty # Fastest build first & for all builds
-      jdk: oraclejdk8 # Trusty default
-    # Full matrix only for merges to master or anything to do with release branches e.g. v3.5
-    - if: (branch = master AND type != pull_request) OR branch ~= /^v\d\.\d.*/
-      jdk: openjdk11 # Bionic default
-    - if: branch = master AND type != pull_request OR branch ~= /^v\d\.\d.*/
-      jdk: openjdk12
-      dist: xenial # just for a little variety
-    - if: branch = master AND type != pull_request OR branch ~= /^v\d\.\d.*/
-      jdk: openjdk13
-    - if: branch = master AND type != pull_request OR branch ~= /^v\d\.\d.*/
-      jdk: openjdk14 # replacement for OS X Java 14 build
-    - if: branch = master AND type != pull_request OR branch ~= /^v\d\.\d.*/
-      os: osx
-      osx_image: xcode11.6 # macOS 10.15.4, Oracle JDK 14.0.1
-      language: java
-      services: # not supported on os x
-      addons:
-        homebrew:
-          packages:
-            - mysql
-            - mariadb@10.3
-      before_script:
-        - brew services start mysql
-        - brew services start postgresql
-        - brew services start mariadb@10.3
-        - sleep 15 # wait for databases to start up
-        # Homebrew postgres workaround - create expected user postgres
-        - /usr/local/opt/postgres/bin/createuser -s postgres
-        # FIXME this is duplicated from linux config, but don't know a better way to do it
-        - mysql -u root -e 'CREATE DATABASE test_db;'
-        - mysql -u root test_db < extensions/database/tests/conf/travis-mysql.sql
-        - psql -c 'CREATE DATABASE test_db;' -U postgres
-        - psql -U postgres test_db < extensions/database/tests/conf/travis-pgsql.sql
-        - cp extensions/database/tests/conf/travis_tests.xml extensions/database/tests/conf/tests.xml
-    - if: branch = master AND type != pull_request OR branch ~= /^v\d\.\d.*/
-      jdk: oraclejdk-ea
-    - if: branch = master AND type != pull_request OR branch ~= /^v\d\.\d.*/
-      jdk: openjdk-ea
-  allow_failures:
-    - os: osx
-    - jdk: openjdk-ea
-    - jdk: oraclejdk-ea
-
-addons:
-  mariadb: '10.3'
-  
-services:
-  - mysql
-  - postgresql 
-
-env:
-  # encrypted Codacy key, see https://docs.travis-ci.com/user/encryption-keys/
-  - secure: "VmS4He99YlI6rdmw8Q25OZ9kUp11sRbt0W1QMBvA5lzNSmhN1Q1KtaMj9AGwpCZWcyGWri4AQxEmloARxACxQHXRmNE7ro2DESGw46RAocBAf+RfBxYTifIyUGu5TnSCQhz56SkgpyWpedZAZWyah9ZxgUMfet4KXFUfeiUgYQA="
-  
-before_install:
-  # Fake out packaging for Travis builds before mvn install
-  - cp packaging/travis_pom.xml packaging/pom.xml
-  - mvn process-resources
-
-before_script:
-  # create test databases for mysql, mariadb and postgresql
-  - mysql -u root -e 'CREATE DATABASE test_db;'
-  - mysql -u root test_db < extensions/database/tests/conf/travis-mysql.sql
-  - psql -c 'CREATE DATABASE test_db;' -U postgres
-  - psql -U postgres test_db < extensions/database/tests/conf/travis-pgsql.sql
-  - cp extensions/database/tests/conf/travis_tests.xml extensions/database/tests/conf/tests.xml
-
-script:
-  - mvn jacoco:prepare-agent test
-
-after_success:
-  - mvn prepare-package -DskipTests=true
-  - mvn jacoco:report coveralls:report
-
-cache:
-  directories:
-  - $HOME/.m2
-

appveyor.yml

@@ -26,7 +26,7 @@ before_test:
     PATH=C:\Program Files\PostgreSQL\9.6\bin\;C:\Program Files\MySQL\MySQL Server 5.7\bin\;%PATH%
     SET MYSQL_PWD=Password12!
     mysql -u root --password=Password12! -e "create database test_db;"
-    mysql -u root test_db --password=Password12! < extensions\database\tests\conf\travis-mysql.sql
+    mysql -u root test_db --password=Password12! < extensions\database\tests\conf\test-mysql.sql
     echo "localhost:*:test_db:postgres:Password12!" > C:\Program Files\PostgreSQL\9.6\pgpass.conf
     echo "localhost:*:test_db:postgres:Password12!" > pgpass.conf
     echo "localhost:*:test_db:postgres:Password12!" > %userprofile%\pgpass.conf
@@ -34,10 +34,10 @@ before_test:
     SET PGPASSWORD=Password12!
     SET PGUSER=postgres
     createdb test_db
-    psql -U postgres test_db < extensions\database\tests\conf\travis-pgsql.sql
+    psql -U postgres test_db < extensions\database\tests\conf\test-pgsql.sql
 
     copy extensions\database\tests\conf\appveyor_tests.xml extensions\database\tests\conf\tests.xml
-    copy packaging\travis_pom.xml packaging\pom.xml
+    copy packaging\test_pom.xml packaging\pom.xml
 - cmd: |-
     mvn process-resources
     mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V

extensions/database/tests/conf/appveyor_tests.xml

@@ -18,12 +18,12 @@
     <parameter name = "pgSqlDbPassword" value="Password12!"/>
     <parameter name = "pgSqlTestTable" value="test_table"/>
     
-    <parameter name = "mariaDbName" value="test_db"/> 
+    <parameter name = "mariadbDbName" value="test_db"/> 
-    <parameter name = "mariaDbHost" value="127.0.0.1"/> 
+    <parameter name = "mariadbDbHost" value="127.0.0.1"/> 
     <parameter name = "mariadbDbPort" value="3306"/> 
-    <parameter name = "mariaDbUser" value="root"/>
+    <parameter name = "mariadbDbUser" value="root"/>
-    <parameter name = "mariaDbPassword" value=""/>
+    <parameter name = "mariadbDbPassword" value=""/>
-    <parameter name = "mariaDbTestTable" value="test_table"/>
+    <parameter name = "mariadbTestTable" value="test_table"/>
 
     <parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
     <parameter name = "sqliteDbHost" value=""/>

extensions/database/tests/conf/github_actions_tests.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<suite name="DatabaseExtensionUnitTestSuite">
+    <parameter name = "mySqlDbName" value="test_db"/> 
+    <parameter name = "mySqlDbHost" value="127.0.0.1"/> 
+    <parameter name = "mySqlDbPort" value="MYSQL_PORT"/> 
+    <parameter name = "mySqlDbUser" value="root"/>
+    <parameter name = "mySqlDbPassword" value="root"/>
+    <parameter name = "mySqlTestTable" value="test_table"/>
+    
+    <parameter name = "pgSqlDbName" value="test_db"/> 
+    <parameter name = "pgSqlDbHost" value="127.0.0.11"/> 
+    <parameter name = "pgSqlDbPort" value="POSTGRES_PORT"/> 
+    <parameter name = "pgSqlDbUser" value="postgres"/>
+    <parameter name = "pgSqlDbPassword" value="postgres"/>
+    <parameter name = "pgSqlTestTable" value="test_table"/>
+    
+    <parameter name = "mariadbDbName" value="test_db"/> 
+    <parameter name = "mariadbDbHost" value="127.0.0.1"/> 
+    <parameter name = "mariadbDbPort" value="MYSQL_PORT"/> 
+    <parameter name = "mariadbDbUser" value="root"/>
+    <parameter name = "mariadbDbPassword" value="root"/>
+    <parameter name = "mariadbTestTable" value="test_table"/>
+
+    <parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
+    <parameter name = "sqliteDbHost" value=""/>
+    <parameter name = "sqliteDbPort" value=""/>
+    <parameter name = "sqliteDbUser" value=""/>
+    <parameter name = "sqliteDbPassword" value=""/>
+    <parameter name = "sqliteDbTestTable" value="test_table"/>
+
+    <test name="DatabaseExtensionUnitTest">
+        <groups>
+            <run>
+            </run>
+        </groups> 
+        <classes>
+            <class name="com.google.refine.extension.database.DatabaseImportControllerTest"/>
+            <class name="com.google.refine.extension.database.DatabaseServiceTest"/>
+            <class name="com.google.refine.extension.database.SimpleTextEncryptorTest"/>
+            <class name="com.google.refine.extension.database.cmd.ConnectCommandTest"/>
+            <class name="com.google.refine.extension.database.cmd.ExecuteQueryCommandTest"/>
+            <class name="com.google.refine.extension.database.cmd.SavedConnectionCommandTest"/>
+            <class name="com.google.refine.extension.database.cmd.TestConnectCommandTest"/>
+            <class name="com.google.refine.extension.database.cmd.TestQueryCommandTest"/>
+            <class name="com.google.refine.extension.database.mariadb.MariaDBConnectionManagerTest"/>
+            <class name="com.google.refine.extension.database.mariadb.MariaDBDatabaseServiceTest"/>
+            <class name="com.google.refine.extension.database.mysql.MySQLConnectionManagerTest"/>
+            <class name="com.google.refine.extension.database.mysql.MySQLDatabaseServiceTest"/>
+            <class name="com.google.refine.extension.database.pgsql.PgSQLConnectionManagerTest"/>
+            <class name="com.google.refine.extension.database.pgsql.PgSQLDatabaseServiceTest" />
+            <class name="com.google.refine.extension.database.sqlite.SQLiteConnectionManagerTest"/>
+            <class name="com.google.refine.extension.database.sqlite.SQLiteDatabaseServiceTest" />
+        </classes>
+    </test>
+</suite>

extensions/database/tests/conf/tests.xml

@@ -22,12 +22,12 @@
     <parameter name = "pgSqlDbPassword" value=""/>
     <parameter name = "pgSqlTestTable" value="test_table"/>
     
-    <parameter name = "mariaDbName" value="test_db"/> 
+    <parameter name = "mariadbDbName" value="test_db"/> 
-    <parameter name = "mariaDbHost" value="127.0.0.1"/> 
+    <parameter name = "mariadbDbHost" value="127.0.0.1"/> 
     <parameter name = "mariadbDbPort" value="3306"/> 
-    <parameter name = "mariaDbUser" value="root"/>
+    <parameter name = "mariadbDbUser" value="root"/>
-    <parameter name = "mariaDbPassword" value=""/>
+    <parameter name = "mariadbDbPassword" value=""/>
-    <parameter name = "mariaDbTestTable" value="test_table"/>
+    <parameter name = "mariadbTestTable" value="test_table"/>
 
     <parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
     <parameter name = "sqliteDbHost" value=""/>

extensions/database/tests/conf/travis_tests.xml

@@ -1,57 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<suite name="DatabaseExtensionUnitTestSuite">
-	<parameter name = "mySqlDbName" value="test_db"/> 
-	<parameter name = "mySqlDbHost" value="127.0.0.1"/> 
-	<parameter name = "mySqlDbPort" value="3306"/> 
-	<parameter name = "mySqlDbUser" value="root"/>
-    <parameter name = "mySqlDbPassword" value=""/>
-    <parameter name = "mySqlTestTable" value="test_table"/>
-    
-	<parameter name = "pgSqlDbName" value="test_db"/> 
-	<parameter name = "pgSqlDbHost" value="127.0.0.1"/> 
-	<parameter name = "pgSqlDbPort" value="5432"/> 
-	<parameter name = "pgSqlDbUser" value="postgres"/>
-    <parameter name = "pgSqlDbPassword" value=""/>
-    <parameter name = "pgSqlTestTable" value="test_table"/>
-    
-    <parameter name = "mariaDbName" value="test_db"/> 
-	<parameter name = "mariaDbHost" value="127.0.0.1"/> 
-	<parameter name = "mariadbDbPort" value="3306"/> 
-	<parameter name = "mariaDbUser" value="root"/>
-    <parameter name = "mariaDbPassword" value=""/>
-    <parameter name = "mariaDbTestTable" value="test_table"/>
-
-	<parameter name = "sqliteDbName" value="extension_test_db.sqlite"/>
-	<parameter name = "sqliteDbHost" value=""/>
-	<parameter name = "sqliteDbPort" value=""/>
-	<parameter name = "sqliteDbUser" value=""/>
-	<parameter name = "sqliteDbPassword" value=""/>
-	<parameter name = "sqliteDbTestTable" value="test_table"/>
-
-	<test name="DatabaseExtensionUnitTest">
-        <!-- Temporarily exclude MySQL and MariaDB from Travis tests until MySQL is working on Travis again. --> 
-        <!-- See https://github.com/OpenRefine/OpenRefine/issues/2043 -->
-        <groups>
-            <run>
-            </run>
-        </groups> 
-		<classes>
-			<class name="com.google.refine.extension.database.DatabaseImportControllerTest"/>
-			<class name="com.google.refine.extension.database.DatabaseServiceTest"/>
-			<class name="com.google.refine.extension.database.SimpleTextEncryptorTest"/>
-			<class name="com.google.refine.extension.database.cmd.ConnectCommandTest"/>
-			<class name="com.google.refine.extension.database.cmd.ExecuteQueryCommandTest"/>
-			<class name="com.google.refine.extension.database.cmd.SavedConnectionCommandTest"/>
-			<class name="com.google.refine.extension.database.cmd.TestConnectCommandTest"/>
-			<class name="com.google.refine.extension.database.cmd.TestQueryCommandTest"/>
-			<class name="com.google.refine.extension.database.mariadb.MariaDBConnectionManagerTest"/>
-			<class name="com.google.refine.extension.database.mariadb.MariaDBDatabaseServiceTest"/>
-		 	<class name="com.google.refine.extension.database.mysql.MySQLConnectionManagerTest"/>
-			<class name="com.google.refine.extension.database.mysql.MySQLDatabaseServiceTest"/>
-		    <class name="com.google.refine.extension.database.pgsql.PgSQLConnectionManagerTest"/>
-			<class name="com.google.refine.extension.database.pgsql.PgSQLDatabaseServiceTest" />
-			<class name="com.google.refine.extension.database.sqlite.SQLiteConnectionManagerTest"/>
-			<class name="com.google.refine.extension.database.sqlite.SQLiteDatabaseServiceTest" />
-		</classes>
-	</test>
-</suite>

extensions/database/tests/src/com/google/refine/extension/database/DatabaseTestConfig.java

@@ -21,7 +21,7 @@ public class DatabaseTestConfig extends DBExtensionTests {
     @BeforeSuite
     @Parameters({ "mySqlDbName", "mySqlDbHost", "mySqlDbPort", "mySqlDbUser", "mySqlDbPassword", "mySqlTestTable",
                   "pgSqlDbName", "pgSqlDbHost", "pgSqlDbPort", "pgSqlDbUser", "pgSqlDbPassword", "pgSqlTestTable",
-                  "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbyDbUser", "mariadbDbPassword", "mariadbTestTable",
+                  "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable",
                   "sqliteDbName", "sqliteTestTable"})
     public void beforeSuite(
             @Optional(DEFAULT_MYSQL_DB_NAME)   String mySqlDbName,     @Optional(DEFAULT_MYSQL_HOST)  String mySqlDbHost, 
@@ -33,7 +33,7 @@ public class DatabaseTestConfig extends DBExtensionTests {
             @Optional(DEFAULT_PGSQL_PASSWORD)  String pgSqlDbPassword, @Optional(DEFAULT_TEST_TABLE)  String pgSqlTestTable,
             
             @Optional(DEFAULT_MARIADB_NAME)      String mariadbDbName,     @Optional(DEFAULT_MARIADB_HOST)  String mariadbDbHost, 
-            @Optional(DEFAULT_MARIADB_PORT)      String mariadbDbPort,     @Optional(DEFAULT_MARIADB_USER)  String mariadbyDbUser,
+            @Optional(DEFAULT_MARIADB_PORT)      String mariadbDbPort,     @Optional(DEFAULT_MARIADB_USER)  String mariadbDbUser,
             @Optional(DEFAULT_MARIADB_PASSWORD)  String mariadbDbPassword, @Optional(DEFAULT_TEST_TABLE)    String mariadbTestTable,
 
             @Optional(DEFAULT_SQLITE_DB_NAME) String sqliteDbName, @Optional(DEFAULT_TEST_TABLE) String sqliteTestTable)
@@ -64,7 +64,7 @@ public class DatabaseTestConfig extends DBExtensionTests {
         mariadbDbConfig.setDatabasePassword(mariadbDbPassword);
         mariadbDbConfig.setDatabasePort(Integer.parseInt(mariadbDbPort));
         mariadbDbConfig.setDatabaseType(MariaDBDatabaseService.DB_NAME);
-        mariadbDbConfig.setDatabaseUser(mariadbyDbUser);
+        mariadbDbConfig.setDatabaseUser(mariadbDbUser);
         mariadbDbConfig.setUseSSL(false);
 
         sqliteDbConfig = new DatabaseConfiguration();

extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBConnectionManagerTest.java

@@ -24,7 +24,7 @@ public class MariaDBConnectionManagerTest extends DBExtensionTests {
    
     
     @BeforeTest
-    @Parameters({ "mariaDbName", "mariaDbHost", "mariaDbPort", "mariaDbUser", "mariaDbPassword", "mariaDbTestTable"})
+    @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariaTestTable"})
     public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName,  @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, 
            @Optional(DEFAULT_MARIADB_PORT)    String mariaDbPort,     @Optional(DEFAULT_MARIADB_USER) String mariaDbUser,
            @Optional(DEFAULT_MARIADB_PASSWORD)  String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE)  String mariaDbTestTable) {

extensions/database/tests/src/com/google/refine/extension/database/mariadb/MariaDBDatabaseServiceTest.java

@@ -30,7 +30,7 @@ public class MariaDBDatabaseServiceTest extends DBExtensionTests{
    
     
     @BeforeTest
-    @Parameters({ "mariaDbName", "mariaDbHost", "mariaDbPort", "mariaDbUser", "mariaDbPassword", "mariaDbTestTable"})
+    @Parameters({ "mariadbDbName", "mariadbDbHost", "mariadbDbPort", "mariadbDbUser", "mariadbDbPassword", "mariadbTestTable"})
     public void beforeTest(@Optional(DEFAULT_MARIADB_NAME) String mariaDbName,  @Optional(DEFAULT_MARIADB_HOST) String mariaDbHost, 
            @Optional(DEFAULT_MARIADB_PORT)    String mariaDbPort,     @Optional(DEFAULT_MARIADB_USER) String mariaDbUser,
            @Optional(DEFAULT_MARIADB_PASSWORD)  String mariaDbPassword, @Optional(DEFAULT_TEST_TABLE)  String mariaDbTestTable) {