Run UI tests in continuous integration (#3393)

* Fixed flaky tests

* Refactored ui_test commans-line, added documentation

* Attempt to build a workflow with cypress

* Fixed CI UX tests build

* Changed cyprss actions for pull-request

* Merged Cypress workflow into the regular PR target workflow

* Refactored Github workflows to include Cypress Tests

* Revert Ci build to pull_request_target

.github/workflows/pull_request.yml

@@ -72,3 +72,48 @@ jobs:
       run: |
         mvn prepare-package -DskipTests=true
         mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
+  cypress_tests:
+    strategy:
+      matrix:
+          browser: ['chrome']
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2.3.4
+
+    - name: Restore dependency cache
+      uses: actions/cache@v2.1.3
+      with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+              ${{ runner.os }}-maven-
+
+    - name: Set up Java 8
+      uses: actions/setup-java@v1
+      with:
+          java-version: 8
+
+    - name: Build OpenRefine
+      run: ./refine build
+
+    - name: Restore Tests dependency cache
+      uses: actions/cache@v2.1.3
+      with:
+          path: '**/node_modules'
+          key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+              ${{ runner.os }}-yarn
+
+    - name: Install test dependencies
+      run: |
+          cd ./main/tests/cypress
+          yarn install
+
+    - name: Test with Cypress on ${{ matrix.browser }}
+      run: | 
+        echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
+        echo REFINE_MEMORY=4096M >> ./refine.ini
+        ./refine ui_test ${{ matrix.browser }} cn3r2t "${{ secrets.CYPRESS_RECORD_KEY }}"
+

.github/workflows/snapshot_release.yml

@@ -6,6 +6,47 @@ on:
       - master
 
 jobs:
+  cypress_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2.3.4
+
+    - name: Restore dependency cache
+      uses: actions/cache@v2.1.3
+      with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+              ${{ runner.os }}-maven-
+
+    - name: Set up Java 8
+      uses: actions/setup-java@v1
+      with:
+          java-version: 8
+
+    - name: Build OpenRefine
+      run: ./refine build
+
+    - name: Restore Tests dependency cache
+      uses: actions/cache@v2.1.3
+      with:
+          path: '**/node_modules'
+          key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+              ${{ runner.os }}-yarn
+
+    - name: Install test dependencies
+      run: |
+          cd ./main/tests/cypress
+          yarn install
+
+    - name: Test with Cypress on chrome
+      run: | 
+        echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
+        echo REFINE_MEMORY=4096M >> ./refine.ini
+        ./refine ui_test chrome cn3r2t "${{ secrets.CYPRESS_RECORD_KEY }}"
+
   build:
 
     services:

docs/docs/technical-reference/build-test-run.md

@@ -64,7 +64,7 @@ If you want to run only the server side portion of the tests, use:
 If you are running the UI tests for the first time, [you must go through the installation process.](functional-tests)  
 If you want to run only the client side portion of the tests, use:
 ```shell
-yarn --cwd ./main/tests/cypress run cypress open
+./refine ui_test chrome
 ```
 
 ## Running

docs/docs/technical-reference/functional-tests.md

@@ -21,12 +21,13 @@ cd ./main/tests/cypress
 yarn install
 ```
 
-Cypress always assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it.
+Cypress tests can be started in two modes:
 
-Once OpenRefine is running, Cypress tests can be started in two modes
 
 ### Development / Debugging mode
 
+Dev mode assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it.
+
 Run :
 
 ```shell
@@ -34,12 +35,15 @@ yarn --cwd ./main/tests/cypress run cypress open
 ```
 
 It will open the Cypress test runner, where you can choose, replay, visualize tests.
-This is the recommended way to run tests when adding or fixing tests
+This is the recommended way to run tests when adding or fixing tests.  
+The runners assumes
 
 ### Command-line mode
 
+Command line mode will starts OpenRefine with a temporary folder for data
+
 ```shell
-yarn --cwd ./main/tests/cypress run cypress run
+./refine ui_test chrome
 ```
 
 It will run all tests in the command-line, without windows, displaying results in the standard output

main/tests/cypress/Readme.md

@@ -1,13 +1,6 @@
-# OpenRefine test suite
+# OpenRefine UI test suite
 
-## Install
+Please refer to the official OpenRefine documentation
 
-```
+-   [How to build tests and run](https://docs.openrefine.org/technical-reference/build-test-run/)
-cd ./main/tests/e2e
+-   [Functional tests](https://docs.openrefine.org/technical-reference/functional-tests)
-npm install
-```
-
-## Usage
-
--   Run OpenRefine on a separate terminal
--   Open the Cypress test runner with `./node_modules/.bin/cypress open`

main/tests/cypress/cypress.json

@@ -2,7 +2,7 @@
 	"integrationFolder": "./cypress/integration",
 	"nodeVersion": "system",
 	"retries": {
-		"runMode": 1,
+		"runMode": 2,
 		"openMode": 1
 	},
 	"env":{

main/tests/cypress/cypress/integration/project/undo_redo/extract.spec.js

@@ -88,7 +88,7 @@ describe(__filename, function () {
 
 		cy.get('.dialog-container').should('exist').should('be.visible');
 		cy.get('.dialog-container button[bind="closeButton"]').click();
-		cy.get('.dialog-container').should('not.be.visible');
+		cy.get('.dialog-container').should('not.exist');
 	});
 
 	it('Ensure action are recorded in the extract panel', function () {

main/tests/cypress/cypress/integration/project/undo_redo/undo_redo.spec.js

@@ -3,8 +3,8 @@ describe(__filename, function () {
 		cy.loadAndVisitProject('food.mini.csv');
 		cy.deleteColumn('NDB_No');
 
-		cy.get('#notification-container').should('be.visible').contains('Remove column NDB_No');
+		cy.get('#notification-container').should('be.visible').should('to.contain', 'Remove column NDB_No');
-		cy.get('#notification-container .notification-action').should('be.visible').contains('Undo');
+		cy.get('#notification-container .notification-action').should('be.visible').should('to.contain', 'Undo');
 	});
 
 	it('Ensure the Undo button is effectively working', function () {
@@ -12,7 +12,8 @@ describe(__filename, function () {
 
 		cy.deleteColumn('NDB_No');
 		// ensure that the column is back in the grid
-		cy.get('#notification-container .notification-action').should('be.visible').contains('Undo').click();
+		cy.get('#notification-container .notification-action').should('be.visible').should('to.contain', 'Undo');
+		cy.get('#notification-container a[bind="undoLink"]').click();
 		cy.get('.data-table th[title="NDB_No"]').should('exist');
 	});
 
@@ -21,39 +22,39 @@ describe(__filename, function () {
 
 		// delete NDB_No
 		cy.deleteColumn('NDB_No');
-		cy.get('#or-proj-undoRedo').contains('1 / 1');
+		cy.get('#or-proj-undoRedo').should('to.contain', '1 / 1');
-		cy.get('.history-panel-body .history-now').contains('Remove column NDB_No');
+		cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column NDB_No');
 
 		// delete Water
 		cy.deleteColumn('Water');
-		cy.get('#or-proj-undoRedo').contains('2 / 2');
+		cy.get('#or-proj-undoRedo').should('to.contain', '2 / 2');
-		cy.get('.history-panel-body .history-now').contains('Remove column Water');
+		cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Water');
 
 		// Delete Shrt_Desc
 		cy.deleteColumn('Shrt_Desc');
-		cy.get('#or-proj-undoRedo').contains('3 / 3');
+		cy.get('#or-proj-undoRedo').should('to.contain', '3 / 3');
-		cy.get('.history-panel-body .history-now').contains('Remove column Shrt_Desc');
+		cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Shrt_Desc');
 
 		// Open the Undo/Redo panel
 		cy.get('#or-proj-undoRedo').click();
 
 		// ensure all previous actions have been recorded
-		cy.get('.history-panel-body .history-past a.history-entry:nth-of-type(2)').contains('Remove column NDB_No');
+		cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column NDB_No');
-		cy.get('.history-panel-body .history-past a.history-entry:nth-of-type(3)').contains('Remove column Water');
+		cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column Water');
-		cy.get('.history-panel-body .history-now').contains('Remove column Shrt_Desc');
+		cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Shrt_Desc');
 
 		// successively undo all modifications
 		cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click();
 		cy.waitForOrOperation();
-		cy.get('.history-panel-body .history-past').contains('Remove column NDB_No');
+		cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column NDB_No');
-		cy.get('.history-panel-body .history-now').contains('Remove column Water');
+		cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Water');
-		cy.get('.history-panel-body .history-future').contains('Remove column Shrt_Desc');
+		cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Shrt_Desc');
 
 		cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click();
 		cy.waitForOrOperation();
-		cy.get('.history-panel-body .history-now').contains('Remove column NDB_No');
+		cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column NDB_No');
-		cy.get('.history-panel-body .history-future').contains('Remove column Water');
+		cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Water');
-		cy.get('.history-panel-body .history-future').contains('Remove column Shrt_Desc');
+		cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Shrt_Desc');
 	});
 
 	// Very long test to run

main/tests/cypress/cypress/integration/project_management/create_project.spec.js

@@ -67,7 +67,7 @@ describe(__filename, function () {
 		// cypress does not support window.location = ...
 		cy.get('h2').contains('HTTP ERROR 404');
 		cy.location().should((location) => {
-			expect(location.href).contains('http://localhost:3333/__/project?');
+			expect(location.href).contains(Cypress.env('OPENREFINE_URL')+'/__/project?');
 		});
 
 		cy.location().then((location) => {

main/tests/cypress/cypress/support/commands.js

@@ -41,7 +41,7 @@ Cypress.Commands.add('doCreateProjectThroughUserInterface', () => {
 	// cypress does not support window.location = ...
 	cy.get('h2').contains('HTTP ERROR 404');
 	cy.location().should((location) => {
-		expect(location.href).contains('http://localhost:3333/__/project?');
+		expect(location.href).contains(Cypress.env('OPENREFINE_URL')+'/__/project?');
 	});
 
 	cy.location().then((location) => {
@@ -67,7 +67,9 @@ Cypress.Commands.add('assertCellEquals', (rowIndex, columnName, value) => {
 	cy.get(`table.data-table thead th[title="${columnName}"]`).then(($elem) => {
 		// there are 3 td at the beginning of each row
 		const columnIndex = $elem.index() + 3;
-		cy.get(`table.data-table tbody tr:nth-child(${cssRowIndex}) td:nth-child(${columnIndex}) div`).contains(value, { timeout: 5000 });
+		cy.get(`table.data-table tbody tr:nth-child(${cssRowIndex}) td:nth-child(${columnIndex}) div.data-table-cell-content > span`).should(($cellSpan)=>{
+			expect($cellSpan.text()).equals(value);
+		});
 	});
 });
 
@@ -92,7 +94,7 @@ Cypress.Commands.add('waitForDialogPanel', () => {
 
 Cypress.Commands.add('confirmDialogPanel', () => {
 	cy.get('body > .dialog-container > .dialog-frame .dialog-footer button[bind="okButton"]').click();
-	cy.get('body > .dialog-container > .dialog-frame').should('not.be.visible');
+	cy.get('body > .dialog-container > .dialog-frame').should('not.exist');
 });
 
 Cypress.Commands.add('columnActionClick', (columnName, actions) => {

main/tests/cypress/cypress/support/index.js

@@ -37,7 +37,7 @@ afterEach(() => {
 });
 
 before(() => {
-	cy.request('http://127.0.0.1:3333/command/core/get-csrf-token').then((response) => {
+	cy.request(Cypress.env('OPENREFINE_URL')+'/command/core/get-csrf-token').then((response) => {
 		// store one unique token for block of runs
 		token = response.body.token;
 	});

main/tests/cypress/cypress/support/openrefine_api.js

@@ -1,8 +1,9 @@
 Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => {
-	cy.request(Cypress.env('OPENREFINE_URL') + '/command/core/get-csrf-token').then((response) => {
+	const openRefineUrl = Cypress.env('OPENREFINE_URL')
+	cy.request( openRefineUrl + '/command/core/get-csrf-token').then((response) => {
 		cy.request({
 			method: 'POST',
-			url: `http://127.0.0.1:3333/command/core/set-preference`,
+			url: `${openRefineUrl}/command/core/set-preference`,
 			body: `name=${preferenceName}&value="${preferenceValue}"&csrf_token=${response.body.token}`,
 			form: false,
 			headers: {
@@ -15,12 +16,13 @@ Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => {
 });
 
 Cypress.Commands.add('cleanupProjects', () => {
+	const openRefineUrl = Cypress.env('OPENREFINE_URL')
 	cy.get('@deletetoken', { log: false }).then((token) => {
 		cy.get('@loadedProjectIds', { log: false }).then((loadedProjectIds) => {
 			for (const projectId of loadedProjectIds) {
 				cy.request({
 					method: 'POST',
-					url: `http://127.0.0.1:3333/command/core/delete-project?csrf_token=` + token,
+					url: `${openRefineUrl}/command/core/delete-project?csrf_token=` + token,
 					body: { project: projectId },
 					form: true,
 				}).then((resp) => {
@@ -32,6 +34,7 @@ Cypress.Commands.add('cleanupProjects', () => {
 });
 
 Cypress.Commands.add('loadProject', (fixture, projectName) => {
+	const openRefineUrl = Cypress.env('OPENREFINE_URL');
 	const openRefineProjectName = projectName ? projectName : fixture;
 	cy.fixture(fixture).then((content) => {
 		cy.get('@token', { log: false }).then((token) => {
@@ -54,7 +57,7 @@ Cypress.Commands.add('loadProject', (fixture, projectName) => {
 
 			cy.request({
 				method: 'POST',
-				url: `http://127.0.0.1:3333/command/core/create-project-from-upload?csrf_token=` + token,
+				url: `${openRefineUrl}/command/core/create-project-from-upload?csrf_token=` + token,
 				body: postData,
 				headers: {
 					'content-type': 'multipart/form-data; boundary=----BOUNDARY',

main/tests/cypress/package.json

@@ -1,16 +1,16 @@
 {
-   "name":"OpenRefine-Cypress-Test-Suite",
+    "name": "OpenRefine-Cypress-Test-Suite",
-   "version":"1.0.0",
+    "version": "1.0.0",
-   "description":"Cypress tests for OpenRefine",
+    "description": "Cypress tests for OpenRefine",
-   "license":"BSD-3-Clause",
+    "license": "BSD-3-Clause",
-   "author":"OpenRefine",
+    "author": "OpenRefine",
-   "private":true,
+    "private": true,
-   "dependencies":{
+    "dependencies": {
-      "cypress":"5.6.0",
+        "cypress": "6.0.1",
-      "cypress-file-upload":"^4.1.1",
+        "cypress-file-upload": "^4.1.1",
-      "cypress-wait-until":"^1.7.1",
+        "cypress-wait-until": "^1.7.1",
-      "dotenv":"^8.2.0",
+        "dotenv": "^8.2.0",
-      "fs-extra":"^9.0.1",
+        "fs-extra": "^9.0.1",
-      "uniqid":"^5.2.0"
+        "uniqid": "^5.2.0"
     }
 }

main/tests/cypress/yarn.lock

@@ -377,10 +377,10 @@ cypress-wait-until@^1.7.1:
   resolved "https://registry.yarnpkg.com/cypress-wait-until/-/cypress-wait-until-1.7.1.tgz#3789cd18affdbb848e3cfc1f918353c7ba1de6f8"
   integrity sha512-8DL5IsBTbAxBjfYgCzdbohPq/bY+IKc63fxtso1C8RWhLnQkZbVESyaclNr76jyxfId6uyzX8+Xnt0ZwaXNtkA==
 
-cypress@5.6.0:
+cypress@6.0.0:
-  version "5.6.0"
+  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/cypress/-/cypress-5.6.0.tgz#6781755c3ddfd644ce3179fcd7389176c0c82280"
+  resolved "https://registry.yarnpkg.com/cypress/-/cypress-6.0.0.tgz#57050773c61e8fe1e5c9871cc034c616fcacded9"
-  integrity sha512-cs5vG3E2JLldAc16+5yQxaVRLLqMVya5RlrfPWkC72S5xrlHFdw7ovxPb61s4wYweROKTyH01WQc2PFzwwVvyQ==
+  integrity sha512-A/w9S15xGxX5UVeAQZacKBqaA0Uqlae9e5WMrehehAdFiLOZj08IgSVZOV8YqA9OH9Z0iBOnmsEkK3NNj43VrA==
   dependencies:
     "@cypress/listr-verbose-renderer" "^0.4.1"
     "@cypress/request" "^2.88.5"

refine

@@ -64,7 +64,7 @@ and <action> is one of
 
    test ................................ Run all OpenRefine tests
    server_test ......................... Run only the server tests
-   ui_test ............................. Run only the UI tests
+   ui_test <browser> <id> <key> ........ Run only the UI tests (If passing a project Id and a Record Key, tests will be recorded in Cypress.io Dashboard)
    extensions_test ..................... Run only the extensions tests
 
    broker .............................. Run OpenRefine Broker
@@ -480,15 +480,31 @@ test() {
 }
 
 ui_test() {
-    INTERACTIVE=$1
+    get_revision
 
-    windmill_prepare
+    BROWSER="$1"
+    CYPRESS_PROJECT_ID="$2"
+    CYPRESS_RECORD_KEY="$3"
+    CYPRESS_RECORD=0
+
+    if [ -z "$BROWSER" ] ; then
+        BROWSER="electron"
+    fi
+    
+    if [ ! -z "$CYPRESS_PROJECT_ID" ] && [ ! -z "$CYPRESS_RECORD_KEY" ] ; then
+        CYPRESS_RECORD=1
+        echo "Tests will be recorded in Cypress Dashboard"
+    elif [ ! -z "$CYPRESS_PROJECT_ID" ] && [ -z "$CYPRESS_RECORD_KEY" ] ; then
+        fail "Found a Cypress project id but no record key"
+    fi
 
     REFINE_DATA_DIR="${TMPDIR:=/tmp}/openrefine-tests"
     
     add_option "-Drefine.headless=true"
+    add_option "-Drefine.autoreload=false"
+    add_option "-Dbutterfly.autoreload=false"
  
-    run fork
+    run fork > /dev/null
     
     echo "Waiting for OpenRefine to load..."
     sleep 5
@@ -499,15 +515,25 @@ ui_test() {
     echo "... proceed with the tests."
     echo ""
 
-    load_data "$REFINE_TEST_DIR/data/food.csv" "Food"
+    echo "Starting Cypress..."
-    sleep 3
+    CYPRESS_RUN_CMD="yarn --cwd ./main/tests/cypress run cypress run --browser $BROWSER --headless --quiet --reporter list --env OPENREFINE_URL=http://$REFINE_HOST:$REFINE_PORT"
-    echo ""
+    if [ "$CYPRESS_RECORD" = "1" ] ; then
+        # if tests are recorded, project id is added to env vars, and --record flag is added to the cmd-line
+        export CYPRESS_PROJECT_ID=$CYPRESS_PROJECT_ID
+        CYPRESS_RUN_CMD="$CYPRESS_RUN_CMD --record --key $CYPRESS_RECORD_KEY --tag $BROWSER,$REVISION"
+    fi
+    export MOZ_FORCE_DISABLE_E10S=1
+    echo $CYPRESS_RUN_CMD
+    $CYPRESS_RUN_CMD
 
-    echo "Starting Windmill..."
+    if [ "$?" = "0" ] ; then
-    if [ -z "$INTERACTIVE" ] ; then
+        UI_TEST_SUCCESS="1"
-        "$WINDMILL" firefox firebug loglevel=WARN http://${REFINE_HOST}:${REFINE_PORT}/ jsdir=$REFINE_TEST_DIR/client/src exit
     else
-        "$WINDMILL" firefox firebug loglevel=WARN http://${REFINE_HOST}:${REFINE_PORT}/
+        UI_TEST_SUCCESS="0"
+    fi
+
+    if [ "$CYPRESS_RECORD" = "1" ] ; then
+        echo "You can review tests on Cypress.io: https://dashboard.cypress.io/projects/$CYPRESS_PROJECT_ID/runs"
     fi
     
     echo ""
@@ -515,6 +541,10 @@ ui_test() {
     /bin/kill -9 $REFINE_PID
     echo "Cleaning up"
     rm -rf "$REFINE_DATA_DIR"
+
+    if [ "$UI_TEST_SUCCESS" = "0" ] ; then
+        error "The UI test suite failed."
+    fi
 }
 
 server_test() {
@@ -926,8 +956,8 @@ case "$ACTION" in
   distclean) mvn distclean;;
   test) test $1;;  
   tests) test $1;;  
-  ui_test) ui_test $1;;  
+  ui_test) ui_test $1 $2 $3;;  
-  ui_tests) ui_test $1;;  
+  ui_tests) ui_test $1 $2 $3;;  
   server_test) server_test $1;;  
   server_tests) server_test $1;;  
   extensions_test) extensions_test $1;;