Run UI tests in continuous integration (#3393)

* Fixed flaky tests

* Refactored ui_test commans-line, added documentation

* Attempt to build a workflow with cypress

* Fixed CI UX tests build

* Changed cyprss actions for pull-request

* Merged Cypress workflow into the regular PR target workflow

* Refactored Github workflows to include Cypress Tests

* Revert Ci build to pull_request_target
This commit is contained in:
Florian Giroud 2020-12-15 20:34:15 +01:00 committed by GitHub
parent 2cf6a359c2
commit 4b6106a386
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 199 additions and 80 deletions

View File

@ -72,3 +72,48 @@ jobs:
run: | run: |
mvn prepare-package -DskipTests=true mvn prepare-package -DskipTests=true
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }} mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
cypress_tests:
strategy:
matrix:
browser: ['chrome']
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.3.4
- name: Restore dependency cache
uses: actions/cache@v2.1.3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java 8
uses: actions/setup-java@v1
with:
java-version: 8
- name: Build OpenRefine
run: ./refine build
- name: Restore Tests dependency cache
uses: actions/cache@v2.1.3
with:
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn
- name: Install test dependencies
run: |
cd ./main/tests/cypress
yarn install
- name: Test with Cypress on ${{ matrix.browser }}
run: |
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
echo REFINE_MEMORY=4096M >> ./refine.ini
./refine ui_test ${{ matrix.browser }} cn3r2t "${{ secrets.CYPRESS_RECORD_KEY }}"

View File

@ -6,6 +6,47 @@ on:
- master - master
jobs: jobs:
cypress_tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.3.4
- name: Restore dependency cache
uses: actions/cache@v2.1.3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java 8
uses: actions/setup-java@v1
with:
java-version: 8
- name: Build OpenRefine
run: ./refine build
- name: Restore Tests dependency cache
uses: actions/cache@v2.1.3
with:
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn
- name: Install test dependencies
run: |
cd ./main/tests/cypress
yarn install
- name: Test with Cypress on chrome
run: |
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
echo REFINE_MEMORY=4096M >> ./refine.ini
./refine ui_test chrome cn3r2t "${{ secrets.CYPRESS_RECORD_KEY }}"
build: build:
services: services:

View File

@ -64,7 +64,7 @@ If you want to run only the server side portion of the tests, use:
If you are running the UI tests for the first time, [you must go through the installation process.](functional-tests) If you are running the UI tests for the first time, [you must go through the installation process.](functional-tests)
If you want to run only the client side portion of the tests, use: If you want to run only the client side portion of the tests, use:
```shell ```shell
yarn --cwd ./main/tests/cypress run cypress open ./refine ui_test chrome
``` ```
## Running ## Running

View File

@ -21,12 +21,13 @@ cd ./main/tests/cypress
yarn install yarn install
``` ```
Cypress always assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it. Cypress tests can be started in two modes:
Once OpenRefine is running, Cypress tests can be started in two modes
### Development / Debugging mode ### Development / Debugging mode
Dev mode assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it.
Run : Run :
```shell ```shell
@ -34,12 +35,15 @@ yarn --cwd ./main/tests/cypress run cypress open
``` ```
It will open the Cypress test runner, where you can choose, replay, visualize tests. It will open the Cypress test runner, where you can choose, replay, visualize tests.
This is the recommended way to run tests when adding or fixing tests This is the recommended way to run tests when adding or fixing tests.
The runners assumes
### Command-line mode ### Command-line mode
Command line mode will starts OpenRefine with a temporary folder for data
```shell ```shell
yarn --cwd ./main/tests/cypress run cypress run ./refine ui_test chrome
``` ```
It will run all tests in the command-line, without windows, displaying results in the standard output It will run all tests in the command-line, without windows, displaying results in the standard output

View File

@ -1,13 +1,6 @@
# OpenRefine test suite # OpenRefine UI test suite
## Install Please refer to the official OpenRefine documentation
``` - [How to build tests and run](https://docs.openrefine.org/technical-reference/build-test-run/)
cd ./main/tests/e2e - [Functional tests](https://docs.openrefine.org/technical-reference/functional-tests)
npm install
```
## Usage
- Run OpenRefine on a separate terminal
- Open the Cypress test runner with `./node_modules/.bin/cypress open`

View File

@ -2,7 +2,7 @@
"integrationFolder": "./cypress/integration", "integrationFolder": "./cypress/integration",
"nodeVersion": "system", "nodeVersion": "system",
"retries": { "retries": {
"runMode": 1, "runMode": 2,
"openMode": 1 "openMode": 1
}, },
"env":{ "env":{

View File

@ -88,7 +88,7 @@ describe(__filename, function () {
cy.get('.dialog-container').should('exist').should('be.visible'); cy.get('.dialog-container').should('exist').should('be.visible');
cy.get('.dialog-container button[bind="closeButton"]').click(); cy.get('.dialog-container button[bind="closeButton"]').click();
cy.get('.dialog-container').should('not.be.visible'); cy.get('.dialog-container').should('not.exist');
}); });
it('Ensure action are recorded in the extract panel', function () { it('Ensure action are recorded in the extract panel', function () {

View File

@ -3,8 +3,8 @@ describe(__filename, function () {
cy.loadAndVisitProject('food.mini.csv'); cy.loadAndVisitProject('food.mini.csv');
cy.deleteColumn('NDB_No'); cy.deleteColumn('NDB_No');
cy.get('#notification-container').should('be.visible').contains('Remove column NDB_No'); cy.get('#notification-container').should('be.visible').should('to.contain', 'Remove column NDB_No');
cy.get('#notification-container .notification-action').should('be.visible').contains('Undo'); cy.get('#notification-container .notification-action').should('be.visible').should('to.contain', 'Undo');
}); });
it('Ensure the Undo button is effectively working', function () { it('Ensure the Undo button is effectively working', function () {
@ -12,7 +12,8 @@ describe(__filename, function () {
cy.deleteColumn('NDB_No'); cy.deleteColumn('NDB_No');
// ensure that the column is back in the grid // ensure that the column is back in the grid
cy.get('#notification-container .notification-action').should('be.visible').contains('Undo').click(); cy.get('#notification-container .notification-action').should('be.visible').should('to.contain', 'Undo');
cy.get('#notification-container a[bind="undoLink"]').click();
cy.get('.data-table th[title="NDB_No"]').should('exist'); cy.get('.data-table th[title="NDB_No"]').should('exist');
}); });
@ -21,39 +22,39 @@ describe(__filename, function () {
// delete NDB_No // delete NDB_No
cy.deleteColumn('NDB_No'); cy.deleteColumn('NDB_No');
cy.get('#or-proj-undoRedo').contains('1 / 1'); cy.get('#or-proj-undoRedo').should('to.contain', '1 / 1');
cy.get('.history-panel-body .history-now').contains('Remove column NDB_No'); cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column NDB_No');
// delete Water // delete Water
cy.deleteColumn('Water'); cy.deleteColumn('Water');
cy.get('#or-proj-undoRedo').contains('2 / 2'); cy.get('#or-proj-undoRedo').should('to.contain', '2 / 2');
cy.get('.history-panel-body .history-now').contains('Remove column Water'); cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Water');
// Delete Shrt_Desc // Delete Shrt_Desc
cy.deleteColumn('Shrt_Desc'); cy.deleteColumn('Shrt_Desc');
cy.get('#or-proj-undoRedo').contains('3 / 3'); cy.get('#or-proj-undoRedo').should('to.contain', '3 / 3');
cy.get('.history-panel-body .history-now').contains('Remove column Shrt_Desc'); cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Shrt_Desc');
// Open the Undo/Redo panel // Open the Undo/Redo panel
cy.get('#or-proj-undoRedo').click(); cy.get('#or-proj-undoRedo').click();
// ensure all previous actions have been recorded // ensure all previous actions have been recorded
cy.get('.history-panel-body .history-past a.history-entry:nth-of-type(2)').contains('Remove column NDB_No'); cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column NDB_No');
cy.get('.history-panel-body .history-past a.history-entry:nth-of-type(3)').contains('Remove column Water'); cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column Water');
cy.get('.history-panel-body .history-now').contains('Remove column Shrt_Desc'); cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Shrt_Desc');
// successively undo all modifications // successively undo all modifications
cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click(); cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click();
cy.waitForOrOperation(); cy.waitForOrOperation();
cy.get('.history-panel-body .history-past').contains('Remove column NDB_No'); cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column NDB_No');
cy.get('.history-panel-body .history-now').contains('Remove column Water'); cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Water');
cy.get('.history-panel-body .history-future').contains('Remove column Shrt_Desc'); cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Shrt_Desc');
cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click(); cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click();
cy.waitForOrOperation(); cy.waitForOrOperation();
cy.get('.history-panel-body .history-now').contains('Remove column NDB_No'); cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column NDB_No');
cy.get('.history-panel-body .history-future').contains('Remove column Water'); cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Water');
cy.get('.history-panel-body .history-future').contains('Remove column Shrt_Desc'); cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Shrt_Desc');
}); });
// Very long test to run // Very long test to run

View File

@ -67,7 +67,7 @@ describe(__filename, function () {
// cypress does not support window.location = ... // cypress does not support window.location = ...
cy.get('h2').contains('HTTP ERROR 404'); cy.get('h2').contains('HTTP ERROR 404');
cy.location().should((location) => { cy.location().should((location) => {
expect(location.href).contains('http://localhost:3333/__/project?'); expect(location.href).contains(Cypress.env('OPENREFINE_URL')+'/__/project?');
}); });
cy.location().then((location) => { cy.location().then((location) => {

View File

@ -41,7 +41,7 @@ Cypress.Commands.add('doCreateProjectThroughUserInterface', () => {
// cypress does not support window.location = ... // cypress does not support window.location = ...
cy.get('h2').contains('HTTP ERROR 404'); cy.get('h2').contains('HTTP ERROR 404');
cy.location().should((location) => { cy.location().should((location) => {
expect(location.href).contains('http://localhost:3333/__/project?'); expect(location.href).contains(Cypress.env('OPENREFINE_URL')+'/__/project?');
}); });
cy.location().then((location) => { cy.location().then((location) => {
@ -67,7 +67,9 @@ Cypress.Commands.add('assertCellEquals', (rowIndex, columnName, value) => {
cy.get(`table.data-table thead th[title="${columnName}"]`).then(($elem) => { cy.get(`table.data-table thead th[title="${columnName}"]`).then(($elem) => {
// there are 3 td at the beginning of each row // there are 3 td at the beginning of each row
const columnIndex = $elem.index() + 3; const columnIndex = $elem.index() + 3;
cy.get(`table.data-table tbody tr:nth-child(${cssRowIndex}) td:nth-child(${columnIndex}) div`).contains(value, { timeout: 5000 }); cy.get(`table.data-table tbody tr:nth-child(${cssRowIndex}) td:nth-child(${columnIndex}) div.data-table-cell-content > span`).should(($cellSpan)=>{
expect($cellSpan.text()).equals(value);
});
}); });
}); });
@ -92,7 +94,7 @@ Cypress.Commands.add('waitForDialogPanel', () => {
Cypress.Commands.add('confirmDialogPanel', () => { Cypress.Commands.add('confirmDialogPanel', () => {
cy.get('body > .dialog-container > .dialog-frame .dialog-footer button[bind="okButton"]').click(); cy.get('body > .dialog-container > .dialog-frame .dialog-footer button[bind="okButton"]').click();
cy.get('body > .dialog-container > .dialog-frame').should('not.be.visible'); cy.get('body > .dialog-container > .dialog-frame').should('not.exist');
}); });
Cypress.Commands.add('columnActionClick', (columnName, actions) => { Cypress.Commands.add('columnActionClick', (columnName, actions) => {

View File

@ -37,7 +37,7 @@ afterEach(() => {
}); });
before(() => { before(() => {
cy.request('http://127.0.0.1:3333/command/core/get-csrf-token').then((response) => { cy.request(Cypress.env('OPENREFINE_URL')+'/command/core/get-csrf-token').then((response) => {
// store one unique token for block of runs // store one unique token for block of runs
token = response.body.token; token = response.body.token;
}); });

View File

@ -1,8 +1,9 @@
Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => { Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => {
cy.request(Cypress.env('OPENREFINE_URL') + '/command/core/get-csrf-token').then((response) => { const openRefineUrl = Cypress.env('OPENREFINE_URL')
cy.request( openRefineUrl + '/command/core/get-csrf-token').then((response) => {
cy.request({ cy.request({
method: 'POST', method: 'POST',
url: `http://127.0.0.1:3333/command/core/set-preference`, url: `${openRefineUrl}/command/core/set-preference`,
body: `name=${preferenceName}&value="${preferenceValue}"&csrf_token=${response.body.token}`, body: `name=${preferenceName}&value="${preferenceValue}"&csrf_token=${response.body.token}`,
form: false, form: false,
headers: { headers: {
@ -15,12 +16,13 @@ Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => {
}); });
Cypress.Commands.add('cleanupProjects', () => { Cypress.Commands.add('cleanupProjects', () => {
const openRefineUrl = Cypress.env('OPENREFINE_URL')
cy.get('@deletetoken', { log: false }).then((token) => { cy.get('@deletetoken', { log: false }).then((token) => {
cy.get('@loadedProjectIds', { log: false }).then((loadedProjectIds) => { cy.get('@loadedProjectIds', { log: false }).then((loadedProjectIds) => {
for (const projectId of loadedProjectIds) { for (const projectId of loadedProjectIds) {
cy.request({ cy.request({
method: 'POST', method: 'POST',
url: `http://127.0.0.1:3333/command/core/delete-project?csrf_token=` + token, url: `${openRefineUrl}/command/core/delete-project?csrf_token=` + token,
body: { project: projectId }, body: { project: projectId },
form: true, form: true,
}).then((resp) => { }).then((resp) => {
@ -32,6 +34,7 @@ Cypress.Commands.add('cleanupProjects', () => {
}); });
Cypress.Commands.add('loadProject', (fixture, projectName) => { Cypress.Commands.add('loadProject', (fixture, projectName) => {
const openRefineUrl = Cypress.env('OPENREFINE_URL');
const openRefineProjectName = projectName ? projectName : fixture; const openRefineProjectName = projectName ? projectName : fixture;
cy.fixture(fixture).then((content) => { cy.fixture(fixture).then((content) => {
cy.get('@token', { log: false }).then((token) => { cy.get('@token', { log: false }).then((token) => {
@ -54,7 +57,7 @@ Cypress.Commands.add('loadProject', (fixture, projectName) => {
cy.request({ cy.request({
method: 'POST', method: 'POST',
url: `http://127.0.0.1:3333/command/core/create-project-from-upload?csrf_token=` + token, url: `${openRefineUrl}/command/core/create-project-from-upload?csrf_token=` + token,
body: postData, body: postData,
headers: { headers: {
'content-type': 'multipart/form-data; boundary=----BOUNDARY', 'content-type': 'multipart/form-data; boundary=----BOUNDARY',

View File

@ -6,7 +6,7 @@
"author": "OpenRefine", "author": "OpenRefine",
"private": true, "private": true,
"dependencies": { "dependencies": {
"cypress":"5.6.0", "cypress": "6.0.1",
"cypress-file-upload": "^4.1.1", "cypress-file-upload": "^4.1.1",
"cypress-wait-until": "^1.7.1", "cypress-wait-until": "^1.7.1",
"dotenv": "^8.2.0", "dotenv": "^8.2.0",

View File

@ -377,10 +377,10 @@ cypress-wait-until@^1.7.1:
resolved "https://registry.yarnpkg.com/cypress-wait-until/-/cypress-wait-until-1.7.1.tgz#3789cd18affdbb848e3cfc1f918353c7ba1de6f8" resolved "https://registry.yarnpkg.com/cypress-wait-until/-/cypress-wait-until-1.7.1.tgz#3789cd18affdbb848e3cfc1f918353c7ba1de6f8"
integrity sha512-8DL5IsBTbAxBjfYgCzdbohPq/bY+IKc63fxtso1C8RWhLnQkZbVESyaclNr76jyxfId6uyzX8+Xnt0ZwaXNtkA== integrity sha512-8DL5IsBTbAxBjfYgCzdbohPq/bY+IKc63fxtso1C8RWhLnQkZbVESyaclNr76jyxfId6uyzX8+Xnt0ZwaXNtkA==
cypress@5.6.0: cypress@6.0.0:
version "5.6.0" version "6.0.0"
resolved "https://registry.yarnpkg.com/cypress/-/cypress-5.6.0.tgz#6781755c3ddfd644ce3179fcd7389176c0c82280" resolved "https://registry.yarnpkg.com/cypress/-/cypress-6.0.0.tgz#57050773c61e8fe1e5c9871cc034c616fcacded9"
integrity sha512-cs5vG3E2JLldAc16+5yQxaVRLLqMVya5RlrfPWkC72S5xrlHFdw7ovxPb61s4wYweROKTyH01WQc2PFzwwVvyQ== integrity sha512-A/w9S15xGxX5UVeAQZacKBqaA0Uqlae9e5WMrehehAdFiLOZj08IgSVZOV8YqA9OH9Z0iBOnmsEkK3NNj43VrA==
dependencies: dependencies:
"@cypress/listr-verbose-renderer" "^0.4.1" "@cypress/listr-verbose-renderer" "^0.4.1"
"@cypress/request" "^2.88.5" "@cypress/request" "^2.88.5"

56
refine
View File

@ -64,7 +64,7 @@ and <action> is one of
test ................................ Run all OpenRefine tests test ................................ Run all OpenRefine tests
server_test ......................... Run only the server tests server_test ......................... Run only the server tests
ui_test ............................. Run only the UI tests ui_test <browser> <id> <key> ........ Run only the UI tests (If passing a project Id and a Record Key, tests will be recorded in Cypress.io Dashboard)
extensions_test ..................... Run only the extensions tests extensions_test ..................... Run only the extensions tests
broker .............................. Run OpenRefine Broker broker .............................. Run OpenRefine Broker
@ -480,15 +480,31 @@ test() {
} }
ui_test() { ui_test() {
INTERACTIVE=$1 get_revision
windmill_prepare BROWSER="$1"
CYPRESS_PROJECT_ID="$2"
CYPRESS_RECORD_KEY="$3"
CYPRESS_RECORD=0
if [ -z "$BROWSER" ] ; then
BROWSER="electron"
fi
if [ ! -z "$CYPRESS_PROJECT_ID" ] && [ ! -z "$CYPRESS_RECORD_KEY" ] ; then
CYPRESS_RECORD=1
echo "Tests will be recorded in Cypress Dashboard"
elif [ ! -z "$CYPRESS_PROJECT_ID" ] && [ -z "$CYPRESS_RECORD_KEY" ] ; then
fail "Found a Cypress project id but no record key"
fi
REFINE_DATA_DIR="${TMPDIR:=/tmp}/openrefine-tests" REFINE_DATA_DIR="${TMPDIR:=/tmp}/openrefine-tests"
add_option "-Drefine.headless=true" add_option "-Drefine.headless=true"
add_option "-Drefine.autoreload=false"
add_option "-Dbutterfly.autoreload=false"
run fork run fork > /dev/null
echo "Waiting for OpenRefine to load..." echo "Waiting for OpenRefine to load..."
sleep 5 sleep 5
@ -499,15 +515,25 @@ ui_test() {
echo "... proceed with the tests." echo "... proceed with the tests."
echo "" echo ""
load_data "$REFINE_TEST_DIR/data/food.csv" "Food" echo "Starting Cypress..."
sleep 3 CYPRESS_RUN_CMD="yarn --cwd ./main/tests/cypress run cypress run --browser $BROWSER --headless --quiet --reporter list --env OPENREFINE_URL=http://$REFINE_HOST:$REFINE_PORT"
echo "" if [ "$CYPRESS_RECORD" = "1" ] ; then
# if tests are recorded, project id is added to env vars, and --record flag is added to the cmd-line
export CYPRESS_PROJECT_ID=$CYPRESS_PROJECT_ID
CYPRESS_RUN_CMD="$CYPRESS_RUN_CMD --record --key $CYPRESS_RECORD_KEY --tag $BROWSER,$REVISION"
fi
export MOZ_FORCE_DISABLE_E10S=1
echo $CYPRESS_RUN_CMD
$CYPRESS_RUN_CMD
echo "Starting Windmill..." if [ "$?" = "0" ] ; then
if [ -z "$INTERACTIVE" ] ; then UI_TEST_SUCCESS="1"
"$WINDMILL" firefox firebug loglevel=WARN http://${REFINE_HOST}:${REFINE_PORT}/ jsdir=$REFINE_TEST_DIR/client/src exit
else else
"$WINDMILL" firefox firebug loglevel=WARN http://${REFINE_HOST}:${REFINE_PORT}/ UI_TEST_SUCCESS="0"
fi
if [ "$CYPRESS_RECORD" = "1" ] ; then
echo "You can review tests on Cypress.io: https://dashboard.cypress.io/projects/$CYPRESS_PROJECT_ID/runs"
fi fi
echo "" echo ""
@ -515,6 +541,10 @@ ui_test() {
/bin/kill -9 $REFINE_PID /bin/kill -9 $REFINE_PID
echo "Cleaning up" echo "Cleaning up"
rm -rf "$REFINE_DATA_DIR" rm -rf "$REFINE_DATA_DIR"
if [ "$UI_TEST_SUCCESS" = "0" ] ; then
error "The UI test suite failed."
fi
} }
server_test() { server_test() {
@ -926,8 +956,8 @@ case "$ACTION" in
distclean) mvn distclean;; distclean) mvn distclean;;
test) test $1;; test) test $1;;
tests) test $1;; tests) test $1;;
ui_test) ui_test $1;; ui_test) ui_test $1 $2 $3;;
ui_tests) ui_test $1;; ui_tests) ui_test $1 $2 $3;;
server_test) server_test $1;; server_test) server_test $1;;
server_tests) server_test $1;; server_tests) server_test $1;;
extensions_test) extensions_test $1;; extensions_test) extensions_test $1;;