Run UI tests in continuous integration (#3393)

* Fixed flaky tests

* Refactored ui_test commans-line, added documentation

* Attempt to build a workflow with cypress

* Fixed CI UX tests build

* Changed cyprss actions for pull-request

* Merged Cypress workflow into the regular PR target workflow

* Refactored Github workflows to include Cypress Tests

* Revert Ci build to pull_request_target
This commit is contained in:
Florian Giroud 2020-12-15 20:34:15 +01:00 committed by GitHub
parent 2cf6a359c2
commit 4b6106a386
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 199 additions and 80 deletions

View File

@ -72,3 +72,48 @@ jobs:
run: |
mvn prepare-package -DskipTests=true
mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
cypress_tests:
strategy:
matrix:
browser: ['chrome']
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.3.4
- name: Restore dependency cache
uses: actions/cache@v2.1.3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java 8
uses: actions/setup-java@v1
with:
java-version: 8
- name: Build OpenRefine
run: ./refine build
- name: Restore Tests dependency cache
uses: actions/cache@v2.1.3
with:
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn
- name: Install test dependencies
run: |
cd ./main/tests/cypress
yarn install
- name: Test with Cypress on ${{ matrix.browser }}
run: |
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
echo REFINE_MEMORY=4096M >> ./refine.ini
./refine ui_test ${{ matrix.browser }} cn3r2t "${{ secrets.CYPRESS_RECORD_KEY }}"

View File

@ -6,6 +6,47 @@ on:
- master
jobs:
cypress_tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2.3.4
- name: Restore dependency cache
uses: actions/cache@v2.1.3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up Java 8
uses: actions/setup-java@v1
with:
java-version: 8
- name: Build OpenRefine
run: ./refine build
- name: Restore Tests dependency cache
uses: actions/cache@v2.1.3
with:
path: '**/node_modules'
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn
- name: Install test dependencies
run: |
cd ./main/tests/cypress
yarn install
- name: Test with Cypress on chrome
run: |
echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
echo REFINE_MEMORY=4096M >> ./refine.ini
./refine ui_test chrome cn3r2t "${{ secrets.CYPRESS_RECORD_KEY }}"
build:
services:

View File

@ -64,7 +64,7 @@ If you want to run only the server side portion of the tests, use:
If you are running the UI tests for the first time, [you must go through the installation process.](functional-tests)
If you want to run only the client side portion of the tests, use:
```shell
yarn --cwd ./main/tests/cypress run cypress open
./refine ui_test chrome
```
## Running

View File

@ -21,12 +21,13 @@ cd ./main/tests/cypress
yarn install
```
Cypress always assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it.
Cypress tests can be started in two modes:
Once OpenRefine is running, Cypress tests can be started in two modes
### Development / Debugging mode
Dev mode assumes that OpenRefine is up and running on the local machine, the tests themselves do not launch OpenRefine, nor restarts it.
Run :
```shell
@ -34,12 +35,15 @@ yarn --cwd ./main/tests/cypress run cypress open
```
It will open the Cypress test runner, where you can choose, replay, visualize tests.
This is the recommended way to run tests when adding or fixing tests
This is the recommended way to run tests when adding or fixing tests.
The runners assumes
### Command-line mode
Command line mode will starts OpenRefine with a temporary folder for data
```shell
yarn --cwd ./main/tests/cypress run cypress run
./refine ui_test chrome
```
It will run all tests in the command-line, without windows, displaying results in the standard output

View File

@ -1,13 +1,6 @@
# OpenRefine test suite
# OpenRefine UI test suite
## Install
Please refer to the official OpenRefine documentation
```
cd ./main/tests/e2e
npm install
```
## Usage
- Run OpenRefine on a separate terminal
- Open the Cypress test runner with `./node_modules/.bin/cypress open`
- [How to build tests and run](https://docs.openrefine.org/technical-reference/build-test-run/)
- [Functional tests](https://docs.openrefine.org/technical-reference/functional-tests)

View File

@ -2,7 +2,7 @@
"integrationFolder": "./cypress/integration",
"nodeVersion": "system",
"retries": {
"runMode": 1,
"runMode": 2,
"openMode": 1
},
"env":{

View File

@ -88,7 +88,7 @@ describe(__filename, function () {
cy.get('.dialog-container').should('exist').should('be.visible');
cy.get('.dialog-container button[bind="closeButton"]').click();
cy.get('.dialog-container').should('not.be.visible');
cy.get('.dialog-container').should('not.exist');
});
it('Ensure action are recorded in the extract panel', function () {

View File

@ -3,8 +3,8 @@ describe(__filename, function () {
cy.loadAndVisitProject('food.mini.csv');
cy.deleteColumn('NDB_No');
cy.get('#notification-container').should('be.visible').contains('Remove column NDB_No');
cy.get('#notification-container .notification-action').should('be.visible').contains('Undo');
cy.get('#notification-container').should('be.visible').should('to.contain', 'Remove column NDB_No');
cy.get('#notification-container .notification-action').should('be.visible').should('to.contain', 'Undo');
});
it('Ensure the Undo button is effectively working', function () {
@ -12,7 +12,8 @@ describe(__filename, function () {
cy.deleteColumn('NDB_No');
// ensure that the column is back in the grid
cy.get('#notification-container .notification-action').should('be.visible').contains('Undo').click();
cy.get('#notification-container .notification-action').should('be.visible').should('to.contain', 'Undo');
cy.get('#notification-container a[bind="undoLink"]').click();
cy.get('.data-table th[title="NDB_No"]').should('exist');
});
@ -21,39 +22,39 @@ describe(__filename, function () {
// delete NDB_No
cy.deleteColumn('NDB_No');
cy.get('#or-proj-undoRedo').contains('1 / 1');
cy.get('.history-panel-body .history-now').contains('Remove column NDB_No');
cy.get('#or-proj-undoRedo').should('to.contain', '1 / 1');
cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column NDB_No');
// delete Water
cy.deleteColumn('Water');
cy.get('#or-proj-undoRedo').contains('2 / 2');
cy.get('.history-panel-body .history-now').contains('Remove column Water');
cy.get('#or-proj-undoRedo').should('to.contain', '2 / 2');
cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Water');
// Delete Shrt_Desc
cy.deleteColumn('Shrt_Desc');
cy.get('#or-proj-undoRedo').contains('3 / 3');
cy.get('.history-panel-body .history-now').contains('Remove column Shrt_Desc');
cy.get('#or-proj-undoRedo').should('to.contain', '3 / 3');
cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Shrt_Desc');
// Open the Undo/Redo panel
cy.get('#or-proj-undoRedo').click();
// ensure all previous actions have been recorded
cy.get('.history-panel-body .history-past a.history-entry:nth-of-type(2)').contains('Remove column NDB_No');
cy.get('.history-panel-body .history-past a.history-entry:nth-of-type(3)').contains('Remove column Water');
cy.get('.history-panel-body .history-now').contains('Remove column Shrt_Desc');
cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column NDB_No');
cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column Water');
cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Shrt_Desc');
// successively undo all modifications
cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click();
cy.waitForOrOperation();
cy.get('.history-panel-body .history-past').contains('Remove column NDB_No');
cy.get('.history-panel-body .history-now').contains('Remove column Water');
cy.get('.history-panel-body .history-future').contains('Remove column Shrt_Desc');
cy.get('.history-panel-body .history-past').should('to.contain', 'Remove column NDB_No');
cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column Water');
cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Shrt_Desc');
cy.get('.history-panel-body .history-past a.history-entry:last-of-type').click();
cy.waitForOrOperation();
cy.get('.history-panel-body .history-now').contains('Remove column NDB_No');
cy.get('.history-panel-body .history-future').contains('Remove column Water');
cy.get('.history-panel-body .history-future').contains('Remove column Shrt_Desc');
cy.get('.history-panel-body .history-now').should('to.contain', 'Remove column NDB_No');
cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Water');
cy.get('.history-panel-body .history-future').should('to.contain', 'Remove column Shrt_Desc');
});
// Very long test to run

View File

@ -67,7 +67,7 @@ describe(__filename, function () {
// cypress does not support window.location = ...
cy.get('h2').contains('HTTP ERROR 404');
cy.location().should((location) => {
expect(location.href).contains('http://localhost:3333/__/project?');
expect(location.href).contains(Cypress.env('OPENREFINE_URL')+'/__/project?');
});
cy.location().then((location) => {

View File

@ -41,7 +41,7 @@ Cypress.Commands.add('doCreateProjectThroughUserInterface', () => {
// cypress does not support window.location = ...
cy.get('h2').contains('HTTP ERROR 404');
cy.location().should((location) => {
expect(location.href).contains('http://localhost:3333/__/project?');
expect(location.href).contains(Cypress.env('OPENREFINE_URL')+'/__/project?');
});
cy.location().then((location) => {
@ -67,7 +67,9 @@ Cypress.Commands.add('assertCellEquals', (rowIndex, columnName, value) => {
cy.get(`table.data-table thead th[title="${columnName}"]`).then(($elem) => {
// there are 3 td at the beginning of each row
const columnIndex = $elem.index() + 3;
cy.get(`table.data-table tbody tr:nth-child(${cssRowIndex}) td:nth-child(${columnIndex}) div`).contains(value, { timeout: 5000 });
cy.get(`table.data-table tbody tr:nth-child(${cssRowIndex}) td:nth-child(${columnIndex}) div.data-table-cell-content > span`).should(($cellSpan)=>{
expect($cellSpan.text()).equals(value);
});
});
});
@ -92,7 +94,7 @@ Cypress.Commands.add('waitForDialogPanel', () => {
Cypress.Commands.add('confirmDialogPanel', () => {
cy.get('body > .dialog-container > .dialog-frame .dialog-footer button[bind="okButton"]').click();
cy.get('body > .dialog-container > .dialog-frame').should('not.be.visible');
cy.get('body > .dialog-container > .dialog-frame').should('not.exist');
});
Cypress.Commands.add('columnActionClick', (columnName, actions) => {

View File

@ -37,7 +37,7 @@ afterEach(() => {
});
before(() => {
cy.request('http://127.0.0.1:3333/command/core/get-csrf-token').then((response) => {
cy.request(Cypress.env('OPENREFINE_URL')+'/command/core/get-csrf-token').then((response) => {
// store one unique token for block of runs
token = response.body.token;
});

View File

@ -1,8 +1,9 @@
Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => {
cy.request(Cypress.env('OPENREFINE_URL') + '/command/core/get-csrf-token').then((response) => {
const openRefineUrl = Cypress.env('OPENREFINE_URL')
cy.request( openRefineUrl + '/command/core/get-csrf-token').then((response) => {
cy.request({
method: 'POST',
url: `http://127.0.0.1:3333/command/core/set-preference`,
url: `${openRefineUrl}/command/core/set-preference`,
body: `name=${preferenceName}&value="${preferenceValue}"&csrf_token=${response.body.token}`,
form: false,
headers: {
@ -15,12 +16,13 @@ Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => {
});
Cypress.Commands.add('cleanupProjects', () => {
const openRefineUrl = Cypress.env('OPENREFINE_URL')
cy.get('@deletetoken', { log: false }).then((token) => {
cy.get('@loadedProjectIds', { log: false }).then((loadedProjectIds) => {
for (const projectId of loadedProjectIds) {
cy.request({
method: 'POST',
url: `http://127.0.0.1:3333/command/core/delete-project?csrf_token=` + token,
url: `${openRefineUrl}/command/core/delete-project?csrf_token=` + token,
body: { project: projectId },
form: true,
}).then((resp) => {
@ -32,6 +34,7 @@ Cypress.Commands.add('cleanupProjects', () => {
});
Cypress.Commands.add('loadProject', (fixture, projectName) => {
const openRefineUrl = Cypress.env('OPENREFINE_URL');
const openRefineProjectName = projectName ? projectName : fixture;
cy.fixture(fixture).then((content) => {
cy.get('@token', { log: false }).then((token) => {
@ -54,7 +57,7 @@ Cypress.Commands.add('loadProject', (fixture, projectName) => {
cy.request({
method: 'POST',
url: `http://127.0.0.1:3333/command/core/create-project-from-upload?csrf_token=` + token,
url: `${openRefineUrl}/command/core/create-project-from-upload?csrf_token=` + token,
body: postData,
headers: {
'content-type': 'multipart/form-data; boundary=----BOUNDARY',

View File

@ -1,16 +1,16 @@
{
"name":"OpenRefine-Cypress-Test-Suite",
"version":"1.0.0",
"description":"Cypress tests for OpenRefine",
"license":"BSD-3-Clause",
"author":"OpenRefine",
"private":true,
"dependencies":{
"cypress":"5.6.0",
"cypress-file-upload":"^4.1.1",
"cypress-wait-until":"^1.7.1",
"dotenv":"^8.2.0",
"fs-extra":"^9.0.1",
"uniqid":"^5.2.0"
}
}
"name": "OpenRefine-Cypress-Test-Suite",
"version": "1.0.0",
"description": "Cypress tests for OpenRefine",
"license": "BSD-3-Clause",
"author": "OpenRefine",
"private": true,
"dependencies": {
"cypress": "6.0.1",
"cypress-file-upload": "^4.1.1",
"cypress-wait-until": "^1.7.1",
"dotenv": "^8.2.0",
"fs-extra": "^9.0.1",
"uniqid": "^5.2.0"
}
}

View File

@ -377,10 +377,10 @@ cypress-wait-until@^1.7.1:
resolved "https://registry.yarnpkg.com/cypress-wait-until/-/cypress-wait-until-1.7.1.tgz#3789cd18affdbb848e3cfc1f918353c7ba1de6f8"
integrity sha512-8DL5IsBTbAxBjfYgCzdbohPq/bY+IKc63fxtso1C8RWhLnQkZbVESyaclNr76jyxfId6uyzX8+Xnt0ZwaXNtkA==
cypress@5.6.0:
version "5.6.0"
resolved "https://registry.yarnpkg.com/cypress/-/cypress-5.6.0.tgz#6781755c3ddfd644ce3179fcd7389176c0c82280"
integrity sha512-cs5vG3E2JLldAc16+5yQxaVRLLqMVya5RlrfPWkC72S5xrlHFdw7ovxPb61s4wYweROKTyH01WQc2PFzwwVvyQ==
cypress@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/cypress/-/cypress-6.0.0.tgz#57050773c61e8fe1e5c9871cc034c616fcacded9"
integrity sha512-A/w9S15xGxX5UVeAQZacKBqaA0Uqlae9e5WMrehehAdFiLOZj08IgSVZOV8YqA9OH9Z0iBOnmsEkK3NNj43VrA==
dependencies:
"@cypress/listr-verbose-renderer" "^0.4.1"
"@cypress/request" "^2.88.5"

62
refine
View File

@ -64,7 +64,7 @@ and <action> is one of
test ................................ Run all OpenRefine tests
server_test ......................... Run only the server tests
ui_test ............................. Run only the UI tests
ui_test <browser> <id> <key> ........ Run only the UI tests (If passing a project Id and a Record Key, tests will be recorded in Cypress.io Dashboard)
extensions_test ..................... Run only the extensions tests
broker .............................. Run OpenRefine Broker
@ -480,15 +480,31 @@ test() {
}
ui_test() {
INTERACTIVE=$1
get_revision
BROWSER="$1"
CYPRESS_PROJECT_ID="$2"
CYPRESS_RECORD_KEY="$3"
CYPRESS_RECORD=0
if [ -z "$BROWSER" ] ; then
BROWSER="electron"
fi
windmill_prepare
if [ ! -z "$CYPRESS_PROJECT_ID" ] && [ ! -z "$CYPRESS_RECORD_KEY" ] ; then
CYPRESS_RECORD=1
echo "Tests will be recorded in Cypress Dashboard"
elif [ ! -z "$CYPRESS_PROJECT_ID" ] && [ -z "$CYPRESS_RECORD_KEY" ] ; then
fail "Found a Cypress project id but no record key"
fi
REFINE_DATA_DIR="${TMPDIR:=/tmp}/openrefine-tests"
add_option "-Drefine.headless=true"
run fork
add_option "-Drefine.autoreload=false"
add_option "-Dbutterfly.autoreload=false"
run fork > /dev/null
echo "Waiting for OpenRefine to load..."
sleep 5
@ -498,16 +514,26 @@ ui_test() {
fi
echo "... proceed with the tests."
echo ""
load_data "$REFINE_TEST_DIR/data/food.csv" "Food"
sleep 3
echo ""
echo "Starting Windmill..."
if [ -z "$INTERACTIVE" ] ; then
"$WINDMILL" firefox firebug loglevel=WARN http://${REFINE_HOST}:${REFINE_PORT}/ jsdir=$REFINE_TEST_DIR/client/src exit
echo "Starting Cypress..."
CYPRESS_RUN_CMD="yarn --cwd ./main/tests/cypress run cypress run --browser $BROWSER --headless --quiet --reporter list --env OPENREFINE_URL=http://$REFINE_HOST:$REFINE_PORT"
if [ "$CYPRESS_RECORD" = "1" ] ; then
# if tests are recorded, project id is added to env vars, and --record flag is added to the cmd-line
export CYPRESS_PROJECT_ID=$CYPRESS_PROJECT_ID
CYPRESS_RUN_CMD="$CYPRESS_RUN_CMD --record --key $CYPRESS_RECORD_KEY --tag $BROWSER,$REVISION"
fi
export MOZ_FORCE_DISABLE_E10S=1
echo $CYPRESS_RUN_CMD
$CYPRESS_RUN_CMD
if [ "$?" = "0" ] ; then
UI_TEST_SUCCESS="1"
else
"$WINDMILL" firefox firebug loglevel=WARN http://${REFINE_HOST}:${REFINE_PORT}/
UI_TEST_SUCCESS="0"
fi
if [ "$CYPRESS_RECORD" = "1" ] ; then
echo "You can review tests on Cypress.io: https://dashboard.cypress.io/projects/$CYPRESS_PROJECT_ID/runs"
fi
echo ""
@ -515,6 +541,10 @@ ui_test() {
/bin/kill -9 $REFINE_PID
echo "Cleaning up"
rm -rf "$REFINE_DATA_DIR"
if [ "$UI_TEST_SUCCESS" = "0" ] ; then
error "The UI test suite failed."
fi
}
server_test() {
@ -926,8 +956,8 @@ case "$ACTION" in
distclean) mvn distclean;;
test) test $1;;
tests) test $1;;
ui_test) ui_test $1;;
ui_tests) ui_test $1;;
ui_test) ui_test $1 $2 $3;;
ui_tests) ui_test $1 $2 $3;;
server_test) server_test $1;;
server_tests) server_test $1;;
extensions_test) extensions_test $1;;