Cypress parallel manual (#3922)

Implemented parallelism fo Cypress Tests

.github/workflows/pull_request.yml

@@ -82,43 +82,69 @@ jobs:
         mvn prepare-package -DskipTests=true
         mvn jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} -DpullRequest=${{ github.event.number }}
 
-  ui_tests:
-    strategy:
-      matrix:
-          browser: ['chrome']
+  prepare_ui_test_matrix:
     runs-on: ubuntu-latest
-
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v2.3.4
-      with:
-        ref: ${{ github.event.pull_request.head.ref }}
-        repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - uses: actions/checkout@v2.3.4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+      - id: set-matrix
+        run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
+        env:
+          browsers: chrome
+  ui_test:
+    needs: prepare_ui_test_matrix
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
 
-    - name: Restore dependency cache
-      uses: actions/cache@v2.1.5
-      with:
+      - name: Restore dependency cache
+        uses: actions/cache@v2.1.4
+        with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-              ${{ runner.os }}-maven-
+            ${{ runner.os }}-maven-
 
-    - name: Set up Java 8
-      uses: actions/setup-java@v2
-      with:
+      - name: Set up Java 8
+        uses: actions/setup-java@v2
+        with:
           distribution: 'adopt'
           java-version: 8
 
-    - name: Build OpenRefine
-      run: ./refine build
+      - name: Build OpenRefine
+        run: ./refine build
 
-    - name: Setup Node
-      uses: actions/setup-node@v2
-      with:
-        node-version: '12'
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
 
-    - name: Restore Tests dependency cache
-      uses: actions/cache@v2.1.5
-      with:
+      - name: Install Edge
+        if: matrix.browser == 'edge'
+        run: |
+          sudo curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
+          sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/
+          sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
+          sudo rm microsoft.gpg
+          sudo apt-get update
+          sudo apt-get install microsoft-edge-dev
+
+      - name: Restore Tests dependency cache
+        uses: actions/cache@v2.1.4
+        with:
           path: |
             ~/cache
             ~/.cache
@@ -126,22 +152,22 @@ jobs:
             !~/cache/exclude
           key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
-              ${{ runner.os }}-yarn
+            ${{ runner.os }}-yarn
 
-    - name: Install test dependencies
-      run: |
+      - name: Install test dependencies
+        run: |
           cd ./main/tests/cypress
           npm i -g yarn
           yarn install
 
-    - name: Lint with ES-Lint & Prettier
-      run: |
-          cd ./main/tests/cypress
-          yarn lint
-
-    - name: Test with Cypress on ${{ matrix.browser }}
-      run: | 
-        echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
-        echo REFINE_MEMORY=4096M >> ./refine.ini
-        ./refine ui_test ${{ matrix.browser }} s5du3k "${{ secrets.CYPRESS_RECORD_KEY }}"
-
+      - name: Test with Cypress on ${{ matrix.browser }}
+        run: | 
+          echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
+          echo REFINE_MEMORY=4096M >> ./refine.ini
+          ./refine ui_tests
+        env:
+          CYPRESS_BROWSER: ${{ matrix.browser }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+          CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
+          CYPRESS_SPECS: ${{ matrix.specs }}

.github/workflows/snapshot_release.yml

@@ -8,51 +8,69 @@ on:
       - 'docs/**'
 
 jobs:
-  ui_tests:
+  prepare_ui_test_matrix:
     runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-          browser: ['edge', 'chrome']
-
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2.3.4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+      - id: set-matrix
+        run: npm install --save glob && node main/tests/cypress/build-test-matrix.js
+        env:
+          browsers: chrome,edge
+  ui_test:
+    needs: prepare_ui_test_matrix
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJSON(needs.prepare_ui_test_matrix.outputs.matrix)}}
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
 
-    - name: Restore dependency cache
-      uses: actions/cache@v2.1.5
-      with:
+      - name: Restore dependency cache
+        uses: actions/cache@v2.1.4
+        with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-              ${{ runner.os }}-maven-
+            ${{ runner.os }}-maven-
 
-    - name: Set up Java 8
-      uses: actions/setup-java@v2
-      with:
+      - name: Set up Java 8
+        uses: actions/setup-java@v2
+        with:
           distribution: 'adopt'
           java-version: 8
 
-    - name: Setup Node
-      uses: actions/setup-node@v2
-      with:
-        node-version: '12'
+      - name: Build OpenRefine
+        run: ./refine build
 
-    - name: Install Edge
-      if: matrix.browser == 'edge'
-      run: | 
-        sudo curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
-        sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/
-        sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
-        sudo rm microsoft.gpg
-        sudo apt-get update
-        sudo apt-get install microsoft-edge-dev
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '12'
 
-    - name: Build OpenRefine
-      run: ./refine build
+      - name: Install Edge
+        if: matrix.browser == 'edge'
+        run: |
+          sudo curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
+          sudo install -o root -g root -m 644 microsoft.gpg /etc/apt/trusted.gpg.d/
+          sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
+          sudo rm microsoft.gpg
+          sudo apt-get update
+          sudo apt-get install microsoft-edge-dev
 
-    - name: Restore Tests dependency cache
-      uses: actions/cache@v2.1.5
-      with:
+      - name: Restore Tests dependency cache
+        uses: actions/cache@v2.1.4
+        with:
           path: |
             ~/cache
             ~/.cache
@@ -60,24 +78,26 @@ jobs:
             !~/cache/exclude
           key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
-              ${{ runner.os }}-yarn
+            ${{ runner.os }}-yarn
 
-    - name: Install test dependencies
-      run: |
+      - name: Install test dependencies
+        run: |
           cd ./main/tests/cypress
           npm i -g yarn
           yarn install
 
-    - name: Lint with ES-Lint & Prettier
-      run: |
-          cd ./main/tests/cypress
-          yarn lint
+      - name: Test with Cypress on ${{ matrix.browser }}
+        run: | 
+          echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
+          echo REFINE_MEMORY=4096M >> ./refine.ini
+          ./refine ui_tests
+        env:
+          CYPRESS_BROWSER: ${{ matrix.browser }}
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
+          CYPRESS_CI_BUILD_ID: '${{ github.run_id }}'
+          CYPRESS_SPECS: ${{ matrix.specs }}
 
-    - name: Test with Cypress on ${{ matrix.browser }}
-      run: | 
-        echo REFINE_MIN_MEMORY=1400M >> ./refine.ini
-        echo REFINE_MEMORY=4096M >> ./refine.ini
-        ./refine ui_test ${{ matrix.browser }} s5du3k "${{ secrets.CYPRESS_RECORD_KEY }}"
 
   build:
 

main/tests/cypress/build-test-matrix.js

@@ -0,0 +1,86 @@
+const glob = require('glob');
+
+// Those specs paths are glob patterns
+const groups = [
+  {
+    specs: [
+      'cypress/integration/create-project/**/*.spec.js',
+      'cypress/integration/extensions/**/*.spec.js',
+      'cypress/integration/import-project/**/*.spec.js',
+      'cypress/integration/language/**/*.spec.js',
+      'cypress/integration/open-project/**/*.spec.js',
+      'cypress/integration/preferences/**/*.spec.js',
+      'cypress/integration/project_management/**/*.spec.js',
+    ],
+  },
+  {
+    specs: [
+      'cypress/integration/project/grid/all-column/**/*.spec.js',
+      'cypress/integration/project/grid/column/*.spec.js',
+      'cypress/integration/project/grid/column/edit-cells/**/*.spec.js',
+    ],
+  },
+  {
+    specs: [
+      'cypress/integration/project/grid/column/edit-column/**/*.spec.js',
+      'cypress/integration/project/grid/column/facet/**/*.spec.js',
+      'cypress/integration/project/grid/column/facet/reconcile/**/*.spec.js',
+    ],
+  },
+  {
+    specs: [
+      'cypress/integration/project/grid/column/transpose/**/*.spec.js',
+      'cypress/integration/project/grid/column/view/**/*.spec.js',
+      'cypress/integration/project/grid/column/reconcile/**/*.spec.js',
+    ],
+  },
+  {
+    specs: [
+      'cypress/integration/project/grid/misc/**/*.spec.js',
+      'cypress/integration/project/grid/row/**/*.spec.js',
+      'cypress/integration/project/grid/viewpanel-header/**/*.spec.js',
+    ],
+  },
+  {
+    specs: [
+      'cypress/integration/project/project-header/**/*.spec.js',
+      'cypress/integration/project/undo_redo/**/*.spec.js',
+      'cypress/integration/tutorial/*.spec.js',
+    ],
+  },
+];
+
+const mergedGroups = groups.map((group) => group.specs.join(','));
+
+// step1 ,find files matched by existing groups
+const matchedFiles = [];
+groups.forEach((group) => {
+  group.specs.forEach((pattern) => {
+    const files = glob.sync(`main/tests/cypress/${pattern}`);
+    matchedFiles.push(...files);
+  });
+});
+
+// step2 , add a last group that contains missed files
+const allSpecFiles = glob.sync(
+  `./main/tests/cypress/cypress/integration/**/*.spec.js`
+);
+const missedFiles = [];
+
+for (const file of allSpecFiles) {
+  const relativeFile = file.substring('./main/tests/cypress/'.length);
+  if (!matchedFiles.includes(file.substring(2))) {
+    missedFiles.push(relativeFile);
+  }
+}
+
+if (missedFiles.length) {
+  mergedGroups.push(missedFiles.join(','));
+}
+
+const browsers = process.env.browsers.split(',');
+console.log(
+  `::set-output name=matrix::{"browser":${JSON.stringify(
+    browsers
+  )}, "specs":${JSON.stringify(mergedGroups)}}`
+);

refine

@@ -67,9 +67,9 @@ and <action> is one of
 
    test ................................ Run all OpenRefine tests
    server_test ......................... Run only the server tests
-   ui_test <browser> <id> <key> ........ Run only the UI tests (If passing a project Id and a Record Key, tests will be recorded in Cypress.io Dashboard)
    extensions_test ..................... Run only the extensions tests
-
+   ui_tests ............................ Run only the UI tests
+   
    broker .............................. Run OpenRefine Broker
    
    broker_appengine_run <id> <ver> ..... Run OpenRefine Broker for Google App Engine in local server
@@ -482,24 +482,20 @@ test() {
     $MVN test
 }
 
-ui_test() {
+
+ui_tests() {
     download http://okfnlabs.org/reconcile-csv/dist/reconcile-csv-0.1.2.jar ./tools/reconcile-csv-0.1.2.jar
     RECONCILE_SERVER_CMD="$JAVA -Xmx2g -jar ./tools/reconcile-csv-0.1.2.jar  ./main/tests/cypress/cypress/fixtures/csv-reconcile-species.csv scientific_name taxon_id"
     echo "Starting reconcile-csv-0.1.2 ..."
     $RECONCILE_SERVER_CMD 2>&1 &
     RECONCILE_SERVER_PID="$!"
     
-
-
     get_revision
 
-    BROWSER="$1"
-    CYPRESS_PROJECT_ID="$2"
-    CYPRESS_RECORD_KEY="$3"
     CYPRESS_RECORD=0
 
-    if [ -z "$BROWSER" ] ; then
-        BROWSER="electron"
+    if [ -z "$CYPRESS_BROWSER" ] ; then
+        CYPRESS_BROWSER="electron"
     fi
     
     if [ ! -z "$CYPRESS_PROJECT_ID" ] && [ ! -z "$CYPRESS_RECORD_KEY" ] ; then
@@ -527,11 +523,14 @@ ui_test() {
     echo ""
 
     echo "Starting Cypress..."
-    CYPRESS_RUN_CMD="yarn --cwd ./main/tests/cypress run cypress run --browser $BROWSER --headless --quiet --reporter list --env OPENREFINE_URL=http://$REFINE_HOST_INTERNAL:$REFINE_PORT"
+    # Cypress needs a unique group id
+    # We're hashing the list of files to generate such Group Id
+    CYPRESS_GROUP=$(echo $CYPRESS_BROWSER$CYPRESS_SPECS | shasum)
+    CYPRESS_RUN_CMD="yarn --cwd ./main/tests/cypress run cypress run --spec "$CYPRESS_SPECS" --browser $CYPRESS_BROWSER --group "$CYPRESS_GROUP" --headless --quiet --reporter list --env OPENREFINE_URL=http://$REFINE_HOST_INTERNAL:$REFINE_PORT"
     if [ "$CYPRESS_RECORD" = "1" ] ; then
         # if tests are recorded, project id is added to env vars, and --record flag is added to the cmd-line
         export CYPRESS_PROJECT_ID=$CYPRESS_PROJECT_ID
-        CYPRESS_RUN_CMD="$CYPRESS_RUN_CMD --record --key $CYPRESS_RECORD_KEY --tag $BROWSER,$REVISION"
+        CYPRESS_RUN_CMD="$CYPRESS_RUN_CMD --record --key $CYPRESS_RECORD_KEY --ci-build-id=$CYPRESS_CI_BUILD_ID --tag $CYPRESS_BROWSER"
     fi
     export MOZ_FORCE_DISABLE_E10S=1
     echo $CYPRESS_RUN_CMD
@@ -560,6 +559,7 @@ ui_test() {
     fi
 }
 
+
 server_test() {
     mvn_prepare
     $MVN test -f main
@@ -991,8 +991,7 @@ case "$ACTION" in
   distclean) mvn distclean;;
   test) test $1;;  
   tests) test $1;;  
-  ui_test) ui_test $1 $2 $3;;  
-  ui_tests) ui_test $1 $2 $3;;  
+  ui_tests) ui_tests;;    
   server_test) server_test $1;;  
   server_tests) server_test $1;;  
   extensions_test) extensions_test $1;;