Reconciliation tests (#3777)

* Reconciliation tests PoC

* removed debug statement

* Added more reconciliation tests

* Fixed typo

* Fixed reconciliation test

* test disabling error handling, troubleshooting

* Attempt to fix flaky integration tests

* restored error handling, fixed flaky test

* Attempt to fix flaly test

* Attempt to fix flaky tests

main/tests/cypress/cypress/fixtures/csv-reconcile-species.csv

@@ -0,0 +1,31 @@
+taxon_id,scientific_name,kingdom,phylum,class,order,family,genus
+2292370,"Cadlinella ornatissima",Animalia,Mollusca,Gastropoda,Nudibranchia,Chromodorididae,Cadlinella
+5213725,"Mola mola",Animalia,Chordata,Actinopterygii,Tetraodontiformes,Molidae,Mola
+2328088,"Protula bispiralis",Animalia,Annelida,Polychaeta,Sabellida,Serpulidae,Protula
+5427631,"Yersinia pestis",Bacteria,Proteobacteria,Gammaproteobacteria,Enterobacteriales,Enterobacteriaceae,Yersinia
+2290924,"Vampyroteuthis infernalis",Animalia,Mollusca,Cephalopoda,Vampyromorpha,Vampyroteuthidae,Vampyroteuthis
+2459658,"Iguana iguana",Animalia,Chordata,Reptilia,Squamata,Iguanidae,Iguana
+1320410,"Paraponera clavata",Animalia,Arthropoda,Insecta,Hymenoptera,Formicidae,Paraponera
+8211794,"Amandinea devilliersiana",Fungi,Ascomycota,Lecanoromycetes,Teloschistales,Physciaceae,Amandinea
+4098695,"Dendrocnide moroides",Plantae,Tracheophyta,Magnoliopsida,Rosales,Urticaceae,Dendrocnide
+2868241,"Monstera deliciosa",Plantae,Tracheophyta,Liliopsida,Alistmatales,Araceae,Monstera
+1865668,"Actias luna",Animalia,Arthropoda,Insecta,Lepidoptera,Saturniidae,Actias
+8023186,"Calostoma cinnabarinum",Fungi,Basidiomycota,Agaricomycetes,Boletales,Calostomataceae,Calostoma
+2255380,"Trichoplax adhaerens",Animalia,Placozoa,,,Trichoplacidae,Trichoplax
+3215941,"Prochlorococcus marinus",Bacteria,Cyanobacteria,Cyanophyceae,Chroococcales,Synechococcaceae,Prochlorococcus
+5220073,"Lagenorhynchus acutus",Animalia,Chordata,Mammalia,Cetacea,Delphinidae,Lagenorhynchus
+5426287,"Hemitrichia serpula",Protozoa,Mycetozoa,Myxomycetes,Trichiales,Trichiaceae,Hemitrichia
+2253634,"Hypsibius dujardini",Animalia,Tardigrada,Eutardigrada,Parachela,Hypsibiidae,Hypsibius
+2895345,"Coffea arabica",Plantae,Tracheophyta,Magnoliopsida,Gentianales,Rubiaceae,Coffea
+2402976,"Facciolella equatorialis",Animalia,Chordata,Actinopterygii,Anguilliformes,Nettastomatidae,Facciolella
+2402978,"Facciolella oxyrhyncha",Animalia,Chordata,Actinopterygii,Anguilliformes,Nettastomatidae,Facciolella
+3926707,"Anemone devinensis",Plantae,Tracheophyta,Magnoliopsida,Ranunculales,Ranunculaceae,Anemone
+3923381,"Anemone robusta",Plantae,Tracheophyta,Magnoliopsida,Ranunculales,Ranunculaceae,Anemone
+2871508,"Amorphophallus titanum",Plantae,Tracheophyta,Liliopsida,Alistmatales,Araceae,Amorphophallus
+2508430,"Lineus longissimus",Animalia,Nemertea,Anopla,,Lineidae,Lineus
+6519335,"Cadlinella sagamiensis",Animalia,Mollusca,Gastropoda,Nudibranchia,Chromodorididae,Cadlinella
+6519570,"Doriprismatica atromarginata",Animalia,Mollusca,Gastropoda,Nudibranchia,Chromodorididae,Doriprismatica
+5724732,"Ceratosoma tenue",Animalia,Mollusca,Gastropoda,Nudibranchia,Chromodorididae,Ceratosoma
+5859604,"Chromodoris magnifica",Animalia,Mollusca,Gastropoda,Nudibranchia,Chromodorididae,Chromodoris
+3193414,"Thalassiosira pseudonana",Chromista,Ochrophyta,Bacillariophyceae,Thalassiosirales,Thalassiosiraceae,Thalassiosira
+2264734,"Physalia physalis",Animalia,Cnidaria,Hydrozoa,Siphonophorae,Physaliidae,Physalia

main/tests/cypress/cypress/integration/preferences/change_preference.spec.js

@@ -1,8 +1,8 @@
 describe(__filename, function () {
   it('Edit a preference', function () {
     cy.visitOpenRefine();
-    const testPreferenceName = 'PreferenceName_' + Date.now();
+    const testPreferenceName = `PreferenceName_${Date.now()}`;
-    const testPreferenceValue = 'PreferenceValue_' + Date.now();
+    const testPreferenceValue = `"PreferenceValue_${Date.now()}"`;
 
     cy.setPreference(testPreferenceName, testPreferenceValue);
 

main/tests/cypress/cypress/integration/project/grid/column/reconcile/actions/clear_reconciliation_data.spec.js

@@ -0,0 +1,29 @@
+describe('Clear reconciliation data', () => {
+  it('Test clearing reconciliation for a reconciled dataset', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['1', '2017-06-23', 'Maryland', 'Hypsibius dujardini'],
+      ['2', '2018-06-09', 'South Carolina', 'Protula bispiralis'],
+      ['3', '2018-06-09', 'West Virginia', 'Monstera deliciosa'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+      ['16', '2017-10-05', 'Maryland', 'Amandinea devilliersiana'],
+      ['24', '2015-05-01', 'West Virginia', 'Faciolela oxyrynca'],
+    ];
+
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species');
+    cy.assertColumnIsReconciled('species');
+
+    cy.columnActionClick('species', [
+      'Reconcile',
+      'Actions',
+      'Clear reconciliation data',
+    ]);
+
+    cy.get('table.data-table').should('not.to.contain', 'Choose new match');
+    // the green bar for matched item should be invisible
+    cy.get(
+      'table.data-table thead div.column-header-recon-stats-matched'
+    ).should('not.be.visible');
+  });
+});

main/tests/cypress/cypress/integration/project/grid/column/reconcile/actions/discard_reconciliation_judgments.spec.js

@@ -0,0 +1,40 @@
+describe('Discard reconciliation judgments', () => {
+  it('Test discard existing reconciliation judgments', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['1', '2017-06-23', 'Maryland', 'Hypsibius dujardini'],
+      ['2', '2018-06-09', 'South Carolina', 'Protula bispiralis'],
+      ['3', '2018-06-09', 'West Virginia', 'Monstera deliciosa'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+      ['16', '2017-10-05', 'Maryland', 'Amandinea devilliersiana'],
+      ['24', '2015-05-01', 'West Virginia', 'Faciolela oxyrynca'],
+    ];
+
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species');
+    cy.assertColumnIsReconciled('species');
+
+    cy.columnActionClick('species', [
+      'Reconcile',
+      'Actions',
+      'Discard reconciliation judgments',
+    ]);
+
+    cy.assertNotificationContainingText('Discard recon judgments for 6 cells');
+
+    // Check that all matches are gone (they contains Choose new match )
+    cy.get('table.data-table td .data-table-cell-content').should(
+      'not.to.contain',
+      'Choose new match'
+    );
+
+    // ensure none of the cells contains 'Search for match'
+    // which means they are not matched and reconciliation judgments are gone
+    cy.getCell(0, 'species').should('to.contain', 'Search for match');
+    cy.getCell(1, 'species').should('to.contain', 'Search for match');
+    cy.getCell(2, 'species').should('to.contain', 'Search for match');
+    cy.getCell(3, 'species').should('to.contain', 'Search for match');
+    cy.getCell(4, 'species').should('to.contain', 'Search for match');
+    cy.getCell(5, 'species').should('to.contain', 'Search for match');
+  });
+});

main/tests/cypress/cypress/integration/project/grid/column/reconcile/actions/match_to_best_candidate.spec.js

@@ -0,0 +1,45 @@
+describe('Match each cell to its best candidate', () => {
+  it('Match each cell to its best candidate', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['1', '2017-06-23', 'Maryland', 'Hypsibius dujardini'],
+      ['2', '2018-06-09', 'South Carolina', 'Protula bispiralis'],
+      ['3', '2018-06-09', 'West Virginia', 'Monstera deliciosa'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+      ['16', '2017-10-05', 'Maryland', 'Amandinea devilliersiana'],
+      ['24', '2015-05-01', 'West Virginia', 'Faciolela oxyrynca'],
+    ];
+
+    cy.loadAndVisitProject(fixture);
+    // here reconcileColumn is called with automatch = false
+    cy.reconcileColumn('species', false);
+    cy.assertColumnIsReconciled('species');
+
+    // before matching, ensure we have no matches
+    cy.getCell(0, 'species').should('to.contain', 'Search for match');
+    cy.getCell(1, 'species').should('to.contain', 'Search for match');
+    cy.getCell(2, 'species').should('to.contain', 'Search for match');
+    cy.getCell(3, 'species').should('to.contain', 'Search for match');
+    cy.getCell(4, 'species').should('to.contain', 'Search for match');
+    cy.getCell(5, 'species').should('to.contain', 'Search for match');
+
+    cy.columnActionClick('species', [
+      'Reconcile',
+      'Actions',
+      'Match each cell to its best candidate',
+    ]);
+
+    cy.assertNotificationContainingText(
+      'Match each of 6 cells to its best candidate'
+    );
+
+    // ensure all cells contains 'Choose new match'
+    // which means they are matched
+    cy.getCell(0, 'species').should('to.contain', 'Choose new match');
+    cy.getCell(1, 'species').should('to.contain', 'Choose new match');
+    cy.getCell(2, 'species').should('to.contain', 'Choose new match');
+    cy.getCell(3, 'species').should('to.contain', 'Choose new match');
+    cy.getCell(4, 'species').should('to.contain', 'Choose new match');
+    cy.getCell(5, 'species').should('to.contain', 'Choose new match');
+  });
+});

main/tests/cypress/cypress/integration/project/grid/column/reconcile/add_entity_identifiers.spec.js

@@ -0,0 +1,38 @@
+describe('Add entity identifiers', () => {
+  it('Add a new column that contains the reconciliation id', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['1', '2017-06-23', 'Maryland', 'Hypsibius dujardini'],
+      ['2', '2018-06-09', 'South Carolina', 'Protula bispiralis'],
+      ['3', '2018-06-09', 'West Virginia', 'Monstera deliciosa'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+      ['16', '2017-10-05', 'Maryland', 'Amandinea devilliersiana'],
+      ['24', '2015-05-01', 'West Virginia', 'Faciolela oxyrynca'],
+    ];
+
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species');
+    cy.assertColumnIsReconciled('species');
+
+    cy.columnActionClick('species', [
+      'Reconcile',
+      'Add entity identifiers column',
+    ]);
+
+    // check the dialog, enter a new column name "id_column"
+    cy.get('.dialog-container .dialog-header').should(
+      'to.contain',
+      'Add column containing entity identifiers on species'
+    );
+    cy.get('.dialog-container .dialog-body input').type('id_column');
+    cy.get('.dialog-container .dialog-footer input').contains('OK').click();
+
+    // Check the cells content for the new column
+    cy.assertCellEquals(0, 'id_column', '2253634'); // untouched
+    cy.assertCellEquals(1, 'id_column', '2328088'); // blanked
+    cy.assertCellEquals(2, 'id_column', '2868241'); // untouched
+    cy.assertCellEquals(3, 'id_column', null); // untouched
+    cy.assertCellEquals(4, 'id_column', '8211794'); // blanked
+    cy.assertCellEquals(5, 'id_column', null); // blanked
+  });
+});

main/tests/cypress/cypress/integration/project/grid/column/reconcile/copy_reconciliation_data.spec.js

@@ -0,0 +1,43 @@
+describe('Copy reconciliation data', () => {
+  it('Copy reconciliation data from species to species_copy', () => {
+    const fixture = [
+      ['species_original', 'species_copy'],
+      ['Hypsibius dujardini', 'Hypsibius dujardini'],
+      ['Protula bispiralis', 'Protula bispiralis'],
+      [null, 'Hypsibius dujardini'], // new line to ensure copy is done one multiple rows
+    ];
+
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species_original');
+    cy.assertColumnIsReconciled('species_original');
+
+    cy.columnActionClick('species_original', [
+      'Reconcile',
+      'Copy reconciliation data',
+    ]);
+
+    // check the dialog, enter a new column name "id_column"
+    cy.get('.dialog-container .dialog-header').should(
+      'to.contain',
+      'Copy recon judgments from column species_original'
+    );
+    cy.get('.dialog-container select[bind="toColumnSelect"]').select(
+      'species_copy'
+    );
+
+    //
+    // cy.assertColumnIsReconciled('species_copy');
+
+    cy.get('.dialog-container .dialog-footer button').contains('Copy').click();
+
+    cy.assertNotificationContainingText(
+      'Copy 3 recon judgments from column species_original to species_copy'
+    );
+
+    // ensure 5 rows are matched based on the identifier
+    // 2 on the original column, 3 on the copy
+    cy.get(
+      'table.data-table td .data-table-cell-content:contains("Choose new match")'
+    ).should('have.length', 5);
+  });
+});

main/tests/cypress/cypress/integration/project/grid/column/reconcile/facets/by_judgment.spec.js

@@ -0,0 +1,26 @@
+describe('Facet by judgment', () => {
+  it('Test clearing reconciliation for a reconciled dataset', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['1', '2017-06-23', 'Maryland', 'Hypsibius dujardini'],
+      ['2', '2018-06-09', 'South Carolina', 'Protula bispiralis'],
+      ['3', '2018-06-09', 'West Virginia', 'Monstera deliciosa'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+      ['16', '2017-10-05', 'Maryland', 'Amandinea devilliersiana'],
+      ['24', '2015-05-01', 'West Virginia', 'Faciolela oxyrynca'],
+    ];
+
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species');
+    cy.assertColumnIsReconciled('species');
+
+    // cleanup automatic facets before doing any testing
+    cy.get('#or-proj-facFil').click();
+    cy.get('#refine-tabs-facets a').contains('Remove All').click();
+
+    cy.columnActionClick('species', ['Reconcile', 'Facets', 'By judgment']);
+
+    // ensure a new facet has been added
+    cy.getFacetContainer('species: judgment').should('exist');
+  });
+});

main/tests/cypress/cypress/integration/project/grid/column/reconcile/reconcile.spec.js

@@ -0,0 +1,244 @@
+// import 'fixture-species';
+
+const fixture = [
+  ['record_id', 'date', 'location', 'species'],
+  ['1', '2017-06-23', 'Maryland', 'Hypsibius dujardini'],
+  ['2', '2018-06-09', 'South Carolina', 'Protula bispiralis'],
+  ['3', '2018-06-09', 'West Virginia', 'Monstera deliciosa'],
+  ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+  ['16', '2017-10-05', 'Maryland', 'Amandinea devilliersiana'],
+  ['24', '2015-05-01', 'West Virginia', 'Faciolela oxyrynca'],
+];
+
+/**
+ * Utility method used by several tests of this scenario
+ * It adds the CSV reconciliation service in the interface
+ */
+const addReconciliationService = () => {
+  cy.get('.recon-dialog-service-list', { log: false }).then(($list) => {
+    if ($list.find('.recon-dialog-service-selector').length > 1) {
+      // cy.get('.recon-dialog-service-selector-remove').click({ multiple: true });
+
+      cy.get('.recon-dialog-service-selector', { log: false }).each(($btn) => {
+        cy.get(
+          '.recon-dialog-service-selector:first-child .recon-dialog-service-selector-remove',
+          { log: false }
+        ).click({ log: false });
+      });
+    }
+  });
+
+  cy.get('.dialog-container button', { log: false })
+    .contains('Add Standard Service...', { log: false })
+    .click({ log: false });
+  cy.get('.dialog-container:last-child input', {
+    log: false,
+  }).type('http://localhost:8000/reconcile', { log: false });
+
+  cy.get('.dialog-container:last-child button', { log: false })
+    .contains('Add Service', { log: false })
+    .click({ log: false });
+
+  cy.get('.recon-dialog-service-selector:last-child', { log: false }).click({
+    log: false,
+  });
+
+  cy.get('.recon-dialog-service-list').should('to.have.css', 'display', 'none');
+};
+
+describe('Base reconciliation tests', () => {
+  it('Load the reconciliation panel, test the layout', () => {
+    cy.loadAndVisitProject(fixture);
+    cy.columnActionClick('species', ['Reconcile', 'Start reconciling']);
+    addReconciliationService();
+    cy.get('.dialog-header').should('to.contain', 'Reconcile column "species"');
+    cy.get('.dialog-body').should(
+      'to.contain',
+      'Reconcile each cell to an entity of one of these types'
+    );
+    cy.get('.dialog-body').should(
+      'to.contain',
+      'Reconcile each cell to an entity of one of these types'
+    );
+    cy.get('.dialog-body .recon-dialog-service-panel').should(
+      'to.contain',
+      'CSV-recon'
+    );
+  });
+
+  it('Reconcile with automatch disabled', () => {
+    cy.loadAndVisitProject(fixture);
+    cy.columnActionClick('species', ['Reconcile', 'Start reconciling']);
+    addReconciliationService();
+
+    cy.get('.dialog-container span')
+      .contains('Auto-match')
+      .siblings('input')
+      .uncheck();
+    cy.get('.dialog-container button').contains('Start Reconciling...').click();
+    cy.assertNotificationContainingText('Reconcile cells in column species');
+    cy.assertColumnIsReconciled('species');
+
+    // "Choose new match" appear when there is a match, if it's not there it means nothing is matched
+    cy.get('table.data-table td .data-table-cell-content').should(
+      'not.to.contain',
+      'Choose new match'
+    );
+  });
+
+  it('Reconcile with automatch enabled', () => {
+    cy.loadAndVisitProject(fixture);
+    cy.columnActionClick('species', ['Reconcile', 'Start reconciling']);
+    addReconciliationService();
+
+    cy.get('.dialog-container span')
+      .contains('Auto-match')
+      .siblings('input')
+      .check();
+    cy.get('.dialog-container button').contains('Start Reconciling...').click();
+    cy.assertNotificationContainingText('Reconcile cells in column species');
+    cy.assertColumnIsReconciled('species');
+
+    // 4 rows should have been automatched
+    cy.get(
+      'table.data-table td .data-table-cell-content:contains("Choose new match")'
+    ).should('have.length', 4);
+  });
+
+  it('Max number of candidates', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+    ];
+    cy.loadAndVisitProject(fixture);
+    cy.columnActionClick('species', ['Reconcile', 'Start reconciling']);
+    addReconciliationService();
+    cy.get('.dialog-container input[bind="maxCandidates"]').type(2);
+    cy.get('.dialog-container button').contains('Start Reconciling...').click();
+    cy.assertColumnIsReconciled('species');
+    cy.get('.data-table-cell-content .data-table-recon-topic').should(
+      'have.length',
+      2
+    );
+  });
+
+  it('Test reconciliation, in the grid', () => {
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species');
+    cy.assertColumnIsReconciled('species');
+    // Test 1, when there is a match
+    // simple assertion to ensure the content of the iframe is loaded for row 0 / species
+    // (recon loads match details from the recon endpoint in an iframe)
+    cy.getCell(0, 'species').within(() => {
+      cy.get('a[href^="http://localhost:8000/"]').trigger('mouseover');
+      cy.get(
+        'a[href^="http://localhost:8000/"] .data-table-topic-popup'
+      ).should('to.exist');
+
+      cy.get('iframe').its('0.contentDocument').should('exist');
+    });
+
+    // Test 2, when there are candidates
+    // ensure the first one loads an iframe on hover
+    cy.getCell(3, 'species').within(() => {
+      cy.get(
+        '.data-table-recon-candidate:first-child .data-table-recon-topic'
+      ).trigger('mouseover');
+      cy.get(
+        '.data-table-recon-candidate:first-child .data-table-topic-popup'
+      ).should('to.exist');
+
+      cy.get(
+        '.data-table-recon-candidate:first-child .data-table-topic-popup iframe'
+      )
+        .its('0.contentDocument')
+        .should('exist');
+
+      // verify the three buttons inside the popup
+      cy.get(
+        '.data-table-recon-candidate:first-child .data-table-topic-popup'
+      ).within(() => {
+        cy.get('button').contains('Match this Cell').should('to.exist');
+        cy.get('button')
+          .contains('Match All Identical Cells')
+          .should('to.exist');
+        cy.get('button').contains('Cancel').should('to.exist');
+      });
+    });
+  });
+
+  it('Match this cell', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+      ['3', '2018-06-09', 'West Virginia', 'Monstera deliciosa'],
+    ];
+
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species');
+    cy.assertColumnIsReconciled('species');
+
+    // over on a candidate (Lineus longissimus)
+    cy.getCell(0, 'species')
+      .find('a')
+      .contains('Lineus longissimus')
+      .trigger('mouseover');
+
+    // click on match
+    cy.getCell(0, 'species').find('button').contains('Match this Cell').click();
+
+    // check notification
+    cy.assertNotificationContainingText('Match Lineus longissimus');
+
+    // check that candidates have disappeared, means matched
+    cy.getCell(0, 'species')
+      .find('.data-table-recon-candidates')
+      .should('not.to.exist');
+  });
+
+  it('Match All identical cell', () => {
+    const fixture = [
+      ['record_id', 'date', 'location', 'species'],
+      ['15', '2018-09-06', 'West Virginia', 'Bos taurus'],
+      ['16', '2018-09-06', 'West Virginia', 'Bos taurus'],
+    ];
+    cy.loadAndVisitProject(fixture);
+    cy.reconcileColumn('species');
+    cy.assertColumnIsReconciled('species');
+
+    // ensure both rows have candidates
+    cy.getCell(0, 'species')
+      .find('.data-table-recon-candidate')
+      .should('have.length', 6);
+
+    cy.getCell(1, 'species')
+      .find('.data-table-recon-candidate')
+      .should('have.length', 6);
+
+    // over on a candidate (Lineus longissimus)
+    cy.getCell(0, 'species')
+      .find('a')
+      .contains('Lineus longissimus')
+      .trigger('mouseover');
+
+    // click on match all
+    cy.getCell(0, 'species')
+      .find('button')
+      .contains('Match All Identical Cells')
+      .click();
+
+    // check notification
+    cy.assertNotificationContainingText(
+      'Match item Lineus longissimus (2508430) for 2 cells'
+    );
+
+    // check that candidates have disappeared on both rows, means matched all
+    cy.getCell(0, 'species')
+      .find('.data-table-recon-candidates')
+      .should('not.to.exist');
+
+    cy.getCell(1, 'species')
+      .find('.data-table-recon-candidates')
+      .should('not.to.exist');
+  });
+});

main/tests/cypress/cypress/integration/project/grid/column/reconcile/use_values_as_identifiers.spec.js

@@ -0,0 +1,30 @@
+describe('Use values as identifiers', () => {
+  it('Test clearing reconciliation for a reconciled dataset', () => {
+    const fixture = [
+      ['identifier'],
+      ['2253634'],
+      ['2328088'],
+      ['2868241'],
+      [null],
+      ['8211794'],
+      [null],
+    ];
+
+    cy.loadAndVisitProject(fixture);
+
+    cy.columnActionClick('identifier', [
+      'Reconcile',
+      'Use values as identifiers',
+    ]);
+
+    cy.get('.dialog-container .dialog-footer button').contains('OK').click();
+
+    // ensure column is reconciled
+    cy.assertColumnIsReconciled('identifier');
+
+    // ensure 4 rows are matched based on the identifier
+    cy.get(
+      'table.data-table td .data-table-cell-content:contains("Choose new match")'
+    ).should('have.length', 4);
+  });
+});

main/tests/cypress/cypress/support/commands.js

@@ -15,6 +15,86 @@ import { addMatchImageSnapshotCommand } from 'cypress-image-snapshot/command';
 
 addMatchImageSnapshotCommand({ customDiffDir: 'cypress/snapshots_diffs' });
 
+/**
+ * Reconcile a column
+ * Internally using the "apply" behavior for not having to go through the whole user interface
+ */
+Cypress.Commands.add('reconcileColumn', (columnName, autoMatch = true) => {
+  cy.setPreference(
+    'reconciliation.standardServices',
+    encodeURIComponent(
+      JSON.stringify([
+        {
+          name: 'CSV Reconciliation service',
+          identifierSpace: 'http://localhost:8000/',
+          schemaSpace: 'http://localhost:8000/',
+          defaultTypes: [],
+          view: { url: 'http://localhost:8000/view/{{id}}' },
+          preview: {
+            width: 500,
+            url: 'http://localhost:8000/view/{{id}}',
+            height: 350,
+          },
+          suggest: {
+            entity: {
+              service_url: 'http://localhost:8000',
+              service_path: '/suggest',
+              flyout_service_url: 'http://localhost:8000',
+              flyout_sercice_path: '/flyout',
+            },
+          },
+          url: 'http://localhost:8000/reconcile',
+          ui: { handler: 'ReconStandardServicePanel', access: 'jsonp' },
+        },
+      ])
+    )
+  ).then(() => {
+    const apply = [
+      {
+        op: 'core/recon',
+        engineConfig: {
+          facets: [],
+          mode: 'row-based',
+        },
+        columnName: columnName,
+        config: {
+          mode: 'standard-service',
+          service: 'http://localhost:8000/reconcile',
+          identifierSpace: 'http://localhost:8000/',
+          schemaSpace: 'http://localhost:8000/',
+          type: {
+            id: '/csv-recon',
+            name: 'CSV-recon',
+          },
+          autoMatch: autoMatch,
+          columnDetails: [],
+          limit: 0,
+        },
+        description: 'Reconcile cells in column species to type /csv-recon',
+      },
+    ];
+    cy.get('a#or-proj-undoRedo').click();
+    cy.get('#refine-tabs-history .history-panel-controls')
+      .contains('Apply')
+      .click();
+    cy.get('.dialog-container .history-operation-json').invoke(
+      'val',
+      JSON.stringify(apply)
+    );
+    cy.get('.dialog-container button[bind="applyButton"]').click();
+  });
+});
+
+/**
+ * Reconcile a column
+ * Internally using the "apply" behavior for not having to go through the whole user interface
+ */
+Cypress.Commands.add('assertColumnIsReconciled', (columnName) => {
+  cy.get(
+    `table.data-table thead th[title="${columnName}"] div.column-header-recon-stats-matched`
+  ).should('to.exist');
+});
+
 /**
  * Return the .facets-container for a given facet name
  */

main/tests/cypress/cypress/support/openrefine_api.js

@@ -2,23 +2,28 @@ const fixtures = require('../fixtures/fixtures.js');
 
 Cypress.Commands.add('setPreference', (preferenceName, preferenceValue) => {
   const openRefineUrl = Cypress.env('OPENREFINE_URL');
-  cy.request(openRefineUrl + '/command/core/get-csrf-token').then(
+  return cy
-    (response) => {
+    .request(openRefineUrl + '/command/core/get-csrf-token')
-      cy.request({
+    .then((response) => {
-        method: 'POST',
+      return cy
-        url: `${openRefineUrl}/command/core/set-preference`,
+        .request({
-        body: `name=${preferenceName}&value="${preferenceValue}"&csrf_token=${response.body.token}`,
+          method: 'POST',
-        form: false,
+          url: `${openRefineUrl}/command/core/set-preference`,
-        headers: {
+          body: `name=${preferenceName}&value=${preferenceValue}&csrf_token=${response.body.token}`,
-          'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
+          form: false,
-        },
+          headers: {
-      }).then((resp) => {
+            'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
-        cy.log(
+          },
-          'Set preference ' + preferenceName + ' with value ' + preferenceValue
+        })
-        );
+        .then((resp) => {
-      });
+          cy.log(
-    }
+            'Set preference ' +
-  );
+              preferenceName +
+              ' with value ' +
+              preferenceValue
+          );
+        });
+    });
 });
 
 Cypress.Commands.add('cleanupProjects', () => {

refine

@@ -483,6 +483,14 @@ test() {
 }
 
 ui_test() {
+    download http://okfnlabs.org/reconcile-csv/dist/reconcile-csv-0.1.2.jar ./tools/reconcile-csv-0.1.2.jar
+    RECONCILE_SERVER_CMD="$JAVA -Xmx2g -jar ./tools/reconcile-csv-0.1.2.jar  ./main/tests/cypress/cypress/fixtures/csv-reconcile-species.csv scientific_name taxon_id"
+    echo "Starting reconcile-csv-0.1.2 ..."
+    $RECONCILE_SERVER_CMD 2>&1 &
+    RECONCILE_SERVER_PID="$!"
+    
+
+
     get_revision
 
     BROWSER="$1"
@@ -542,6 +550,8 @@ ui_test() {
     echo ""
     echo "Killing OpenRefine"
     /bin/kill -9 $REFINE_PID
+    echo "Killing Reconciliation Server"
+    /bin/kill -9 $RECONCILE_SERVER_PID
     echo "Cleaning up"
     rm -rf "$REFINE_DATA_DIR"