From 75b1863c1cc423167a1e0b2bb5282e3ab110b8e7 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 1 Feb 2018 08:58:30 +0000 Subject: [PATCH] Testing for variables which generate wikibase values --- .../wikidata/schema/WbItemVariableTest.java | 51 ++++++++++ .../schema/WbLanguageVariableTest.java | 17 ++++ .../schema/WbLocationVariableTest.java | 42 ++++++++ .../wikidata/schema/WbStatementExprTest.java | 13 +++ .../wikidata/schema/WbStringVariableTest.java | 42 ++++++++ .../wikidata/schema/WbVariableTest.java | 95 +++++++++++++++++++ main/src/com/google/refine/model/Recon.java | 14 +++ 7 files changed, 274 insertions(+) create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemVariableTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLanguageVariableTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationVariableTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStringVariableTest.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemVariableTest.java new file mode 100644 index 000000000..c963f9a6a --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbItemVariableTest.java @@ -0,0 +1,51 @@ +package org.openrefine.wikidata.schema; + +import java.util.Collections; + +import org.openrefine.wikidata.schema.entityvalues.ReconItemIdValue; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; + +import com.google.refine.model.Cell; +import com.google.refine.model.Recon; +import com.google.refine.model.ReconCandidate; + +public class WbItemVariableTest extends WbVariableTest { + + @Override + public WbVariableExpr initVariableExpr() { + return new WbItemVariable(); + } + + @Test + public void testReconciledCell() { + Recon recon = Recon.makeWikidataRecon(3782378L); + recon.judgment = Recon.Judgment.Matched; + recon.match = new ReconCandidate("Q123", "some item", null, 100.0); + Cell cell = new Cell("some value", recon); + evaluatesTo(new ReconItemIdValue(recon, "some value"), cell); + } + + @Test + public void testNewItemCell() { + Recon recon = Recon.makeWikidataRecon(3782378L); + recon.judgment = Recon.Judgment.New; + recon.candidates = Collections.singletonList(new ReconCandidate("Q123", "some item", null, 100.0)); + Cell cell = new Cell("some value", recon); + evaluatesTo(new ReconItemIdValue(recon, "some value"), cell); + } + + @Test + public void testUnmatchedCell() { + Recon recon = Recon.makeWikidataRecon(3782378L); + recon.judgment = Recon.Judgment.None; + recon.candidates = Collections.singletonList(new ReconCandidate("Q123", "some item", null, 100.0)); + Cell cell = new Cell("some value", recon); + isSkipped(cell); + } + + @Test + public void testUnreconciledCell() { + isSkipped("some value"); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLanguageVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLanguageVariableTest.java new file mode 100644 index 000000000..cb644ac23 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLanguageVariableTest.java @@ -0,0 +1,17 @@ +package org.openrefine.wikidata.schema; + +import org.testng.annotations.Test; + +public class WbLanguageVariableTest extends WbVariableTest { + + @Override + public WbVariableExpr initVariableExpr() { + return new WbLanguageVariable(); + } + + @Test + public void testValidLanguageCode() { + // TODO + } + +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationVariableTest.java new file mode 100644 index 000000000..1ce6138b2 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbLocationVariableTest.java @@ -0,0 +1,42 @@ +package org.openrefine.wikidata.schema; + +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; + + +public class WbLocationVariableTest extends WbVariableTest { + + @Override + public WbVariableExpr initVariableExpr() { + return new WbLocationVariable(); + } + + @Test + public void testWithSlash() { + evaluatesTo(Datamodel.makeGlobeCoordinatesValue(1.234, 5.678, + WbLocationConstant.defaultPrecision, GlobeCoordinatesValue.GLOBE_EARTH), "1.234/5.678"); + } + + @Test + public void testWithComma() { + evaluatesTo(Datamodel.makeGlobeCoordinatesValue(1.234, 5.678, + WbLocationConstant.defaultPrecision, GlobeCoordinatesValue.GLOBE_EARTH), "1.234,5.678"); + } + + @Test + public void testWhitespace() { + evaluatesTo(Datamodel.makeGlobeCoordinatesValue(1.234, 5.678, + WbLocationConstant.defaultPrecision, GlobeCoordinatesValue.GLOBE_EARTH), " 1.234, 5.678"); + } + + @Test + public void testOnlyOneValue() { + isSkipped("1.2348"); + } + + @Test + public void testEmpty() { + isSkipped(""); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java new file mode 100644 index 000000000..96e1a4047 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStatementExprTest.java @@ -0,0 +1,13 @@ +package org.openrefine.wikidata.schema; + +import java.util.Collections; + +import org.testng.annotations.Test; + +public class WbStatementExprTest { + @Test + public void testCreation() { + WbItemConstant q5 = new WbItemConstant("Q5", "human"); + new WbStatementExpr(q5, Collections.emptyList(), Collections.emptyList()); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStringVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStringVariableTest.java new file mode 100644 index 000000000..5cb9beeb7 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbStringVariableTest.java @@ -0,0 +1,42 @@ +package org.openrefine.wikidata.schema; + +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.StringValue; + +public class WbStringVariableTest extends WbVariableTest { + + @Override + public WbVariableExpr initVariableExpr() { + return new WbStringVariable(); + } + + @Test + public void testEmpty() { + isSkipped(""); + } + + @Test + public void testSimpleString() { + evaluatesTo(Datamodel.makeStringValue("apfelstrudel"), "apfelstrudel"); + } + + /** + * It is not up to the evaluator to clean up the strings it gets. + * This is flagged later on by scrutinizers. + */ + @Test + public void testTrailingWhitespace() { + evaluatesTo(Datamodel.makeStringValue("dirty \t"), "dirty \t"); + } + + @Test + public void testLeadingWhitespace() { + evaluatesTo(Datamodel.makeStringValue(" dirty"), " dirty"); + } + + @Test + public void testDoubleWhitespace() { + evaluatesTo(Datamodel.makeStringValue("very dirty"), "very dirty"); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java new file mode 100644 index 000000000..3f962561d --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbVariableTest.java @@ -0,0 +1,95 @@ +package org.openrefine.wikidata.schema; + +import java.io.IOException; + +import org.openrefine.wikidata.qa.QAWarning; +import org.openrefine.wikidata.qa.QAWarningStore; +import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; + +import com.google.refine.model.Cell; +import com.google.refine.model.ModelException; +import com.google.refine.model.Project; +import com.google.refine.model.Row; +import com.google.refine.tests.RefineTest; + +public abstract class WbVariableTest extends RefineTest { + + protected WbVariableExpr variable; + protected Project project; + protected Row row; + protected ExpressionContext ctxt; + protected QAWarningStore warningStore; + + /** + * This should return a variable expression, to be tested with the helpers below. + * @return + */ + public abstract WbVariableExpr initVariableExpr(); + + @BeforeMethod + public void createProject() throws IOException, ModelException { + project = createCSVProject("Wikidata variable test project", "column A\nrow1"); + warningStore = new QAWarningStore(); + row = project.rows.get(0); + ctxt = new ExpressionContext("http://www.wikidata.org/entity/", 0, + row, project.columnModel, warningStore); + variable = initVariableExpr(); + variable.setColumnName("column A"); + } + + /** + * Test that a particular cell value evaluates to some object + * @param expected + * the expected evaluation of the value + * @param input + * the cell value used by the variable + */ + public void evaluatesTo(T expected, String input) { + Cell cell = new Cell(input, null); + evaluatesTo(expected, cell); + } + + /** + * Test that a particular cell evaluates to some object + * @param expected + * the expected evaluation of the value + * @param cell + * the cell used by the variable + */ + public void evaluatesTo(T expected, Cell cell) { + row.setCell(0, cell); + try { + T result = variable.evaluate(ctxt); + Assert.assertEquals(expected, result); + } catch (SkipSchemaExpressionException e) { + Assert.fail("Value was skipped by evaluator"); + } + } + + /** + * Test that the variable rejects a particular cell value + * @param input + * the cell value to reject + */ + public void isSkipped(String input) { + Cell cell = new Cell(input, null); + isSkipped(cell); + } + + /** + * Test that a particular cell should be rejected by the variable + * @param cell + */ + protected void isSkipped(Cell cell) { + row.setCell(0, cell); + try { + variable.evaluate(ctxt); + Assert.fail("Value was not skipped by evaluator"); + } catch (SkipSchemaExpressionException e) { + return; + } + } +} diff --git a/main/src/com/google/refine/model/Recon.java b/main/src/com/google/refine/model/Recon.java index ff7e04fc8..d131b3f7d 100644 --- a/main/src/com/google/refine/model/Recon.java +++ b/main/src/com/google/refine/model/Recon.java @@ -51,8 +51,14 @@ import com.google.refine.util.Pool; public class Recon implements HasFields, Jsonizable { + /** + * Freebase schema URLs kept for compatibility with legacy reconciliation results + */ private static final String FREEBASE_SCHEMA_SPACE = "http://rdf.freebase.com/ns/type.object.id"; private static final String FREEBASE_IDENTIFIER_SPACE = "http://rdf.freebase.com/ns/type.object.mid"; + + private static final String WIKIDATA_SCHEMA_SPACE = "http://www.wikidata.org/prop/direct/"; + private static final String WIKIDATA_IDENTIFIER_SPACE = "http://www.wikidata.org/entity/"; static public enum Judgment { None, @@ -110,6 +116,7 @@ public class Recon implements HasFields, Jsonizable { public ReconCandidate match = null; public int matchRank = -1; + @Deprecated static public Recon makeFreebaseRecon(long judgmentHistoryEntry) { return new Recon( judgmentHistoryEntry, @@ -117,6 +124,13 @@ public class Recon implements HasFields, Jsonizable { FREEBASE_SCHEMA_SPACE); } + static public Recon makeWikidataRecon(long judgmentHistoryEntry) { + return new Recon( + judgmentHistoryEntry, + WIKIDATA_IDENTIFIER_SPACE, + WIKIDATA_SCHEMA_SPACE); + } + public Recon(long judgmentHistoryEntry, String identifierSpace, String schemaSpace) { id = System.currentTimeMillis() * 1000000 + Math.round(Math.random() * 1000000); this.judgmentHistoryEntry = judgmentHistoryEntry;