From 15ba7fbdde16106611cbafe1c73ec05e7feb7faa Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 3 Mar 2018 19:34:56 +0000 Subject: [PATCH] Schema editing cleanup --- .../module/scripts/warningsrenderer.js | 7 ++- .../openrefine/wikidata/qa/EditInspector.java | 1 + .../wikidata/schema/ExpressionContext.java | 47 +++++++++++++------ .../wikidata/schema/WbItemDocumentExpr.java | 9 ++++ .../wikidata/schema/WbMonolingualExpr.java | 3 ++ .../wikidata/schema/WbPropConstant.java | 3 ++ .../wikidata/schema/WbReferenceExpr.java | 2 + .../wikidata/schema/WbSnakExpr.java | 3 ++ .../wikidata/schema/WbStringConstant.java | 2 + .../entityvalues/SuggestedEntityIdValue.java | 2 + .../SuggestedPropertyIdValue.java | 5 ++ .../schema/ExpressionContextTest.java | 38 +++++++++++++++ 12 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java diff --git a/extensions/wikidata/module/scripts/warningsrenderer.js b/extensions/wikidata/module/scripts/warningsrenderer.js index cf788c46f..120ef1f0e 100644 --- a/extensions/wikidata/module/scripts/warningsrenderer.js +++ b/extensions/wikidata/module/scripts/warningsrenderer.js @@ -15,10 +15,15 @@ WarningsRenderer._renderEntity = function(entity) { fullLabel = entity.label + ' (' + id + ')'; } + var url = entity.iri; + if (!url && entity.value) { + url = 'http://www.wikidata.org/entity/'+entity.value.id; + } + if (is_new) { return ''+fullLabel+''; } else { - return ''+fullLabel+''; + return ''+fullLabel+''; } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java index b90bb3101..8a74bb3c4 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java @@ -74,6 +74,7 @@ public class EditInspector { // so that all newly created entities appear in the batch WikibaseAPIUpdateScheduler scheduler = new WikibaseAPIUpdateScheduler(); editBatch = scheduler.schedule(editBatch); + Map updates = ItemUpdate.groupBySubject(editBatch); List mergedUpdates = updates.values().stream().collect(Collectors.toList()); for(EditScrutinizer scrutinizer : scrutinizers.values()) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java index 471c50ee0..71109a02a 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/ExpressionContext.java @@ -1,9 +1,11 @@ package org.openrefine.wikidata.schema; +import org.apache.commons.lang.Validate; import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.qa.QAWarningStore; import com.google.refine.model.Cell; +import com.google.refine.model.Column; import com.google.refine.model.ColumnModel; import com.google.refine.model.Row; @@ -11,7 +13,7 @@ import com.google.refine.model.Row; * A class holding all the necessary information about * the context in which a schema expression is evaluated. * - * @author antonin + * @author Antonin Delpeuch * */ public class ExpressionContext { @@ -23,12 +25,17 @@ public class ExpressionContext { /** * Builds an expression context to evaluate a schema on a row - * @param baseIRI: the siteIRI of the schema - * @param rowId: the id of the row currently visited - * @param row: the row itself - * @param columnModel: lets us access cells by column name - * @param warningStore: where to store the issues encountered when - * evaluating (can be set to null if these issues should be ignored) + * @param baseIRI + * the siteIRI of the schema + * @param rowId + * the id of the row currently visited + * @param row + * the row itself + * @param columnModel + * lets us access cells by column name + * @param warningStore + * where to store the issues encountered when + * evaluating (can be set to null if these issues should be ignored) */ public ExpressionContext( String baseIRI, @@ -36,9 +43,12 @@ public class ExpressionContext { Row row, ColumnModel columnModel, QAWarningStore warningStore) { + Validate.notNull(baseIRI); this.baseIRI = baseIRI; this.rowId = rowId; + Validate.notNull(row); this.row = row; + Validate.notNull(columnModel); this.columnModel = columnModel; this.warningStore = warningStore; } @@ -46,14 +56,23 @@ public class ExpressionContext { public String getBaseIRI() { return baseIRI; } - - public int getCellIndexByName(String name) { - return columnModel.getColumnByName(name).getCellIndex(); - } - + /** + * Retrieves a cell in the current row, by column name. + * If the column does not exist, null is returned. + * + * @param name + * the name of the column to retrieve the cell from + * @return + * the cell + */ public Cell getCellByName(String name) { - int idx = getCellIndexByName(name); - return row.getCell(idx); + Column column = columnModel.getColumnByName(name); + if (column != null) { + int idx = column.getCellIndex(); + return row.getCell(idx); + } else { + return null; + } } public int getRowId() { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java index 6ce9c21a4..7529603ef 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java @@ -1,7 +1,9 @@ package org.openrefine.wikidata.schema; +import java.util.Collections; import java.util.List; +import org.jsoup.helper.Validate; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdateBuilder; @@ -34,8 +36,15 @@ public class WbItemDocumentExpr extends JacksonJsonizable implements WbExpressio @JsonProperty("subject") WbExpression subjectExpr, @JsonProperty("nameDescs") List nameDescExprs, @JsonProperty("statementGroups") List statementGroupExprs) { + Validate.notNull(subjectExpr); this.subject = subjectExpr; + if(nameDescExprs == null) { + nameDescExprs = Collections.emptyList(); + } this.nameDescs = nameDescExprs; + if(statementGroupExprs == null) { + statementGroupExprs = Collections.emptyList(); + } this.statementGroups = statementGroupExprs; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java index b3b352e99..f5d41d5d0 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbMonolingualExpr.java @@ -1,5 +1,6 @@ package org.openrefine.wikidata.schema; +import org.apache.commons.lang.Validate; import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.helpers.Datamodel; @@ -19,7 +20,9 @@ public class WbMonolingualExpr implements WbExpression { public WbMonolingualExpr( @JsonProperty("language") WbExpression languageExpr, @JsonProperty("value") WbExpression valueExpr) { + Validate.notNull(languageExpr); this.languageExpr = languageExpr; + Validate.notNull(valueExpr); this.valueExpr = valueExpr; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java index 8856a45cc..7d1ebf399 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java @@ -1,5 +1,6 @@ package org.openrefine.wikidata.schema; +import org.jsoup.helper.Validate; import org.openrefine.wikidata.schema.entityvalues.SuggestedPropertyIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; @@ -23,7 +24,9 @@ public class WbPropConstant implements WbExpression { @JsonProperty("pid") String pid, @JsonProperty("label") String label, @JsonProperty("datatype") String datatype) { + Validate.notNull(pid); this.pid = pid; + Validate.notNull(label); this.label = label; this.datatype = datatype; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java index 8344a9305..8f760b2f6 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java @@ -3,6 +3,7 @@ package org.openrefine.wikidata.schema; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang.Validate; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.Reference; @@ -28,6 +29,7 @@ public class WbReferenceExpr implements WbExpression { @JsonCreator public WbReferenceExpr( @JsonProperty("snaks") List snakExprs) { + Validate.notNull(snakExprs); this.snakExprs = snakExprs; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java index efd20e8ff..0ae3c9c08 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java @@ -1,5 +1,6 @@ package org.openrefine.wikidata.schema; +import org.jsoup.helper.Validate; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; @@ -28,7 +29,9 @@ public class WbSnakExpr implements WbExpression { public WbSnakExpr( @JsonProperty("prop") WbExpression propExpr, @JsonProperty("value") WbExpression valueExpr) { + Validate.notNull(propExpr); this.prop = propExpr; + Validate.notNull(valueExpr); this.value = valueExpr; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java index 67d17f016..e8fade299 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java @@ -1,5 +1,6 @@ package org.openrefine.wikidata.schema; +import org.apache.commons.lang.Validate; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.StringValue; @@ -13,6 +14,7 @@ public class WbStringConstant implements WbExpression { @JsonCreator public WbStringConstant(@JsonProperty("value") String value) { + Validate.notNull(value); this.value = value; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedEntityIdValue.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedEntityIdValue.java index 4b06a72e6..a0f7d9bfd 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedEntityIdValue.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedEntityIdValue.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; import org.wikidata.wdtk.datamodel.helpers.Hash; +import org.wikidata.wdtk.datamodel.helpers.ToString; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; @@ -70,4 +71,5 @@ public abstract class SuggestedEntityIdValue implements PrefetchedEntityIdValue public int hashCode() { return Hash.hashCode(this); } + } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedPropertyIdValue.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedPropertyIdValue.java index ef072dc3c..c0b8a513b 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedPropertyIdValue.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/entityvalues/SuggestedPropertyIdValue.java @@ -1,5 +1,6 @@ package org.openrefine.wikidata.schema.entityvalues; +import org.wikidata.wdtk.datamodel.helpers.ToString; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; public class SuggestedPropertyIdValue extends SuggestedEntityIdValue implements PropertyIdValue { @@ -13,4 +14,8 @@ public class SuggestedPropertyIdValue extends SuggestedEntityIdValue implements return ET_PROPERTY; } + @Override + public String toString() { + return "suggested "+ToString.toString(this)+" (\""+getLabel()+"\")"; + } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java new file mode 100644 index 000000000..dd71e53ce --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/ExpressionContextTest.java @@ -0,0 +1,38 @@ +package org.openrefine.wikidata.schema; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.google.refine.model.Project; +import com.google.refine.tests.RefineTest; + +public class ExpressionContextTest extends RefineTest { + + Project project = null; + + @BeforeMethod + public void setUp() { + project = createCSVProject("a,b\nc\nd,e"); + } + + @Test + public void testGetCellByColumnName() { + ExpressionContext ctxt = new ExpressionContext("foo:", 1, project.rows.get(1), project.columnModel, null); + assertEquals("e", ctxt.getCellByName("b").value); + } + + @Test + public void testNonExistentColumn() { + ExpressionContext ctxt = new ExpressionContext("foo:", 1, project.rows.get(1), project.columnModel, null); + assertNull(ctxt.getCellByName("auie")); + } + + @Test + public void testGetRowId() { + ExpressionContext ctxt = new ExpressionContext("foo:", 1, project.rows.get(1), project.columnModel, null); + assertEquals(1, ctxt.getRowId()); + } +}