diff --git a/extensions/wikidata/lib-local/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar b/extensions/wikidata/lib-local/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar deleted file mode 100644 index 4cefe5a59..000000000 Binary files a/extensions/wikidata/lib-local/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar and /dev/null differ diff --git a/extensions/wikidata/lib-local/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar b/extensions/wikidata/lib-local/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar deleted file mode 100644 index fb099272b..000000000 Binary files a/extensions/wikidata/lib-local/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar and /dev/null differ diff --git a/extensions/wikidata/pom.xml b/extensions/wikidata/pom.xml index d11858f4b..00c2b0b01 100644 --- a/extensions/wikidata/pom.xml +++ b/extensions/wikidata/pom.xml @@ -93,45 +93,6 @@ - - org.apache.maven.plugins - maven-install-plugin - 2.5.2 - - - install-wdtk-datamodel - process-resources - - ${basedir}/lib-local/wdtk-datamodel-0.9.0-SNAPSHOT-jar-with-dependencies.jar - default - org.wikidata.wdtk - wdtk-datamodel - 0.9.0-SNAPSHOT - jar - true - - - install-file - - - - install-wdtk-wikibaseapi - process-resources - - ${basedir}/lib-local/wdtk-wikibaseapi-0.9.0-SNAPSHOT.jar - default - org.wikidata.wdtk - wdtk-wikibaseapi - 0.9.0-SNAPSHOT - jar - true - - - install-file - - - - org.apache.maven.plugins maven-clean-plugin @@ -163,12 +124,12 @@ org.wikidata.wdtk wdtk-wikibaseapi - 0.9.0-SNAPSHOT + 0.10.0 org.wikidata.wdtk wdtk-datamodel - 0.9.0-SNAPSHOT + 0.10.0 org.slf4j diff --git a/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java b/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java index 4618febc2..5c8e98ef7 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/editing/EditBatchProcessor.java @@ -211,7 +211,10 @@ public class EditBatchProcessor { } catch (MediaWikiApiErrorException e) { e.printStackTrace(); Thread.sleep(5000); - } + } catch (IOException e) { + e.printStackTrace(); + Thread.sleep(5000); + } retries--; } if (currentDocs == null) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java index 4a4153f57..5e4e86a10 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSValuePrinter.java @@ -31,6 +31,7 @@ import org.openrefine.wikidata.updates.scheduler.QuickStatementsUpdateScheduler; import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; import org.wikidata.wdtk.datamodel.interfaces.StringValue; @@ -50,12 +51,6 @@ import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; */ public class QSValuePrinter implements ValueVisitor { - @Override - public String visit(DatatypeIdValue value) { - // impossible case (this is actually a bug in WDTK, DatatypeIdValue should not subclass Value) - throw new IllegalArgumentException(); - } - @Override public String visit(EntityIdValue value) { if (ReconEntityIdValue.class.isInstance(value) && ((ReconEntityIdValue) value).isNew()) { @@ -76,12 +71,10 @@ public class QSValuePrinter implements ValueVisitor { @Override public String visit(QuantityValue value) { - String unitPrefix = "http://www.wikidata.org/entity/Q"; - String unitIri = value.getUnit(); + ItemIdValue unit = value.getUnitItemId(); String unitRepresentation = "", boundsRepresentation = ""; - if (!unitIri.isEmpty()) { - if (!unitIri.startsWith(unitPrefix)) return null; // QuickStatements only accepts Qids as units - unitRepresentation = "U" + unitIri.substring(unitPrefix.length()); + if (unit != null) { + unitRepresentation = "U" + unit.getId().substring(1); } if (value.getLowerBound() != null) { // bounds are always null at the same time so we know they are both not null diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java index ea711fe56..314fc1550 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java @@ -45,22 +45,19 @@ public class QuantityScrutinizer extends SnakScrutinizer { addIssue(issue); } Set allowedUnits = _fetcher.allowedUnits(pid); - String currentUnit = null; - if (value.getUnit() != null && !value.getUnit().equals("")) { - currentUnit = value.getUnit(); + ItemIdValue currentUnit = null; + if (value.getUnitItemId() != null) { + currentUnit = value.getUnitItemId(); } if(allowedUnits != null && - !allowedUnits.stream().map(u -> u != null ? u.getIri() : null) - .collect(Collectors.toSet()).contains(currentUnit)) { + !allowedUnits.contains(currentUnit)) { String issueType = currentUnit == null ? noUnitProvidedType : invalidUnitType; QAWarning issue = new QAWarning(issueType, pid.getId(), QAWarning.Severity.IMPORTANT, 1); issue.setProperty("property_entity", pid); issue.setProperty("example_value", value.getNumericValue().toString()); issue.setProperty("example_item_entity", entityId); if (currentUnit != null) { - issue.setProperty("unit_entity", - // this is a hack but it will not be needed anymore in the upcoming version of Wikidata-Toolkit - Datamodel.makeWikidataItemIdValue(currentUnit.substring(currentUnit.indexOf("Q")))); + issue.setProperty("unit_entity", value.getUnitItemId()); } addIssue(issue); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java index 724c8656d..53b89ba5e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java @@ -28,6 +28,7 @@ import java.util.List; import org.jsoup.helper.Validate; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; +import org.openrefine.wikidata.utils.StatementGroupJson; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java b/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java index d9571ffcb..46def6711 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/updates/ItemUpdate.java @@ -36,6 +36,8 @@ import java.util.function.Function; import java.util.stream.Collectors; import org.jsoup.helper.Validate; +import org.openrefine.wikidata.utils.StatementGroupJson; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.implementation.StatementGroupImpl; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; @@ -176,7 +178,7 @@ public class ItemUpdate { * * @return the list of all added statements */ - @JsonProperty("addedStatements") + @JsonIgnore // exposed as statement groups below public List getAddedStatements() { return addedStatements; } @@ -295,6 +297,7 @@ public class ItemUpdate { * * @return a grouped version of getAddedStatements() */ + @JsonIgnore public List getAddedStatementGroups() { Map> map = new HashMap<>(); for (Statement statement : getAddedStatements()) { @@ -306,10 +309,22 @@ public class ItemUpdate { } List result = new ArrayList<>(); for (Map.Entry> entry : map.entrySet()) { + // We have to do this rather than use Datamodel in order to preserve the + // custom entity id values which can link to new items. result.add(new StatementGroupImpl(entry.getValue())); } return result; } + + /** + * Json serialization for preview of item updates. Because StatementGroup + * is not designed for serialization (so its format is not specified by WDTK), + * we add a wrapper on top to specify it. + */ + @JsonProperty("addedStatementGroups") + public List getAddedStatementGroupsJson() { + return this.getAddedStatementGroups().stream().map(s -> new StatementGroupJson(s)).collect(Collectors.toList()); + } /** * Group a list of ItemUpdates by subject: this is useful to make one single @@ -339,6 +354,7 @@ public class ItemUpdate { /** * Is this update about a new item? */ + @JsonProperty("new") public boolean isNew() { return EntityIdValue.SITE_LOCAL.equals(getItemId().getSiteIri()); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java b/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java index 7fac7b09e..6bb801379 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java @@ -122,11 +122,6 @@ public class PointerExtractor implements ValueVisitor> { return pointers; } - @Override - public Set visit(DatatypeIdValue value) { - return null; - } - @Override public Set visit(EntityIdValue value) { if (ReconItemIdValue.class.isInstance(value)) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/utils/StatementGroupJson.java b/extensions/wikidata/src/org/openrefine/wikidata/utils/StatementGroupJson.java new file mode 100644 index 000000000..71796f129 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/utils/StatementGroupJson.java @@ -0,0 +1,43 @@ +package org.openrefine.wikidata.utils; + +import java.util.List; + +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Wikidata-Toolkit's StatementGroup class is not designed to be serialized, + * so its serialization via Jackson is not specified. This adds annotations + * to specify its behaviour. + * + * @author Antonin Delpeuch + */ +public class StatementGroupJson { + + protected final StatementGroup statementGroup; + + public StatementGroupJson(StatementGroup s) { + statementGroup = s; + } + + @JsonProperty("subject") + public EntityIdValue getSubject() { + return statementGroup.getSubject(); + } + + @JsonProperty("property") + public PropertyIdValue getProperty() { + return statementGroup.getProperty(); + } + + @JsonProperty("statements") + public List getStatements() { + return statementGroup.getStatements(); + } + +} diff --git a/extensions/wikidata/tests/data/updates/statement_groups.json b/extensions/wikidata/tests/data/updates/statement_groups.json new file mode 100644 index 000000000..bf5f019b1 --- /dev/null +++ b/extensions/wikidata/tests/data/updates/statement_groups.json @@ -0,0 +1,83 @@ +{ + "addedAliases": [], + "addedStatementGroups": [ + { + "property": { + "type": "wikibase-entityid", + "value": { + "entity-type": "property", + "id": "P348", + "numeric-id": 348 + } + }, + "statements": [ + { + "mainsnak": { + "property": "P348", + "snaktype": "novalue" + }, + "rank": "normal", + "type": "statement" + } + ], + "subject": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q34", + "numeric-id": 34 + } + } + }, + { + "property": { + "type": "wikibase-entityid", + "value": { + "entity-type": "property", + "id": "P52", + "numeric-id": 52 + } + }, + "statements": [ + { + "mainsnak": { + "datatype": "wikibase-item", + "datavalue": { + "entityType": "http://www.wikidata.org/ontology#Item", + "id": "Q1234", + "iri": "http://localhost/entity/Q1234", + "label": "new item", + "reconInternalId": 1234, + "siteIri": "http://localhost/entity/", + "types": [] + }, + "property": "P52", + "snaktype": "value" + }, + "rank": "normal", + "type": "statement" + } + ], + "subject": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q34", + "numeric-id": 34 + } + } + } + ], + "deletedStatements": [], + "descriptions": [], + "labels": [], + "new": false, + "subject": { + "type": "wikibase-entityid", + "value": { + "entity-type": "item", + "id": "Q34", + "numeric-id": 34 + } + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java index 9443f167d..2aef6295c 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/ItemUpdateTest.java @@ -28,6 +28,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -48,6 +49,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; import org.wikidata.wdtk.datamodel.interfaces.StatementRank; +import com.google.refine.tests.util.TestUtils; + public class ItemUpdateTest { private ItemIdValue existingSubject = Datamodel.makeWikidataItemIdValue("Q34"); @@ -110,6 +113,23 @@ public class ItemUpdateTest { assertEquals(Arrays.asList(statement1, statement2), update.getAddedStatements()); assertEquals(statementGroups, update.getAddedStatementGroups().stream().collect(Collectors.toSet())); } + + /** + * Test disabled because it fails due to + * https://github.com/Wikidata/Wikidata-Toolkit/issues/417 + * (not fixed as of WDTK 0.10.0). + * + * This bug is not critical as the extraneous serialized data + * is ignored by Wikibase. + * + * @todo reenable once a later version is released + */ + @Test(enabled=false) + public void testSerializeStatements() throws IOException { + ItemUpdate update = new ItemUpdateBuilder(existingSubject).addStatement(statement1).addStatement(statement2) + .build(); + TestUtils.isSerializedTo(update, TestingData.jsonFromFile("updates/statement_groups.json")); + } @Test public void testDeleteStatements() { diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java index 6bab3955a..5671c173e 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java @@ -66,7 +66,6 @@ public class PointerExtractorTest { @Test public void testExtractDatavalues() { - assertEmpty(Datamodel.makeDatatypeIdValue("string")); assertEmpty(Datamodel.makeGlobeCoordinatesValue(1.34, 2.354, 0.1, GlobeCoordinatesValue.GLOBE_EARTH)); assertEmpty(Datamodel.makeStringValue("est")); assertEmpty(Datamodel.makeMonolingualTextValue("srtu", "en"));