diff --git a/extensions/wikidata/pom.xml b/extensions/wikidata/pom.xml index 8d3df0a13..ecc7a841d 100644 --- a/extensions/wikidata/pom.xml +++ b/extensions/wikidata/pom.xml @@ -16,7 +16,7 @@ 2.0.9 - 0.12.0-SNAPSHOT + 0.12.1 @@ -127,7 +127,7 @@ provided - org.openrefine.dependencies.wdtk + org.wikidata.wdtk wdtk-wikibaseapi ${wdtk.version} @@ -137,12 +137,12 @@ ${jackson.version} - org.openrefine.dependencies.wdtk + org.wikidata.wdtk wdtk-datamodel ${wdtk.version} - org.openrefine.dependencies.wdtk + org.wikidata.wdtk wdtk-util ${wdtk.version} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSSnakPrinter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSSnakPrinter.java new file mode 100644 index 000000000..50d347c9f --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QSSnakPrinter.java @@ -0,0 +1,52 @@ +package org.openrefine.wikidata.exporters; + +import org.wikidata.wdtk.datamodel.interfaces.NoValueSnak; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.SnakVisitor; +import org.wikidata.wdtk.datamodel.interfaces.SomeValueSnak; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; +import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; + +/** + * Represents a snak in QuickStatements format. + * + * @author Antonin Delpeuch + * + */ +public class QSSnakPrinter implements SnakVisitor { + + protected final boolean reference; + protected final ValueVisitor valuePrinter = new QSValuePrinter(); + + /** + * @param reference indicates whether to print snaks as reference, or as main/qualifier snaks + */ + public QSSnakPrinter(boolean reference) { + this.reference = reference; + } + + @Override + public String visit(ValueSnak snak) { + String valStr = snak.getValue().accept(valuePrinter); + return toQS(snak.getPropertyId(), valStr); + } + + @Override + public String visit(SomeValueSnak snak) { + return toQS(snak.getPropertyId(), "somevalue"); + } + + @Override + public String visit(NoValueSnak snak) { + return toQS(snak.getPropertyId(), "novalue"); + } + + protected String toQS(PropertyIdValue property, String value) { + String pid = property.getId(); + if (reference) { + pid = pid.replace('P', 'S'); + } + return "\t" + pid + "\t" + value; + } + +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java index a13f1a20e..dd2c6d162 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java @@ -42,8 +42,6 @@ import org.wikidata.wdtk.datamodel.interfaces.Reference; import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; -import org.wikidata.wdtk.datamodel.interfaces.Value; -import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; import com.google.refine.browsing.Engine; import com.google.refine.exporters.WriterExporter; @@ -55,8 +53,13 @@ public class QuickStatementsExporter implements WriterExporter { public static final String impossibleSchedulingErrorMessage = "This edit batch cannot be performed with QuickStatements due to the structure of its new items."; public static final String noSchemaErrorMessage = "No schema was provided. You need to align your project with Wikidata first."; + + protected final QSSnakPrinter mainSnakPrinter; + protected final QSSnakPrinter referenceSnakPrinter; public QuickStatementsExporter() { + mainSnakPrinter = new QSSnakPrinter(false); + referenceSnakPrinter = new QSSnakPrinter(true); } @Override @@ -148,39 +151,36 @@ public class QuickStatementsExporter implements WriterExporter { throws IOException { Claim claim = statement.getClaim(); - Value val = claim.getValue(); - ValueVisitor vv = new QSValuePrinter(); - String targetValue = val.accept(vv); - if (targetValue != null) { // issue #2320 - if (!add) { - // According to: https://www.wikidata.org/wiki/Help:QuickStatements#Removing_statements, - // Removing statements won't be followed by qualifiers or references. - writer.write("- "); - writer.write(qid + "\t" + pid + "\t" + targetValue); + Snak mainSnak = claim.getMainSnak(); + String mainSnakQS = mainSnak.accept(mainSnakPrinter); + if (!add) { + // According to: https://www.wikidata.org/wiki/Help:QuickStatements#Removing_statements, + // Removing statements won't be followed by qualifiers or references. + writer.write("- "); + writer.write(qid + mainSnakQS); + writer.write("\n"); + } else { // add statements + if (statement.getReferences().isEmpty()) { + writer.write(qid + mainSnakQS); + for (SnakGroup q : claim.getQualifiers()) { + translateSnakGroup(q, false, writer); + } writer.write("\n"); - } else { // add statements - if (statement.getReferences().isEmpty()) { - writer.write(qid + "\t" + pid + "\t" + targetValue); + } else { + // According to: https://www.wikidata.org/wiki/Help:QuickStatements#Add_statement_with_sources + // Existing statements with an exact match (property and value) will not be added again; + // however additional references might be added to the statement. + + // So, to handle multiple references, we can duplicate the statement just with different references. + for (Reference r : statement.getReferences()) { + writer.write(qid + mainSnakQS); for (SnakGroup q : claim.getQualifiers()) { translateSnakGroup(q, false, writer); } - writer.write("\n"); - } else { - // According to: https://www.wikidata.org/wiki/Help:QuickStatements#Add_statement_with_sources - // Existing statements with an exact match (property and value) will not be added again; - // however additional references might be added to the statement. - - // So, to handle multiple references, we can duplicate the statement just with different references. - for (Reference r : statement.getReferences()) { - writer.write(qid + "\t" + pid + "\t" + targetValue); - for (SnakGroup q : claim.getQualifiers()) { - translateSnakGroup(q, false, writer); - } - for (SnakGroup g : r.getSnakGroups()) { - translateSnakGroup(g, true, writer); - } - writer.write("\n"); + for (SnakGroup g : r.getSnakGroups()) { + translateSnakGroup(g, true, writer); } + writer.write("\n"); } } } @@ -189,21 +189,11 @@ public class QuickStatementsExporter implements WriterExporter { protected void translateSnakGroup(SnakGroup sg, boolean reference, Writer writer) throws IOException { for (Snak s : sg.getSnaks()) { - translateSnak(s, reference, writer); - } - } - - protected void translateSnak(Snak s, boolean reference, Writer writer) - throws IOException { - String pid = s.getPropertyId().getId(); - if (reference) { - pid = pid.replace('P', 'S'); - } - Value val = s.getValue(); - ValueVisitor vv = new QSValuePrinter(); - String valStr = val.accept(vv); - if (valStr != null) { - writer.write("\t" + pid + "\t" + valStr); + if (reference) { + writer.write(s.accept(referenceSnakPrinter)); + } else { + writer.write(s.accept(mainSnakPrinter)); + } } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/Constraint.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/Constraint.java index 99cdf3812..259712bd8 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/Constraint.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/Constraint.java @@ -5,6 +5,7 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.HashSet; import java.util.List; @@ -31,11 +32,11 @@ public class Constraint { List snakGroupList = statement.getClaim().getQualifiers(); for(SnakGroup group : snakGroupList) { for (Snak snak : group.getSnaks()) { - if (group.getProperty().getId().equals(CONSTRAINT_STATUS)) { - constraintStatus = (ItemIdValue) snak.getValue(); + if (group.getProperty().getId().equals(CONSTRAINT_STATUS) && snak instanceof ValueSnak) { + constraintStatus = (ItemIdValue) ((ValueSnak)snak).getValue(); } - else if (group.getProperty().getId().equals(CONSTRAINT_EXCEPTIONS)) { - constraintExceptions.add((EntityIdValue) snak.getValue()); + else if (group.getProperty().getId().equals(CONSTRAINT_EXCEPTIONS) && snak instanceof ValueSnak) { + constraintExceptions.add((EntityIdValue) ((ValueSnak)snak).getValue()); } } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ConflictsWithScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ConflictsWithScrutinizer.java index 562a6c555..29733c075 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ConflictsWithScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ConflictsWithScrutinizer.java @@ -7,6 +7,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.ArrayList; import java.util.HashMap; @@ -32,11 +33,11 @@ public class ConflictsWithScrutinizer extends EditScrutinizer { this.itemList = new ArrayList<>(); for(SnakGroup group : specs) { for (Snak snak : group.getSnaks()) { - if (group.getProperty().getId().equals(conflictsWithPropertyPid)){ - pid = (PropertyIdValue) snak.getValue(); + if (group.getProperty().getId().equals(conflictsWithPropertyPid) && snak instanceof ValueSnak) { + pid = (PropertyIdValue) ((ValueSnak)snak).getValue(); } - if (group.getProperty().getId().equals(itemOfPropertyConstraintPid)){ - this.itemList.add(snak.getValue()); + if (group.getProperty().getId().equals(itemOfPropertyConstraintPid) && snak instanceof ValueSnak) { + this.itemList.add(((ValueSnak)snak).getValue()); } } } @@ -59,7 +60,11 @@ public class ConflictsWithScrutinizer extends EditScrutinizer { Map> propertyIdValueValueMap = new HashMap<>(); for (Statement statement : update.getAddedStatements()){ PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId(); - Value value = statement.getClaim().getMainSnak().getValue(); + Value value = null; + Snak mainSnak = statement.getClaim().getMainSnak(); + if (mainSnak instanceof ValueSnak) { + value = ((ValueSnak)mainSnak).getValue(); + } Set values; if (value != null) { if (propertyIdValueValueMap.containsKey(pid)) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DifferenceWithinRangeScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DifferenceWithinRangeScrutinizer.java index e51043317..15faa1222 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DifferenceWithinRangeScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DifferenceWithinRangeScrutinizer.java @@ -53,9 +53,12 @@ public class DifferenceWithinRangeScrutinizer extends EditScrutinizer { public void scrutinize(ItemUpdate update) { Map propertyIdValueValueMap = new HashMap<>(); for (Statement statement : update.getAddedStatements()){ - PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId(); - Value value = statement.getClaim().getMainSnak().getValue(); - propertyIdValueValueMap.put(pid, value); + Snak mainSnak = statement.getClaim().getMainSnak(); + if (mainSnak instanceof ValueSnak) { + PropertyIdValue pid = mainSnak.getPropertyId(); + Value value = ((ValueSnak)mainSnak).getValue(); + propertyIdValueValueMap.put(pid, value); + } } for(PropertyIdValue propertyId : propertyIdValueValueMap.keySet()){ diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java index c83baac6c..2a54409f5 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/DistinctValuesScrutinizer.java @@ -26,8 +26,10 @@ package org.openrefine.wikidata.qa.scrutinizers; import org.openrefine.wikidata.qa.QAWarning; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.HashMap; import java.util.List; @@ -59,10 +61,11 @@ public class DistinctValuesScrutinizer extends StatementScrutinizer { @Override public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) { - PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId(); + Snak mainSnak = statement.getClaim().getMainSnak(); + PropertyIdValue pid = mainSnak.getPropertyId(); List statementList = _fetcher.getConstraintsByType(pid, distinctValuesConstraintQid); - if (!statementList.isEmpty()) { - Value mainSnakValue = statement.getClaim().getMainSnak().getValue(); + if (!statementList.isEmpty() && mainSnak instanceof ValueSnak) { + Value mainSnakValue = ((ValueSnak)mainSnak).getValue(); Map seen = _seenValues.get(pid); if (seen == null) { seen = new HashMap(); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java index ebab5e25c..ffe4e5dcf 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java @@ -32,6 +32,7 @@ import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.ArrayList; import java.util.List; @@ -155,7 +156,9 @@ public abstract class EditScrutinizer { for (SnakGroup group : groups) { if (group.getProperty().getId().equals(pid)) { for (Snak snak : group.getSnaks()) - results.add(snak.getValue()); + if (snak instanceof ValueSnak) { + results.add(((ValueSnak)snak).getValue()); + } } } return results; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java index bacdcc936..5dbc92a44 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/FormatScrutinizer.java @@ -33,6 +33,7 @@ import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.StringValue; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.HashMap; import java.util.HashSet; @@ -118,8 +119,8 @@ public class FormatScrutinizer extends SnakScrutinizer { @Override public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { - if (snak.getValue() instanceof StringValue) { - String value = ((StringValue) snak.getValue()).getString(); + if (snak instanceof ValueSnak && ((ValueSnak)snak).getValue() instanceof StringValue) { + String value = ((StringValue) ((ValueSnak)snak).getValue()).getString(); PropertyIdValue pid = snak.getPropertyId(); Set patterns = getPattern(pid); for (Pattern pattern : patterns) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java index 315324a89..665599aaf 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java @@ -27,9 +27,11 @@ import org.openrefine.wikidata.qa.QAWarning; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.HashMap; import java.util.HashSet; @@ -115,9 +117,13 @@ public class InverseConstraintScrutinizer extends StatementScrutinizer { return; // TODO support for deleted statements } - Value mainSnakValue = statement.getClaim().getMainSnak().getValue(); + Snak mainSnak = statement.getClaim().getMainSnak(); + if (! (mainSnak instanceof ValueSnak)) { + return; + } + Value mainSnakValue = ((ValueSnak)mainSnak).getValue(); if (mainSnakValue instanceof ItemIdValue) { - PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId(); + PropertyIdValue pid = mainSnak.getPropertyId(); PropertyIdValue inversePid = getInverseConstraint(pid); if (inversePid != null) { EntityIdValue targetEntityId = (EntityIdValue) mainSnakValue; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizer.java index 9598ba19e..2cbe2eaae 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizer.java @@ -7,6 +7,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.ArrayList; import java.util.HashMap; @@ -35,11 +36,14 @@ public class ItemRequiresScrutinizer extends EditScrutinizer { this.itemList = new ArrayList<>(); for(SnakGroup group : specs) { for (Snak snak : group.getSnaks()) { + if (! (snak instanceof ValueSnak)) { + continue; + } if (group.getProperty().getId().equals(itemRequiresPropertyPid)){ - pid = (PropertyIdValue) snak.getValue(); + pid = (PropertyIdValue) ((ValueSnak)snak).getValue(); } if (group.getProperty().getId().equals(itemOfPropertyConstraintPid)){ - this.itemList.add(snak.getValue()); + this.itemList.add(((ValueSnak)snak).getValue()); } } } @@ -60,10 +64,11 @@ public class ItemRequiresScrutinizer extends EditScrutinizer { public void scrutinize(ItemUpdate update) { Map> propertyIdValueValueMap = new HashMap<>(); for (Statement statement : update.getAddedStatements()) { - PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId(); - Value value = statement.getClaim().getMainSnak().getValue(); + Snak mainSnak = statement.getClaim().getMainSnak(); + PropertyIdValue pid = mainSnak.getPropertyId(); Set values; - if (value != null) { + if (mainSnak instanceof ValueSnak) { + Value value = ((ValueSnak)mainSnak).getValue(); if (propertyIdValueValueMap.containsKey(pid)) { values = propertyIdValueValueMap.get(pid); } else { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java index 8073b1881..474cdbb6a 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java @@ -9,6 +9,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.List; import java.util.Set; @@ -58,9 +59,9 @@ public class QuantityScrutinizer extends SnakScrutinizer { @Override public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { - if (snak.getValue() instanceof QuantityValue && added) { + if (snak instanceof ValueSnak && ((ValueSnak)snak).getValue() instanceof QuantityValue && added) { PropertyIdValue pid = snak.getPropertyId(); - QuantityValue value = (QuantityValue)snak.getValue(); + QuantityValue value = (QuantityValue)((ValueSnak)snak).getValue(); if(!_fetcher.getConstraintsByType(pid, noBoundsConstraintQid).isEmpty() && (value.getUpperBound() != null || value.getLowerBound() != null)) { QAWarning issue = new QAWarning(boundsDisallowedType, pid.getId(), QAWarning.Severity.IMPORTANT, 1); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java index 6a2dca6de..4721dfadd 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java @@ -7,6 +7,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.List; import java.util.Set; @@ -57,7 +58,10 @@ public class RestrictedValuesScrutinizer extends SnakScrutinizer { @Override public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { PropertyIdValue pid = snak.getPropertyId(); - Value value = snak.getValue(); + Value value = null; + if (snak instanceof ValueSnak) { + value = ((ValueSnak)snak).getValue(); + } List allowedValueConstraintDefinitions = _fetcher.getConstraintsByType(pid, allowedValuesConstraintQid); List disallowedValueConstraintDefinitions = _fetcher.getConstraintsByType(pid, disallowedValuesConstraintQid); Set allowedValues = null, disallowedValues = null; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java index b16a5014b..6bf67180e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SelfReferentialScrutinizer.java @@ -26,6 +26,7 @@ package org.openrefine.wikidata.qa.scrutinizers; import org.openrefine.wikidata.qa.QAWarning; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; /** * A scrutinizer that checks for self-referential statements. These statements @@ -40,7 +41,7 @@ public class SelfReferentialScrutinizer extends SnakScrutinizer { @Override public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { - if (entityId.equals(snak.getValue())) { + if (snak instanceof ValueSnak && entityId.equals(((ValueSnak)snak).getValue())) { QAWarning issue = new QAWarning(type, null, QAWarning.Severity.WARNING, 1); issue.setProperty("example_entity", entityId); addIssue(issue); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UseAsQualifierScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UseAsQualifierScrutinizer.java index f921269fe..56a12b49c 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UseAsQualifierScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UseAsQualifierScrutinizer.java @@ -8,6 +8,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import java.util.ArrayList; import java.util.HashMap; @@ -30,11 +31,14 @@ public class UseAsQualifierScrutinizer extends EditScrutinizer { this.itemList = new ArrayList<>(); for(SnakGroup group : specs) { for (Snak snak : group.getSnaks()) { + if (! (snak instanceof ValueSnak)) { + continue; + } if (group.getProperty().getId().equals(property)){ - pid = (PropertyIdValue) snak.getValue(); + pid = (PropertyIdValue) ((ValueSnak)snak).getValue(); } if (group.getProperty().getId().equals(itemOfPropertyConstraintPid)){ - this.itemList.add(snak.getValue()); + this.itemList.add(((ValueSnak)snak).getValue()); } } } @@ -58,15 +62,18 @@ public class UseAsQualifierScrutinizer extends EditScrutinizer { List qualifiersList = statement.getClaim().getQualifiers(); for(SnakGroup qualifier : qualifiersList) { - PropertyIdValue qualifierPid = Datamodel.makeWikidataPropertyIdValue(qualifier.getProperty().getId()); + PropertyIdValue qualifierPid = qualifier.getProperty(); List itemList; for (Snak snak : qualifier.getSnaks()) { + if (!(snak instanceof ValueSnak)) { + continue; + } if (qualifiersMap.containsKey(qualifierPid)){ itemList = qualifiersMap.get(qualifierPid); - }else { + } else { itemList = new ArrayList<>(); } - itemList.add(snak.getValue()); + itemList.add(((ValueSnak)snak).getValue()); qualifiersMap.put(qualifierPid, itemList); } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java index 1a70302a9..3919169b7 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizer.java @@ -28,6 +28,7 @@ import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.Snak; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; /** * A scrutinizer that inspects the values of snaks and terms @@ -56,7 +57,9 @@ public abstract class ValueScrutinizer extends SnakScrutinizer { @Override public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { - scrutinize(snak.getValue()); + if (snak instanceof ValueSnak) { + scrutinize(((ValueSnak)snak).getValue()); + } } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java index 373818535..516ec91cf 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java @@ -92,7 +92,7 @@ public class WbQuantityExpr implements WbExpression { if (getUnitExpr() != null) { ItemIdValue unit = getUnitExpr().evaluate(ctxt); - return Datamodel.makeQuantityValue(parsedAmount, lowerBound, upperBound, unit.getIri()); + return Datamodel.makeQuantityValue(parsedAmount, lowerBound, upperBound, unit); } return Datamodel.makeQuantityValue(parsedAmount, lowerBound, upperBound); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java b/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java index 703ec9c79..e4cbbe286 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/updates/scheduler/PointerExtractor.java @@ -40,6 +40,7 @@ import org.wikidata.wdtk.datamodel.interfaces.StringValue; import org.wikidata.wdtk.datamodel.interfaces.TimeValue; import org.wikidata.wdtk.datamodel.interfaces.UnsupportedValue; import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; /** @@ -101,7 +102,9 @@ public class PointerExtractor implements ValueVisitor> { public Set extractPointers(Snak snak) { Set result = new HashSet<>(); result.addAll(extractPointers(snak.getPropertyId())); - result.addAll(extractPointers(snak.getValue())); + if (snak instanceof ValueSnak) { + result.addAll(extractPointers(((ValueSnak)snak).getValue())); + } return result; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/utils/SnakUtils.java b/extensions/wikidata/src/org/openrefine/wikidata/utils/SnakUtils.java new file mode 100644 index 000000000..20673318d --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/utils/SnakUtils.java @@ -0,0 +1,42 @@ +package org.openrefine.wikidata.utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; + +public class SnakUtils { + + /** + * Groups snaks into a list of snak groups. + * The order of the first snaks in each group is preserved. + * + * @param snaks + * @return + */ + public static List groupSnaks(List snaks) { + Map> snakGroups = new HashMap<>(); + List propertyOrder = new ArrayList(); + for (Snak snak : snaks) { + List existingSnaks = snakGroups.get(snak.getPropertyId()); + if(existingSnaks == null) { + existingSnaks = new ArrayList(); + snakGroups.put(snak.getPropertyId(), existingSnaks); + propertyOrder.add(snak.getPropertyId()); + } + if (!existingSnaks.contains(snak)) { + existingSnaks.add(snak); + } + } + return propertyOrder.stream() + .map(pid -> Datamodel.makeSnakGroup(snakGroups.get(pid))) + .collect(Collectors.toList()); + } + +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSSnakPrinterTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSSnakPrinterTest.java new file mode 100644 index 000000000..4a79bb913 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSSnakPrinterTest.java @@ -0,0 +1,40 @@ +package org.openrefine.wikidata.exporters; + +import org.testng.Assert; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; + +public class QSSnakPrinterTest { + + private QSSnakPrinter refPrinter = new QSSnakPrinter(true); + private QSSnakPrinter mainPrinter = new QSSnakPrinter(false); + private PropertyIdValue pid = Datamodel.makeWikidataPropertyIdValue("P123"); + private ItemIdValue qid = Datamodel.makeWikidataItemIdValue("Q456"); + + @Test + public void testReferenceValueSnak() { + Snak snak = Datamodel.makeValueSnak(pid, qid); + Assert.assertEquals(snak.accept(refPrinter), "\tS123\tQ456"); + } + + @Test + public void testMainValueSnak() { + Snak snak = Datamodel.makeValueSnak(pid, qid); + Assert.assertEquals(snak.accept(mainPrinter), "\tP123\tQ456"); + } + + @Test + public void testReferenceNoValueSnak() { + Snak snak = Datamodel.makeNoValueSnak(pid); + Assert.assertEquals(snak.accept(refPrinter), "\tS123\tnovalue"); + } + + @Test + public void testMainSomeValueSnak() { + Snak snak = Datamodel.makeSomeValueSnak(pid); + Assert.assertEquals(snak.accept(mainPrinter), "\tP123\tsomevalue"); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java index 8612391f2..bd3552b10 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QSValuePrinterTest.java @@ -92,25 +92,25 @@ public class QSValuePrinterTest { @Test public void printSimpleQuantityValue() { - assertPrints("10.00", Datamodel.makeQuantityValue(new BigDecimal("10.00"), null, null, "1")); + assertPrints("10.00", Datamodel.makeQuantityValue(new BigDecimal("10.00"))); } @Test public void printQuantityValueWithUnit() { - assertPrints("10.00U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"), null, null, - "http://www.wikidata.org/entity/Q11573")); + assertPrints("10.00U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"), + Datamodel.makeWikidataItemIdValue("Q11573"))); } @Test public void printQuantityValueWithBounds() { assertPrints("10.00[9.0,11.05]", Datamodel.makeQuantityValue(new BigDecimal("10.00"), new BigDecimal("9.0"), - new BigDecimal("11.05"), "1")); + new BigDecimal("11.05"))); } @Test public void printFullQuantity() { assertPrints("10.00[9.0,11.05]U11573", Datamodel.makeQuantityValue(new BigDecimal("10.00"), - new BigDecimal("9.0"), new BigDecimal("11.05"), "http://www.wikidata.org/entity/Q11573")); + new BigDecimal("9.0"), new BigDecimal("11.05"), Datamodel.makeWikidataItemIdValue("Q11573"))); } // String values diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java index 8f74729ab..87bf4b461 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/exporters/QuickStatementsExporterTest.java @@ -133,6 +133,30 @@ public class QuickStatementsExporterTest extends WikidataRefineTest { assertEquals("Q1377\tP38\tQ865528\tP38\tQ1377\n", export(update)); } + + @Test + public void testSomeValue() + throws IOException { + PropertyIdValue pid = Datamodel.makeWikidataPropertyIdValue("P123"); + Claim claim = Datamodel.makeClaim(qid1, Datamodel.makeSomeValueSnak(pid), Collections.emptyList()); + Statement statement = Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, ""); + + ItemUpdate update = new ItemUpdateBuilder(qid1).addStatement(statement).build(); + + assertEquals("Q1377\tP123\tsomevalue\n", export(update)); + } + + @Test + public void testNoValue() + throws IOException { + PropertyIdValue pid = Datamodel.makeWikidataPropertyIdValue("P123"); + Claim claim = Datamodel.makeClaim(qid1, Datamodel.makeNoValueSnak(pid), Collections.emptyList()); + Statement statement = Datamodel.makeStatement(claim, Collections.emptyList(), StatementRank.NORMAL, ""); + + ItemUpdate update = new ItemUpdateBuilder(qid1).addStatement(statement).build(); + + assertEquals("Q1377\tP123\tnovalue\n", export(update)); + } /** * issue #2320 diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java index 8065cd09c..37b5a98b5 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java @@ -45,7 +45,7 @@ public class QuantityScrutinizerTest extends ValueScrutinizerTest{ new BigDecimal("1.545")); private QuantityValue wrongUnitValue = Datamodel.makeQuantityValue( - new BigDecimal("1.234"), "Q346721"); + new BigDecimal("1.234"), Datamodel.makeWikidataItemIdValue("Q346721")); private QuantityValue goodUnitValue = Datamodel.makeQuantityValue( new BigDecimal("1.234"), (ItemIdValue) allowedUnit); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbQuantityExprTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbQuantityExprTest.java index b5dcf2bbd..ff4431b61 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbQuantityExprTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbQuantityExprTest.java @@ -44,13 +44,13 @@ public class WbQuantityExprTest extends WbExpressionTest { public void testWithoutUnit() throws SkipSchemaExpressionException { setRow("4.00"); - evaluatesTo(Datamodel.makeQuantityValue(new BigDecimal("4.00"), null, null, "1"), exprWithoutUnit); + evaluatesTo(Datamodel.makeQuantityValue(new BigDecimal("4.00")), exprWithoutUnit); } @Test public void testOverflow() { setRow(14341937500d); - evaluatesTo(Datamodel.makeQuantityValue(new BigDecimal("14341937500"), null, null, "1"), exprWithoutUnit); + evaluatesTo(Datamodel.makeQuantityValue(new BigDecimal("14341937500")), exprWithoutUnit); } @Test @@ -64,7 +64,7 @@ public class WbQuantityExprTest extends WbExpressionTest { throws SkipSchemaExpressionException { setRow("56.094", recon("Q42")); evaluatesTo( - Datamodel.makeQuantityValue(new BigDecimal("56.094"), null, null, "http://www.wikidata.org/entity/Q42"), + Datamodel.makeQuantityValue(new BigDecimal("56.094"), Datamodel.makeWikidataItemIdValue("Q42")), exprWithUnit); } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java index ebb2b36b1..a1a16e6d1 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/updates/scheduler/PointerExtractorTest.java @@ -73,7 +73,7 @@ public class PointerExtractorTest { assertEmpty(Datamodel.makeMonolingualTextValue("srtu", "en")); assertEmpty(Datamodel.makeWikidataPropertyIdValue("P78")); assertEmpty(Datamodel.makeQuantityValue(new BigDecimal("898"))); - assertEmpty(Datamodel.makeQuantityValue(new BigDecimal("7.87"), "http://www.wikidata.org/entity/Q34")); + assertEmpty(Datamodel.makeQuantityValue(new BigDecimal("7.87"), Datamodel.makeWikidataItemIdValue("Q34"))); assertEmpty(Datamodel.makeTimeValue(1898, (byte) 2, (byte) 3, TimeValue.CM_GREGORIAN_PRO)); assertEmpty(mock(UnsupportedValue.class)); } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/utils/SnakUtilsTests.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/utils/SnakUtilsTests.java new file mode 100644 index 000000000..3a0ece370 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/utils/SnakUtilsTests.java @@ -0,0 +1,40 @@ +package org.openrefine.wikidata.utils; + +import java.util.Arrays; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; +import org.wikidata.wdtk.datamodel.interfaces.StringValue; + +public class SnakUtilsTests { + + PropertyIdValue pidA = Datamodel.makeWikidataPropertyIdValue("P1"); + PropertyIdValue pidB = Datamodel.makeWikidataPropertyIdValue("P2"); + PropertyIdValue pidC = Datamodel.makeWikidataPropertyIdValue("P3"); + StringValue value1 = Datamodel.makeStringValue("value1"); + StringValue value2 = Datamodel.makeStringValue("value2"); + + Snak snakA1 = Datamodel.makeValueSnak(pidA, value1); + Snak snakA2 = Datamodel.makeValueSnak(pidA, value2); + Snak snakB1 = Datamodel.makeValueSnak(pidB, value1); + Snak snakC1 = Datamodel.makeValueSnak(pidC, value1); + Snak snakC2 = Datamodel.makeValueSnak(pidC, value2); + + @Test + public void testGroupSnaks() { + List snakGroups = SnakUtils.groupSnaks(Arrays.asList(snakA2, snakB1, snakC1, snakA1, snakC2)); + + List expected = Arrays.asList( + Datamodel.makeSnakGroup(Arrays.asList(snakA2, snakA1)), + Datamodel.makeSnakGroup(Arrays.asList(snakB1)), + Datamodel.makeSnakGroup(Arrays.asList(snakC1, snakC2)) + ); + + Assert.assertEquals(snakGroups, expected); + } +}