From 9cb4a462775784c0866c1cd008a14ba4c38de543 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sun, 10 Jun 2018 08:17:24 +0100 Subject: [PATCH 1/9] Add support for quantity-related constraints in Wikidata QA --- .../wikidata/qa/ConstraintFetcher.java | 33 ++++++ .../openrefine/wikidata/qa/EditInspector.java | 2 + .../qa/WikidataConstraintFetcher.java | 82 +++++++++++++- .../qa/scrutinizers/QuantityScrutinizer.java | 68 ++++++++++++ .../wikidata/schema/WbItemVariable.java | 2 +- .../wikidata/qa/MockConstraintFetcher.java | 54 ++++++++++ .../scrutinizers/QuantityScrutinizerTest.java | 102 ++++++++++++++++++ .../qa/scrutinizers/ValueScrutinizerTest.java | 6 +- 8 files changed, 344 insertions(+), 5 deletions(-) create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java index 9933bb02d..a0f5402ee 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java @@ -25,7 +25,9 @@ package org.openrefine.wikidata.qa; import java.util.Set; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Value; /** * An object that fetches constraints about properties. @@ -53,6 +55,11 @@ public interface ConstraintFetcher { * @return the pid of the inverse property */ PropertyIdValue getInversePid(PropertyIdValue pid); + + /** + * Is this property supposed to be symmetric (its own inverse)? + */ + boolean isSymmetric(PropertyIdValue pid); /** * Is this property for values only? @@ -80,6 +87,18 @@ public interface ConstraintFetcher { * (null if any) */ Set mandatoryQualifiers(PropertyIdValue pid); + + /** + * Get the set of allowed values for this property (null if no such constraint). + * This set may contain null if one of the allowed values in novalue or somevalue. + */ + Set allowedValues(PropertyIdValue pid); + + /** + * Get the set of disallowed values for this property (null if no such constraint). + * This set may contain null if one of the allowed values in novalue or somevalue. + */ + Set disallowedValues(PropertyIdValue pid); /** * Is this property expected to have at most one value per item? @@ -91,4 +110,18 @@ public interface ConstraintFetcher { */ boolean hasDistinctValues(PropertyIdValue pid); + /** + * Can statements using this property have uncertainty bounds? + */ + boolean boundsAllowed(PropertyIdValue pid); + + /** + * Is this property expected to have integer values only? + */ + boolean integerValued(PropertyIdValue pid); + + /** + * Returns the allowed units for this property. If empty, no unit is allowed. If null, any unit is allowed. + */ + Set allowedUnits(PropertyIdValue pid); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java index 83bfa37e9..0a61f2f50 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java @@ -35,6 +35,7 @@ import org.openrefine.wikidata.qa.scrutinizers.InverseConstraintScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.NewItemScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.NoEditsMadeScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.QualifierCompatibilityScrutinizer; +import org.openrefine.wikidata.qa.scrutinizers.QuantityScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.RestrictedPositionScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.SelfReferentialScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.SingleValueScrutinizer; @@ -73,6 +74,7 @@ public class EditInspector { register(new DistinctValuesScrutinizer()); register(new NoEditsMadeScrutinizer()); register(new WhitespaceScrutinizer()); + register(new QuantityScrutinizer()); } /** diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java index 8b301a530..29f9337e4 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java @@ -24,6 +24,7 @@ package org.openrefine.wikidata.qa; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -31,6 +32,7 @@ import java.util.stream.Stream; import org.openrefine.wikidata.utils.EntityCache; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; import org.wikidata.wdtk.datamodel.interfaces.Snak; @@ -56,6 +58,7 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { public static String INVERSE_CONSTRAINT_QID = "Q21510855"; public static String INVERSE_PROPERTY_PID = "P2306"; + public static String SYMMETRIC_CONSTRAINT_QID = "Q21510862"; public static String USED_ONLY_AS_VALUES_CONSTRAINT_QID = "Q21528958"; @@ -68,9 +71,22 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { public static String MANDATORY_QUALIFIERS_CONSTRAINT_QID = "Q21510856"; public static String MANDATORY_QUALIFIERS_CONSTRAINT_PID = "P2306"; + + public static String ALLOWED_VALUES_CONSTRAINT_QID = "Q21510859"; + public static String ALLOWED_VALUES_CONSTRAINT_PID = "P2305"; + + public static String DISALLOWED_VALUES_CONSTRAINT_QID = "Q52558054"; + public static String DISALLOWED_VALUES_CONSTRAINT_PID = "P2305"; public static String SINGLE_VALUE_CONSTRAINT_QID = "Q19474404"; public static String DISTINCT_VALUES_CONSTRAINT_QID = "Q21502410"; + + public static String NO_BOUNDS_CONSTRAINT_QID = "Q51723761"; + public static String INTEGER_VALUED_CONSTRAINT_QID = "Q52848401"; + + public static String ALLOWED_UNITS_CONSTRAINT_QID = "Q21514353"; + public static String ALLOWED_UNITS_CONSTRAINT_PID = "P2305"; + // The following constraints still need to be implemented: @@ -122,7 +138,10 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { if (specs != null) { List properties = findValues(specs, ALLOWED_QUALIFIERS_CONSTRAINT_PID); - return properties.stream().map(e -> (PropertyIdValue) e).collect(Collectors.toSet()); + return properties.stream() + .filter(e -> e != null) + .map(e -> (PropertyIdValue) e) + .collect(Collectors.toSet()); } return null; } @@ -133,7 +152,10 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { if (specs != null) { List properties = findValues(specs, MANDATORY_QUALIFIERS_CONSTRAINT_PID); - return properties.stream().map(e -> (PropertyIdValue) e).collect(Collectors.toSet()); + return properties.stream() + .filter(e -> e != null) + .map(e -> (PropertyIdValue) e) + .collect(Collectors.toSet()); } return null; } @@ -147,6 +169,58 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { public boolean hasDistinctValues(PropertyIdValue pid) { return getSingleConstraint(pid, DISTINCT_VALUES_CONSTRAINT_QID) != null; } + + @Override + public boolean isSymmetric(PropertyIdValue pid) { + return getSingleConstraint(pid, SYMMETRIC_CONSTRAINT_QID) != null; + } + + @Override + public Set allowedValues(PropertyIdValue pid) { + List specs = getSingleConstraint(pid, ALLOWED_VALUES_CONSTRAINT_QID); + + if (specs != null) { + List properties = findValues(specs, ALLOWED_VALUES_CONSTRAINT_PID); + return properties.stream().collect(Collectors.toSet()); + } + return null; + } + + @Override + public Set disallowedValues(PropertyIdValue pid) { + List specs = getSingleConstraint(pid, DISALLOWED_VALUES_CONSTRAINT_QID); + + if (specs != null) { + List properties = findValues(specs, DISALLOWED_VALUES_CONSTRAINT_PID); + return properties.stream().collect(Collectors.toSet()); + } + return null; + } + + @Override + public boolean boundsAllowed(PropertyIdValue pid) { + return getSingleConstraint(pid, NO_BOUNDS_CONSTRAINT_QID) == null; + } + + @Override + public boolean integerValued(PropertyIdValue pid) { + return getSingleConstraint(pid, INTEGER_VALUED_CONSTRAINT_QID) != null; + } + + @Override + public Set allowedUnits(PropertyIdValue pid) { + List specs = getSingleConstraint(pid, ALLOWED_UNITS_CONSTRAINT_QID); + + if (specs != null) { + List properties = findValues(specs, ALLOWED_UNITS_CONSTRAINT_PID); + if (properties.contains(null)) { + return Collections.emptySet(); + } else { + return properties.stream().map(e -> (ItemIdValue) e).collect(Collectors.toSet()); + } + } + return null; + } /** * Returns a single constraint for a particular type and a property, or null if @@ -193,7 +267,9 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { PropertyDocument doc = (PropertyDocument) EntityCache.getEntityDocument(pid); StatementGroup group = doc.findStatementGroup(WIKIDATA_CONSTRAINT_PID); if (group != null) { - return group.getStatements(); + return group.getStatements().stream() + .filter(s -> s.getValue() != null && s.getValue() instanceof EntityIdValue) + .collect(Collectors.toList()); } else { return new ArrayList(); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java new file mode 100644 index 000000000..a32f3e2a2 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java @@ -0,0 +1,68 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import java.util.Set; +import java.util.stream.Collectors; + +import org.openrefine.wikidata.qa.QAWarning; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; + +/** + * Scrutinizer checking for units and bounds in quantities. + * + * @author Antonin Delpeuch + * + */ +public class QuantityScrutinizer extends SnakScrutinizer { + + public static final String boundsDisallowedType = "bounds-disallowed"; + public static final String integerConstraintType = "values-should-be-integers"; + public static final String invalidUnitType = "invalid-unit"; + public static final String noUnitProvidedType = "no-unit-provided"; + + @Override + public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { + if (QuantityValue.class.isInstance(snak.getValue()) && added) { + PropertyIdValue pid = snak.getPropertyId(); + QuantityValue value = (QuantityValue)snak.getValue(); + + if(!_fetcher.boundsAllowed(pid) && (value.getUpperBound() != null || value.getLowerBound() != null)) { + QAWarning issue = new QAWarning(boundsDisallowedType, pid.getId(), QAWarning.Severity.IMPORTANT, 1); + issue.setProperty("property_entity", pid); + issue.setProperty("example_value", value.getNumericValue().toString()); + issue.setProperty("example_item_entity", entityId); + addIssue(issue); + } + if(_fetcher.integerValued(pid) && value.getNumericValue().scale() > 0) { + QAWarning issue = new QAWarning(integerConstraintType, pid.getId(), QAWarning.Severity.IMPORTANT, 1); + issue.setProperty("property_entity", pid); + issue.setProperty("example_value", value.getNumericValue().toString()); + issue.setProperty("example_item_entity", entityId); + addIssue(issue); + } + Set allowedUnits = _fetcher.allowedUnits(pid); + String currentUnit = null; + if (value.getUnit() != null && value.getUnit() != "") { + currentUnit = value.getUnit(); + } + if(allowedUnits != null && + !allowedUnits.stream().map(u -> u != null ? u.getIri() : null) + .collect(Collectors.toSet()).contains(currentUnit)) { + String issueType = currentUnit == null ? noUnitProvidedType : invalidUnitType; + QAWarning issue = new QAWarning(issueType, pid.getId(), QAWarning.Severity.IMPORTANT, 1); + issue.setProperty("property_entity", pid); + issue.setProperty("example_value", value.getNumericValue().toString()); + issue.setProperty("example_item_entity", entityId); + if (currentUnit != null) { + issue.setProperty("unit_entity", currentUnit); + } + addIssue(issue); + } + } + } + +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java index 00c634580..af7d2fd72 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java @@ -63,7 +63,7 @@ public class WbItemVariable extends WbVariableExpr { throws SkipSchemaExpressionException { if (cell.recon != null && (Judgment.Matched.equals(cell.recon.judgment) || Judgment.New.equals(cell.recon.judgment))) { - if (!cell.recon.identifierSpace.equals(Datamodel.SITE_WIKIDATA)) { + if (cell.recon.identifierSpace == null || !cell.recon.identifierSpace.equals(Datamodel.SITE_WIKIDATA)) { QAWarning warning = new QAWarning("invalid-identifier-space", null, QAWarning.Severity.INFO, 1); warning.setProperty("example_cell", cell.value.toString()); ctxt.addWarning(warning); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java index b13c7ccff..ee2824611 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java @@ -29,18 +29,32 @@ import java.util.Set; import java.util.stream.Collectors; import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Value; public class MockConstraintFetcher implements ConstraintFetcher { public static PropertyIdValue pidWithInverse = Datamodel.makeWikidataPropertyIdValue("P350"); public static PropertyIdValue inversePid = Datamodel.makeWikidataPropertyIdValue("P57"); + public static PropertyIdValue symmetricPid = Datamodel.makeWikidataPropertyIdValue("P783"); public static PropertyIdValue allowedQualifierPid = Datamodel.makeWikidataPropertyIdValue("P34"); public static PropertyIdValue mandatoryQualifierPid = Datamodel.makeWikidataPropertyIdValue("P97"); public static PropertyIdValue mainSnakPid = Datamodel.makeWikidataPropertyIdValue("P1234"); public static PropertyIdValue qualifierPid = Datamodel.makeWikidataPropertyIdValue("P987"); public static PropertyIdValue referencePid = Datamodel.makeWikidataPropertyIdValue("P384"); + + public static PropertyIdValue restrictedValuesPid = Datamodel.makeWikidataPropertyIdValue("P8121"); + public static ItemIdValue particularValue1 = Datamodel.makeWikidataItemIdValue("Q389"); + public static ItemIdValue particularValue2 = Datamodel.makeWikidataItemIdValue("Q378"); + + public static PropertyIdValue allowedUnitsPid = Datamodel.makeWikidataPropertyIdValue("P34787"); + public static ItemIdValue allowedUnit = Datamodel.makeWikidataItemIdValue("Q7887"); + public static PropertyIdValue noUnitsPid = Datamodel.makeWikidataPropertyIdValue("P334211"); + + public static PropertyIdValue noBoundsPid = Datamodel.makeWikidataPropertyIdValue("P8932"); + public static PropertyIdValue integerPid = Datamodel.makeWikidataPropertyIdValue("P389"); @Override public String getFormatRegex(PropertyIdValue pid) { @@ -94,4 +108,44 @@ public class MockConstraintFetcher implements ConstraintFetcher { return true; } + @Override + public boolean isSymmetric(PropertyIdValue pid) { + return pid.equals(symmetricPid); + } + + @Override + public Set allowedValues(PropertyIdValue pid) { + if (restrictedValuesPid.equals(pid)) { + return Collections.singleton(particularValue1); + } + return null; + } + + @Override + public Set disallowedValues(PropertyIdValue pid) { + if (restrictedValuesPid.equals(pid)) { + return Collections.singleton(particularValue2); + } + return null; + } + + @Override + public boolean boundsAllowed(PropertyIdValue pid) { + return !noBoundsPid.equals(pid); + } + + @Override + public boolean integerValued(PropertyIdValue pid) { + return integerPid.equals(pid); + } + + @Override + public Set allowedUnits(PropertyIdValue pid) { + if(allowedUnitsPid.equals(pid)) { + return Collections.singleton(allowedUnit); + } else if(noUnitsPid.equals(pid)) { + return Collections.singleton(null); + } + return null; + } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java new file mode 100644 index 000000000..f0dbb0acf --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizerTest.java @@ -0,0 +1,102 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import java.math.BigDecimal; + +import org.openrefine.wikidata.qa.MockConstraintFetcher; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; + +public class QuantityScrutinizerTest extends ValueScrutinizerTest{ + + private QuantityValue exactValue = Datamodel.makeQuantityValue( + new BigDecimal("1.234")); + + private QuantityValue integerValue = Datamodel.makeQuantityValue( + new BigDecimal("132")); + + private QuantityValue trailingZeros = Datamodel.makeQuantityValue( + new BigDecimal("132.00")); + + private QuantityValue valueWithBounds = Datamodel.makeQuantityValue( + new BigDecimal("1.234"), + new BigDecimal("1.200"), + new BigDecimal("1.545")); + + private QuantityValue wrongUnitValue = Datamodel.makeQuantityValue( + new BigDecimal("1.234"), "Q346721"); + + private QuantityValue goodUnitValue = Datamodel.makeQuantityValue( + new BigDecimal("1.234"), MockConstraintFetcher.allowedUnit.getIri()); + + @Override + public EditScrutinizer getScrutinizer() { + return new QuantityScrutinizer(); + } + + @Test + public void testBoundsAllowed() { + scrutinize(valueWithBounds); + assertNoWarningRaised(); + } + + @Test + public void testBoundsDisallowed() { + scrutinize(MockConstraintFetcher.noBoundsPid, valueWithBounds); + assertWarningsRaised(QuantityScrutinizer.boundsDisallowedType); + } + + @Test + public void testFractionalAllowed() { + scrutinize(exactValue); + assertNoWarningRaised(); + } + + @Test + public void testFractionalDisallowed() { + scrutinize(MockConstraintFetcher.integerPid, exactValue); + assertWarningsRaised(QuantityScrutinizer.integerConstraintType); + } + + @Test + public void testTrailingZeros() { + scrutinize(MockConstraintFetcher.integerPid, trailingZeros); + assertWarningsRaised(QuantityScrutinizer.integerConstraintType); + } + + @Test + public void testInteger() { + scrutinize(MockConstraintFetcher.integerPid, integerValue); + assertNoWarningRaised(); + } + + @Test + public void testUnitReqired() { + scrutinize(MockConstraintFetcher.allowedUnitsPid, integerValue); + assertWarningsRaised(QuantityScrutinizer.noUnitProvidedType); + } + + @Test + public void testWrongUnit() { + scrutinize(MockConstraintFetcher.allowedUnitsPid, wrongUnitValue); + assertWarningsRaised(QuantityScrutinizer.invalidUnitType); + } + + @Test + public void testGoodUnit() { + scrutinize(MockConstraintFetcher.allowedUnitsPid, goodUnitValue); + assertNoWarningRaised(); + } + + @Test + public void testUnitForbidden() { + scrutinize(MockConstraintFetcher.noUnitsPid, goodUnitValue); + assertWarningsRaised(QuantityScrutinizer.invalidUnitType); + } + + @Test + public void testNoUnit() { + scrutinize(MockConstraintFetcher.noUnitsPid, integerValue); + assertNoWarningRaised(); + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizerTest.java index 21ec3fdb3..bba04c5c6 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizerTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ValueScrutinizerTest.java @@ -35,7 +35,11 @@ public abstract class ValueScrutinizerTest extends SnakScrutinizerTest { public static final PropertyIdValue defaultPid = Datamodel.makeWikidataPropertyIdValue("P328"); public void scrutinize(Value value) { - scrutinize(Datamodel.makeValueSnak(defaultPid, value)); + scrutinize(defaultPid, value); + } + + public void scrutinize(PropertyIdValue pid, Value value) { + scrutinize(Datamodel.makeValueSnak(pid, value)); } public void scrutinizeLabel(MonolingualTextValue text) { From 3c181c9070063515e8b8c3884e9ae54155e029ad Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 9 Jun 2018 12:31:55 +0100 Subject: [PATCH 2/9] Add support for symmetric constraint --- .../qa/scrutinizers/InverseConstraintScrutinizer.java | 3 +++ .../qa/scrutinizers/InverseConstaintScrutinizerTest.java | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java index 6a1a7b619..88bc2c3ae 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstraintScrutinizer.java @@ -59,6 +59,9 @@ public class InverseConstraintScrutinizer extends StatementScrutinizer { return _inverse.get(pid); } else { PropertyIdValue inversePid = _fetcher.getInversePid(pid); + if (inversePid == null && _fetcher.isSymmetric(pid)) { + inversePid = pid; + } _inverse.put(pid, inversePid); _statements.put(pid, new HashMap>()); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstaintScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstaintScrutinizerTest.java index a44bddbd6..a4313d9f5 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstaintScrutinizerTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/InverseConstaintScrutinizerTest.java @@ -37,6 +37,7 @@ public class InverseConstaintScrutinizerTest extends StatementScrutinizerTest { private ItemIdValue idB = TestingData.newIdB; private PropertyIdValue pidWithInverse = MockConstraintFetcher.pidWithInverse; private PropertyIdValue inversePid = MockConstraintFetcher.inversePid; + private PropertyIdValue symmetricPid = MockConstraintFetcher.symmetricPid; @Override public EditScrutinizer getScrutinizer() { @@ -50,6 +51,14 @@ public class InverseConstaintScrutinizerTest extends StatementScrutinizerTest { scrutinize(update); assertWarningsRaised(InverseConstraintScrutinizer.type); } + + @Test + public void testSymmetric() { + ItemUpdate update = new ItemUpdateBuilder(idA) + .addStatement(TestingData.generateStatement(idA, symmetricPid, idB)).build(); + scrutinize(update); + assertWarningsRaised(InverseConstraintScrutinizer.type); + } @Test public void testNoSymmetricClosure() { From 48c2b90a936618aed3e4fb6c43bed16225855c3f Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 9 Jun 2018 13:08:52 +0100 Subject: [PATCH 3/9] Add constraint checks for one-of and none-of constraints --- .../openrefine/wikidata/qa/EditInspector.java | 2 + .../RestrictedValuesScrutinizer.java | 32 +++++++++ .../wikidata/qa/MockConstraintFetcher.java | 15 ++-- .../RestrictedValuesScrutinizerTest.java | 68 +++++++++++++++++++ 4 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java index 0a61f2f50..08493e19d 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java @@ -37,6 +37,7 @@ import org.openrefine.wikidata.qa.scrutinizers.NoEditsMadeScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.QualifierCompatibilityScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.QuantityScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.RestrictedPositionScrutinizer; +import org.openrefine.wikidata.qa.scrutinizers.RestrictedValuesScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.SelfReferentialScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.SingleValueScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.UnsourcedScrutinizer; @@ -75,6 +76,7 @@ public class EditInspector { register(new NoEditsMadeScrutinizer()); register(new WhitespaceScrutinizer()); register(new QuantityScrutinizer()); + register(new RestrictedValuesScrutinizer()); } /** diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java new file mode 100644 index 000000000..5aed7b8b4 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizer.java @@ -0,0 +1,32 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import java.util.Set; + +import org.openrefine.wikidata.qa.QAWarning; +import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.Value; + + +public class RestrictedValuesScrutinizer extends SnakScrutinizer { + + public static String type = "forbidden-value"; + + @Override + public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { + PropertyIdValue pid = snak.getPropertyId(); + Value value = snak.getValue(); + + Set allowedValues = _fetcher.allowedValues(pid); + Set disallowedValues = _fetcher.disallowedValues(pid); + if((allowedValues != null && !allowedValues.contains(value)) || + (disallowedValues != null && disallowedValues.contains(value))) { + QAWarning issue = new QAWarning(type, pid.getId(), QAWarning.Severity.IMPORTANT, 1); + issue.setProperty("property_entity", pid); + issue.setProperty("example_value_entity", value); + issue.setProperty("example_subject_entity", entityId); + addIssue(issue); + } + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java index ee2824611..c04e1f91f 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java @@ -45,9 +45,10 @@ public class MockConstraintFetcher implements ConstraintFetcher { public static PropertyIdValue qualifierPid = Datamodel.makeWikidataPropertyIdValue("P987"); public static PropertyIdValue referencePid = Datamodel.makeWikidataPropertyIdValue("P384"); - public static PropertyIdValue restrictedValuesPid = Datamodel.makeWikidataPropertyIdValue("P8121"); - public static ItemIdValue particularValue1 = Datamodel.makeWikidataItemIdValue("Q389"); - public static ItemIdValue particularValue2 = Datamodel.makeWikidataItemIdValue("Q378"); + public static PropertyIdValue allowedValuesPid = Datamodel.makeWikidataPropertyIdValue("P8121"); + public static ItemIdValue allowedValueQid = Datamodel.makeWikidataItemIdValue("Q389"); + public static PropertyIdValue forbiddenValuesPid = Datamodel.makeWikidataPropertyIdValue("P8141"); + public static ItemIdValue forbiddenValueQid = Datamodel.makeWikidataItemIdValue("Q378"); public static PropertyIdValue allowedUnitsPid = Datamodel.makeWikidataPropertyIdValue("P34787"); public static ItemIdValue allowedUnit = Datamodel.makeWikidataItemIdValue("Q7887"); @@ -115,16 +116,16 @@ public class MockConstraintFetcher implements ConstraintFetcher { @Override public Set allowedValues(PropertyIdValue pid) { - if (restrictedValuesPid.equals(pid)) { - return Collections.singleton(particularValue1); + if (allowedValuesPid.equals(pid)) { + return Arrays.asList(allowedValueQid, null).stream().collect(Collectors.toSet()); } return null; } @Override public Set disallowedValues(PropertyIdValue pid) { - if (restrictedValuesPid.equals(pid)) { - return Collections.singleton(particularValue2); + if (forbiddenValuesPid.equals(pid)) { + return Collections.singleton(forbiddenValueQid); } return null; } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java new file mode 100644 index 000000000..1bdb0d979 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java @@ -0,0 +1,68 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import org.openrefine.wikidata.qa.MockConstraintFetcher; +import org.openrefine.wikidata.testing.TestingData; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; + +public class RestrictedValuesScrutinizerTest extends SnakScrutinizerTest { + + private ItemIdValue qid = Datamodel.makeWikidataItemIdValue("Q3487"); + + @Override + public EditScrutinizer getScrutinizer() { + return new RestrictedValuesScrutinizer(); + } + + @Test + public void testNoConstraint() { + scrutinize(TestingData.generateStatement(qid, + Datamodel.makeWikidataPropertyIdValue("P28732"), + qid)); + assertNoWarningRaised(); + } + + @Test + public void testAllowedValue() { + scrutinize(TestingData.generateStatement(qid, + MockConstraintFetcher.allowedValuesPid, + MockConstraintFetcher.allowedValueQid)); + assertNoWarningRaised(); + } + + @Test + public void testAllowedValueFailing() { + scrutinize(TestingData.generateStatement(qid, + MockConstraintFetcher.allowedValuesPid, + qid)); + assertWarningsRaised(RestrictedValuesScrutinizer.type); + } + + @Test + public void testDisallowedValue() { + scrutinize(TestingData.generateStatement(qid, + MockConstraintFetcher.forbiddenValuesPid, + qid)); + assertNoWarningRaised(); + } + + @Test + public void testDisallowedValueFailing() { + scrutinize(TestingData.generateStatement(qid, + MockConstraintFetcher.forbiddenValuesPid, + MockConstraintFetcher.forbiddenValueQid)); + assertWarningsRaised(RestrictedValuesScrutinizer.type); + } + + @Test + public void testNoValueAllowed() { + + } + + @Test + public void testNoValueDisallowed() { + + } + +} From 4bb3505aa5286a802756dcd6c17eda85722f29c6 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 9 Jun 2018 13:53:51 +0100 Subject: [PATCH 4/9] Add English descriptions of new issues --- .../wikidata/module/langs/translation-en.json | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/extensions/wikidata/module/langs/translation-en.json b/extensions/wikidata/module/langs/translation-en.json index e3a9cf53c..95ea5cb64 100644 --- a/extensions/wikidata/module/langs/translation-en.json +++ b/extensions/wikidata/module/langs/translation-en.json @@ -216,6 +216,26 @@ "ignored-coordinates": { "title": "Invalid geographic coordinates.", "body": "Some coordinates are incorrectly formatted, such as {example_value}. See the allowed formats." + }, + "forbidden-value": { + "title": "Invalid values for {property_entity}", + "body": "Items such as {example_value_entity} added on {example_subject_entity} are not allowed as values for {property_entity}." + }, + "bounds-disallowed": { + "title": "Quantity bounds supplied for {property_entity}", + "body": "Values are not expected to have uncertainty bounds, but {example_value} added on {example_item_entity} has some. See the manual to learn how to fix their format." + }, + "values-should-be-integers": { + "title": "Non-integer values of {property_entity}", + "body": "Values are expected to be integers, but {example_value} added on {example_item_entity} has a fractional part. See the manual to learn how to fix their format." + }, + "invalid-unit": { + "title": "{property_entity} with invalid units", + "body": "Units such as {unit_entity} used on {example_item_entity} are invalid for {property_entity}." + }, + "no-unit-provided": { + "title": "Unit missing for {property_entity}", + "body": "Values such as {example_value} on {example_item_entity} are expected to have units." } } } From ddbe4fe2a2c5b003ad7d293c97358443122c17f2 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 9 Jun 2018 13:54:16 +0100 Subject: [PATCH 5/9] Fix unit checking with novalue claims --- .../openrefine/wikidata/qa/WikidataConstraintFetcher.java | 6 +----- .../wikidata/qa/scrutinizers/QuantityScrutinizer.java | 4 +++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java index 29f9337e4..20036b26f 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java @@ -213,11 +213,7 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { if (specs != null) { List properties = findValues(specs, ALLOWED_UNITS_CONSTRAINT_PID); - if (properties.contains(null)) { - return Collections.emptySet(); - } else { - return properties.stream().map(e -> (ItemIdValue) e).collect(Collectors.toSet()); - } + return properties.stream().map(e -> e == null ? null : (ItemIdValue) e).collect(Collectors.toSet()); } return null; } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java index a32f3e2a2..d1f1dc9a9 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java @@ -58,7 +58,9 @@ public class QuantityScrutinizer extends SnakScrutinizer { issue.setProperty("example_value", value.getNumericValue().toString()); issue.setProperty("example_item_entity", entityId); if (currentUnit != null) { - issue.setProperty("unit_entity", currentUnit); + issue.setProperty("unit_entity", + // this is a hack but it will not be needed anymore in the upcoming version of Wikidata-Toolkit + Datamodel.makeWikidataItemIdValue(currentUnit.substring(currentUnit.indexOf("Q")))); } addIssue(issue); } From e2ae09f5bee6e2c01e76d216f6d209998908405b Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sun, 10 Jun 2018 08:14:24 +0100 Subject: [PATCH 6/9] Add support for Wikidata single best value constraint --- .../src/org/openrefine/wikidata/qa/ConstraintFetcher.java | 5 +++++ .../openrefine/wikidata/qa/WikidataConstraintFetcher.java | 6 ++++++ .../wikidata/qa/scrutinizers/SingleValueScrutinizer.java | 5 ++++- .../org/openrefine/wikidata/qa/MockConstraintFetcher.java | 5 +++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java index a0f5402ee..4303c1bf6 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java @@ -104,6 +104,11 @@ public interface ConstraintFetcher { * Is this property expected to have at most one value per item? */ boolean hasSingleValue(PropertyIdValue pid); + + /** + * Is this property expected to have a single best value only? + */ + boolean hasSingleBestValue(PropertyIdValue pid); /** * Is this property expected to have distinct values? diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java index 20036b26f..50ee412cc 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java @@ -79,6 +79,7 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { public static String DISALLOWED_VALUES_CONSTRAINT_PID = "P2305"; public static String SINGLE_VALUE_CONSTRAINT_QID = "Q19474404"; + public static String SINGLE_BEST_VALUE_CONSTRAINT_QID = "Q52060874"; public static String DISTINCT_VALUES_CONSTRAINT_QID = "Q21502410"; public static String NO_BOUNDS_CONSTRAINT_QID = "Q51723761"; @@ -164,6 +165,11 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { public boolean hasSingleValue(PropertyIdValue pid) { return getSingleConstraint(pid, SINGLE_VALUE_CONSTRAINT_QID) != null; } + + @Override + public boolean hasSingleBestValue(PropertyIdValue pid) { + return getSingleConstraint(pid, SINGLE_BEST_VALUE_CONSTRAINT_QID) != null; + } @Override public boolean hasDistinctValues(PropertyIdValue pid) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java index bdd7bb3f4..0609205d1 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java @@ -35,6 +35,9 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement; * For now this scrutinizer only checks for uniqueness at the item level (it * ignores qualifiers and references). * + * Given that all ranks are currently set to Normal, this also checks for + * single best values. + * * @author Antonin Delpeuch * */ @@ -54,7 +57,7 @@ public class SingleValueScrutinizer extends EditScrutinizer { issue.setProperty("property_entity", pid); issue.setProperty("example_entity", update.getItemId()); addIssue(issue); - } else if (_fetcher.hasSingleValue(pid)) { + } else if (_fetcher.hasSingleValue(pid) || _fetcher.hasSingleBestValue(pid)) { seenSingleProperties.add(pid); } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java index c04e1f91f..4e3f90879 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java @@ -103,6 +103,11 @@ public class MockConstraintFetcher implements ConstraintFetcher { public boolean hasSingleValue(PropertyIdValue pid) { return true; } + + @Override + public boolean hasSingleBestValue(PropertyIdValue pid) { + return false; + } @Override public boolean hasDistinctValues(PropertyIdValue pid) { From 1910833ad81fe77796fae1c70ae9d0f0e6ece04c Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sun, 10 Jun 2018 09:24:22 +0100 Subject: [PATCH 7/9] Add entity type constraint check --- .../wikidata/module/langs/translation-en.json | 4 +++ .../wikidata/qa/ConstraintFetcher.java | 5 ++++ .../openrefine/wikidata/qa/EditInspector.java | 2 ++ .../qa/WikidataConstraintFetcher.java | 15 ++++++++++ .../scrutinizers/EntityTypeScrutinizer.java | 22 ++++++++++++++ .../wikidata/qa/MockConstraintFetcher.java | 7 +++++ .../EntityTypeScrutinizerTest.java | 29 +++++++++++++++++++ 7 files changed, 84 insertions(+) create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizer.java create mode 100644 extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizerTest.java diff --git a/extensions/wikidata/module/langs/translation-en.json b/extensions/wikidata/module/langs/translation-en.json index 95ea5cb64..8ac957f43 100644 --- a/extensions/wikidata/module/langs/translation-en.json +++ b/extensions/wikidata/module/langs/translation-en.json @@ -236,6 +236,10 @@ "no-unit-provided": { "title": "Unit missing for {property_entity}", "body": "Values such as {example_value} on {example_item_entity} are expected to have units." + }, + "invalid-entity-type": { + "title": "{property_entity} used on items", + "body": "Uses of {property_entity} on items such as {example_entity} are invalid." } } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java index 4303c1bf6..3e5e11bdd 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java @@ -129,4 +129,9 @@ public interface ConstraintFetcher { * Returns the allowed units for this property. If empty, no unit is allowed. If null, any unit is allowed. */ Set allowedUnits(PropertyIdValue pid); + + /** + * Can this property be used on items? + */ + boolean usableOnItems(PropertyIdValue pid); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java index 08493e19d..86b876b8c 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java @@ -30,6 +30,7 @@ import java.util.stream.Collectors; import org.openrefine.wikidata.qa.scrutinizers.DistinctValuesScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.EditScrutinizer; +import org.openrefine.wikidata.qa.scrutinizers.EntityTypeScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.FormatScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.InverseConstraintScrutinizer; import org.openrefine.wikidata.qa.scrutinizers.NewItemScrutinizer; @@ -77,6 +78,7 @@ public class EditInspector { register(new WhitespaceScrutinizer()); register(new QuantityScrutinizer()); register(new RestrictedValuesScrutinizer()); + register(new EntityTypeScrutinizer()); } /** diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java index 50ee412cc..8ed778c75 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java @@ -31,6 +31,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.openrefine.wikidata.utils.EntityCache; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument; @@ -88,6 +89,10 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { public static String ALLOWED_UNITS_CONSTRAINT_QID = "Q21514353"; public static String ALLOWED_UNITS_CONSTRAINT_PID = "P2305"; + public static String ALLOWED_ENTITY_TYPES_QID = "Q52004125"; + public static String ALLOWED_ITEM_TYPE_QID = "Q29934200"; + public static String ALLOWED_ENTITY_TYPES_PID = "P2305"; + // The following constraints still need to be implemented: @@ -223,6 +228,16 @@ public class WikidataConstraintFetcher implements ConstraintFetcher { } return null; } + + @Override + public boolean usableOnItems(PropertyIdValue pid) { + List constraint = getSingleConstraint(pid, ALLOWED_ENTITY_TYPES_QID); + if (constraint != null) { + return findValues(constraint, ALLOWED_ENTITY_TYPES_PID).contains( + Datamodel.makeWikidataItemIdValue(ALLOWED_ITEM_TYPE_QID)); + } + return true; + } /** * Returns a single constraint for a particular type and a property, or null if diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizer.java new file mode 100644 index 000000000..af822dc28 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizer.java @@ -0,0 +1,22 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import org.openrefine.wikidata.qa.QAWarning; +import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; + + +public class EntityTypeScrutinizer extends SnakScrutinizer { + + public final static String type = "invalid-entity-type"; + + @Override + public void scrutinize(Snak snak, EntityIdValue entityId, boolean added) { + PropertyIdValue pid = snak.getPropertyId(); + if(!_fetcher.usableOnItems(pid)) { + QAWarning issue = new QAWarning(type, null, QAWarning.Severity.WARNING, 1); + issue.setProperty("example_entity", entityId); + addIssue(issue); + } + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java index 4e3f90879..330d96f3a 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/MockConstraintFetcher.java @@ -56,6 +56,8 @@ public class MockConstraintFetcher implements ConstraintFetcher { public static PropertyIdValue noBoundsPid = Datamodel.makeWikidataPropertyIdValue("P8932"); public static PropertyIdValue integerPid = Datamodel.makeWikidataPropertyIdValue("P389"); + + public static PropertyIdValue propertyOnlyPid = Datamodel.makeWikidataPropertyIdValue("P372"); @Override public String getFormatRegex(PropertyIdValue pid) { @@ -154,4 +156,9 @@ public class MockConstraintFetcher implements ConstraintFetcher { } return null; } + + @Override + public boolean usableOnItems(PropertyIdValue pid) { + return !propertyOnlyPid.equals(pid); + } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizerTest.java new file mode 100644 index 000000000..baaf6f44e --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/EntityTypeScrutinizerTest.java @@ -0,0 +1,29 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import org.openrefine.wikidata.qa.MockConstraintFetcher; +import org.openrefine.wikidata.testing.TestingData; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; + +public class EntityTypeScrutinizerTest extends StatementScrutinizerTest { + + private static ItemIdValue qid = Datamodel.makeWikidataItemIdValue("Q343"); + + @Override + public EditScrutinizer getScrutinizer() { + return new EntityTypeScrutinizer(); + } + + @Test + public void testAllowed() { + scrutinize(TestingData.generateStatement(qid, qid)); + assertNoWarningRaised(); + } + + @Test + public void testDisallowed() { + scrutinize(TestingData.generateStatement(qid, MockConstraintFetcher.propertyOnlyPid, qid)); + assertWarningsRaised(EntityTypeScrutinizer.type); + } +} From ab3adc0329f44d7c4aed73d04ba4f75ff424dee5 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sun, 10 Jun 2018 09:30:06 +0100 Subject: [PATCH 8/9] Remove empty tests --- .../scrutinizers/RestrictedValuesScrutinizerTest.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java index 1bdb0d979..5ba9b58eb 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/RestrictedValuesScrutinizerTest.java @@ -54,15 +54,5 @@ public class RestrictedValuesScrutinizerTest extends SnakScrutinizerTest { MockConstraintFetcher.forbiddenValueQid)); assertWarningsRaised(RestrictedValuesScrutinizer.type); } - - @Test - public void testNoValueAllowed() { - - } - - @Test - public void testNoValueDisallowed() { - - } } From b87a52195c0d1a63abdf17099c1d8c600d8c6fd2 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Mon, 11 Jun 2018 09:19:34 +0100 Subject: [PATCH 9/9] Cleanup for Codacy --- .../org/openrefine/wikidata/qa/WikidataConstraintFetcher.java | 1 - .../wikidata/qa/scrutinizers/QuantityScrutinizer.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java index 8ed778c75..9d2c3a963 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/WikidataConstraintFetcher.java @@ -24,7 +24,6 @@ package org.openrefine.wikidata.qa; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java index d1f1dc9a9..ea711fe56 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/QuantityScrutinizer.java @@ -46,7 +46,7 @@ public class QuantityScrutinizer extends SnakScrutinizer { } Set allowedUnits = _fetcher.allowedUnits(pid); String currentUnit = null; - if (value.getUnit() != null && value.getUnit() != "") { + if (value.getUnit() != null && !value.getUnit().equals("")) { currentUnit = value.getUnit(); } if(allowedUnits != null &&