diff --git a/extensions/wikidata/module/langs/translation-en.json b/extensions/wikidata/module/langs/translation-en.json index f4e9c9aa2..bea905f03 100644 --- a/extensions/wikidata/module/langs/translation-en.json +++ b/extensions/wikidata/module/langs/translation-en.json @@ -148,6 +148,14 @@ "warnings-messages/multi-valued-property-is-required-for-new-item/body": "This property is expected to have more than one statement on each item but it has single statement, for instance on {example_entity}.", "warnings-messages/multi-valued-property-is-required-for-existing-item/title": "{property_entity} should have more than one statement on existing items.", "warnings-messages/multi-valued-property-is-required-for-existing-item/body": "This property is expected to have more than one statement on each item but it has single statement, for instance on {example_entity}. If the item already has statements with this property in Wikidata, then this warning can be ignored.", + "warnings-messages/new-item-requires-property-to-have-certain-values/title": "{property_entity} requires property {added_property_entity} to have certain values.", + "warnings-messages/new-item-requires-property-to-have-certain-values/body": "Property {added_property_entity} doesn't have appropriate values such as on item {example_entity}.", + "warnings-messages/new-item-requires-certain-other-statement/title": "{property_entity} requires statement with property {added_property_entity}.", + "warnings-messages/new-item-requires-certain-other-statement/body": "This property is expected to have another statement with property {added_property_entity} such as on item {example_entity}.", + "warnings-messages/existing-item-requires-property-to-have-certain-values/title": "{property_entity} requires property {added_property_entity} to have certain values.", + "warnings-messages/existing-item-requires-property-to-have-certain-values/body": "Property {added_property_entity} doesn't have appropriate values such as on item {example_entity}. If the item already has statements with property {added_property_entity} and with appropriate values in Wikidata, then this warning can be ignored.", + "warnings-messages/existing-item-requires-certain-other-statement/title": "{property_entity} requires statement with property {added_property_entity}.", + "warnings-messages/existing-item-requires-certain-other-statement/body": "This property is expected to have another statement with property {added_property_entity} such as on item {example_entity}. If the item already has statements with property {added_property_entity} in Wikidata, then this warning can be ignored.", "warnings-messages/ignored-qualifiers/title": "Some qualifiers were ignored.", "warnings-messages/ignored-qualifiers/body": "Qualifier values could not be parsed, so they will not be added to the corresponding statements.", "warnings-messages/ignored-references/title": "Some references were ignored.", diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java index d9b86e575..a50522959 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java @@ -72,6 +72,7 @@ public class EditInspector { register(new MultiValueScrutinizer()); register(new DifferenceWithinRangeScrutinizer()); register(new ConflictsWithScrutinizer()); + register(new ItemRequiresScrutinizer()); } /** diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizer.java new file mode 100644 index 000000000..391c21d5e --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizer.java @@ -0,0 +1,104 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import org.openrefine.wikidata.qa.QAWarning; +import org.openrefine.wikidata.updates.ItemUpdate; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.Value; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ItemRequiresScrutinizer extends EditScrutinizer { + + public static final String newItemRequireValuesType = "new-item-requires-property-to-have-certain-values"; + public static final String newItemRequirePropertyType = "new-item-requires-certain-other-statement"; + public static final String existingItemRequireValuesType = "existing-item-requires-property-to-have-certain-values"; + public static final String existingItemRequirePropertyType = "existing-item-requires-certain-other-statement"; + public static String ITEM_REQUIRES_CONSTRAINT_QID = "Q21503247"; + public static String ITEM_REQUIRES_PROPERTY_PID = "P2306"; + public static String ITEM_OF_PROPERTY_CONSTRAINT_PID = "P2305"; + + class ItemRequiresConstraint { + final PropertyIdValue itemRequiresPid; + final List itemList; + + ItemRequiresConstraint(Statement statement) { + List specs = statement.getClaim().getQualifiers(); + PropertyIdValue pid = null; + this.itemList = new ArrayList<>(); + for(SnakGroup group : specs) { + for (Snak snak : group.getSnaks()) { + if (group.getProperty().getId().equals(ITEM_REQUIRES_PROPERTY_PID)){ + pid = (PropertyIdValue) snak.getValue(); + } + if (group.getProperty().getId().equals(ITEM_OF_PROPERTY_CONSTRAINT_PID)){ + this.itemList.add(snak.getValue()); + } + } + } + this.itemRequiresPid = pid; + } + } + + @Override + public void scrutinize(ItemUpdate update) { + Map> propertyIdValueValueMap = new HashMap<>(); + for (Statement statement : update.getAddedStatements()) { + PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId(); + Value value = statement.getClaim().getMainSnak().getValue(); + Set values; + if (value != null) { + if (propertyIdValueValueMap.containsKey(pid)) { + values = propertyIdValueValueMap.get(pid); + } else { + values = new HashSet<>(); + } + values.add(value); + propertyIdValueValueMap.put(pid, values); + } + } + + for (PropertyIdValue propertyId : propertyIdValueValueMap.keySet()) { + List constraintDefinitions = _fetcher.getConstraintsByType(propertyId, ITEM_REQUIRES_CONSTRAINT_QID); + for (Statement statement : constraintDefinitions) { + ItemRequiresConstraint constraint = new ItemRequiresConstraint(statement); + PropertyIdValue itemRequiresPid = constraint.itemRequiresPid; + List itemList = constraint.itemList; + if (!propertyIdValueValueMap.containsKey(itemRequiresPid)) { + QAWarning issue = new QAWarning(update.isNew() ? newItemRequirePropertyType : existingItemRequirePropertyType, propertyId.getId() + itemRequiresPid.getId(), update.isNew() ? QAWarning.Severity.WARNING : QAWarning.Severity.INFO, 1); + issue.setProperty("property_entity", propertyId); + issue.setProperty("added_property_entity", itemRequiresPid); + issue.setProperty("example_entity", update.getItemId()); + addIssue(issue); + } else if (raiseWarning(propertyIdValueValueMap, itemRequiresPid, itemList)) { + QAWarning issue = new QAWarning(update.isNew() ? newItemRequireValuesType : existingItemRequireValuesType, propertyId.getId() + itemRequiresPid.getId(), update.isNew() ? QAWarning.Severity.WARNING : QAWarning.Severity.INFO, 1); + issue.setProperty("property_entity", propertyId); + issue.setProperty("added_property_entity", itemRequiresPid); + issue.setProperty("example_entity", update.getItemId()); + addIssue(issue); + } + } + } + } + + private boolean raiseWarning(Map> propertyIdValueValueMap, PropertyIdValue itemRequiresPid, List itemList) { + if (itemList.isEmpty()){ + return false; + } + + for (Value value : itemList) { + if (propertyIdValueValueMap.get(itemRequiresPid).contains(value)){ + return false; + } + } + + return true; + } +} diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizerTest.java new file mode 100644 index 000000000..c5172e642 --- /dev/null +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ItemRequiresScrutinizerTest.java @@ -0,0 +1,146 @@ +package org.openrefine.wikidata.qa.scrutinizers; + +import org.openrefine.wikidata.qa.ConstraintFetcher; +import org.openrefine.wikidata.testing.TestingData; +import org.openrefine.wikidata.updates.ItemUpdate; +import org.openrefine.wikidata.updates.ItemUpdateBuilder; +import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.implementation.StatementImpl; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueSnak; + +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.openrefine.wikidata.qa.scrutinizers.ItemRequiresScrutinizer.ITEM_OF_PROPERTY_CONSTRAINT_PID; +import static org.openrefine.wikidata.qa.scrutinizers.ItemRequiresScrutinizer.ITEM_REQUIRES_CONSTRAINT_QID; +import static org.openrefine.wikidata.qa.scrutinizers.ItemRequiresScrutinizer.ITEM_REQUIRES_PROPERTY_PID; + +public class ItemRequiresScrutinizerTest extends ScrutinizerTest { + + public static PropertyIdValue propertyIdValue = Datamodel.makeWikidataPropertyIdValue("P157"); + public static ItemIdValue itemValue = Datamodel.makeWikidataItemIdValue("Q3187975"); + public static ItemIdValue entityIdValue = Datamodel.makeWikidataItemIdValue(ITEM_REQUIRES_CONSTRAINT_QID); + public static PropertyIdValue propertyParameterPID = Datamodel.makeWikidataPropertyIdValue(ITEM_REQUIRES_PROPERTY_PID); + public static PropertyIdValue propertyParameterValue = Datamodel.makeWikidataPropertyIdValue("P1196"); + public static PropertyIdValue itemParameterPID = Datamodel.makeWikidataPropertyIdValue(ITEM_OF_PROPERTY_CONSTRAINT_PID); + public static Value requiredValue = Datamodel.makeWikidataItemIdValue("Q149086"); + + @Override + public EditScrutinizer getScrutinizer() { + return new ItemRequiresScrutinizer(); + } + + @Test + public void testExistingItemTrigger() { + ItemIdValue idA = TestingData.existingId; + Snak mainSnak = Datamodel.makeValueSnak(propertyIdValue, itemValue); + Statement statement = new StatementImpl("P157", mainSnak, idA); + ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(statement).build(); + + Snak qualifierSnak1 = Datamodel.makeValueSnak(propertyParameterPID, propertyParameterValue); + Snak qualifierSnak2 = Datamodel.makeValueSnak(itemParameterPID, requiredValue); + List constraintQualifiers = makeSnakGroupList(qualifierSnak1, qualifierSnak2); + List constraintDefinitions = constraintParameterStatementList(entityIdValue, constraintQualifiers); + + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, ITEM_REQUIRES_CONSTRAINT_QID)).thenReturn(constraintDefinitions); + setFetcher(fetcher); + + scrutinize(updateA); + assertWarningsRaised(ItemRequiresScrutinizer.existingItemRequirePropertyType); + } + + @Test + public void testWrongValue() { + ItemIdValue idA = TestingData.existingId; + Snak mainSnak = Datamodel.makeValueSnak(propertyIdValue, itemValue); + Statement statement = new StatementImpl("P157", mainSnak, idA); + Snak requiredPropertySnak = Datamodel.makeValueSnak(propertyParameterValue, itemValue); + Statement requiredStatement = new StatementImpl("P1196", requiredPropertySnak, idA); + ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(statement).addStatement(requiredStatement).build(); + + Snak qualifierSnak1 = Datamodel.makeValueSnak(propertyParameterPID, propertyParameterValue); + Snak qualifierSnak2 = Datamodel.makeValueSnak(itemParameterPID, requiredValue); + List constraintQualifiers = makeSnakGroupList(qualifierSnak1, qualifierSnak2); + List constraintDefinitions = constraintParameterStatementList(entityIdValue, constraintQualifiers); + + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, ITEM_REQUIRES_CONSTRAINT_QID)).thenReturn(constraintDefinitions); + setFetcher(fetcher); + + scrutinize(updateA); + assertWarningsRaised(ItemRequiresScrutinizer.existingItemRequireValuesType); + } + + @Test + public void testCorrectValue() { + ItemIdValue idA = TestingData.existingId; + Snak mainSnak = Datamodel.makeValueSnak(propertyIdValue, itemValue); + Statement statement = new StatementImpl("P157", mainSnak, idA); + Snak requiredPropertySnak = Datamodel.makeValueSnak(propertyParameterValue, requiredValue); + Statement requiredStatement = new StatementImpl("P1196", requiredPropertySnak, idA); + ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(statement).addStatement(requiredStatement).build(); + + Snak qualifierSnak1 = Datamodel.makeValueSnak(propertyParameterPID, propertyParameterValue); + Snak qualifierSnak2 = Datamodel.makeValueSnak(itemParameterPID, requiredValue); + List constraintQualifiers = makeSnakGroupList(qualifierSnak1, qualifierSnak2); + List constraintDefinitions = constraintParameterStatementList(entityIdValue, constraintQualifiers); + + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, ITEM_REQUIRES_CONSTRAINT_QID)).thenReturn(constraintDefinitions); + setFetcher(fetcher); + + scrutinize(updateA); + assertNoWarningRaised(); + } + + @Test + public void testNewItemTrigger() { + ItemIdValue idA = TestingData.newIdA; + Snak mainSnak = Datamodel.makeValueSnak(propertyIdValue, itemValue); + Statement statement = new StatementImpl("P157", mainSnak, idA); + ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(statement).build(); + + Snak qualifierSnak1 = Datamodel.makeValueSnak(propertyParameterPID, propertyParameterValue); + Snak qualifierSnak2 = Datamodel.makeValueSnak(itemParameterPID, requiredValue); + List constraintQualifiers = makeSnakGroupList(qualifierSnak1, qualifierSnak2); + List constraintDefinitions = constraintParameterStatementList(entityIdValue, constraintQualifiers); + + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, ITEM_REQUIRES_CONSTRAINT_QID)).thenReturn(constraintDefinitions); + setFetcher(fetcher); + + scrutinize(updateA); + assertWarningsRaised(ItemRequiresScrutinizer.newItemRequirePropertyType); + } + + @Test + public void testExistingItemNoIssue() { + ItemIdValue id = TestingData.existingId; + ValueSnak mainSnak1 = Datamodel.makeValueSnak(propertyIdValue, itemValue); + ValueSnak mainSnak2 = Datamodel.makeValueSnak(propertyParameterValue, requiredValue); + Statement statement1 = new StatementImpl("P157", mainSnak1,id); + Statement statement2 = new StatementImpl("P1196", mainSnak2,id); + ItemUpdate update = new ItemUpdateBuilder(id).addStatement(statement1).addStatement(statement2).build(); + + Snak qualifierSnak1 = Datamodel.makeValueSnak(propertyParameterPID, propertyParameterValue); + Snak qualifierSnak2 = Datamodel.makeValueSnak(itemParameterPID, requiredValue); + List constraintQualifiers = makeSnakGroupList(qualifierSnak1, qualifierSnak2); + List constraintDefinitions = constraintParameterStatementList(entityIdValue, constraintQualifiers); + + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, ITEM_REQUIRES_CONSTRAINT_QID)).thenReturn(constraintDefinitions); + + setFetcher(fetcher); + scrutinize(update); + assertNoWarningRaised(); + } +}