Add Citation needed scrutinizer (#2925)

Implemented Citation needed Cconstraint as part of #2354

Test class added with appropriate testc cases

Updated severity level to critical as well as the messages

merged unsourced and citation-needed scrutinizer

updated severity levels and warning messages

extensions/wikidata/module/langs/translation-en.json

@@ -158,6 +158,8 @@
     "warnings-messages/existing-item-requires-certain-other-statement/body": "This property is expected to have another statement with property {added_property_entity} such as on item {example_entity}. If the item already has statements with property {added_property_entity} in Wikidata, then this warning can be ignored.",
     "warnings-messages/values-should-not-be-used-as-qualifier/title": "Invalid value for qualifier {qualifier_entity} on statement {statement_entity}",
     "warnings-messages/values-should-not-be-used-as-qualifier/body": "On items such as {example_entity}, an invalid value for qualifier {qualifier_entity} was supplied on statement {statement_entity}.",
+    "warnings-messages/no-references-provided/title": "{property_entity} requires at least one reference.",
+    "warnings-messages/no-references-provided/body": "The property {property_entity} specifically requires at least one reference for its statements.",
     "warnings-messages/ignored-qualifiers/title": "Some qualifiers were ignored.",
     "warnings-messages/ignored-qualifiers/body": "Qualifier values could not be parsed, so they will not be added to the corresponding statements.",
     "warnings-messages/ignored-references/title": "Some references were ignored.",

extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/UnsourcedScrutinizer.java

@@ -23,23 +23,43 @@
  ******************************************************************************/
 package org.openrefine.wikidata.qa.scrutinizers;
 
-import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
+import org.openrefine.wikidata.qa.QAWarning;
+import org.openrefine.wikidata.updates.ItemUpdate;
+import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
+import org.wikidata.wdtk.datamodel.interfaces.Reference;
 import org.wikidata.wdtk.datamodel.interfaces.Statement;
 
+import java.util.List;
+
 /**
  * A scrutinizer checking for unsourced statements
  * 
  * @author Antonin Delpeuch
  *
  */
-public class UnsourcedScrutinizer extends StatementScrutinizer {
+public class UnsourcedScrutinizer extends EditScrutinizer {
 
-    public static final String type = "unsourced-statements";
+    public static final String CITATION_NEEDED_QID = "Q54554025";
+    public static final String generalType = "unsourced-statements";
+    public static final String constraintItemType = "no-references-provided";
 
     @Override
-    public void scrutinize(Statement statement, EntityIdValue entityId, boolean added) {
+    public void scrutinize(ItemUpdate update) {
-        if (statement.getReferences().isEmpty() && added) {
+        for (Statement statement : update.getAddedStatements()) {
-            warning(type);
+            PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId();
+            List<Statement> constraintDefinitions = _fetcher.getConstraintsByType(pid, CITATION_NEEDED_QID);
+            List<Reference> referenceList = statement.getReferences();
+
+            if (referenceList.isEmpty()) {
+                if (!constraintDefinitions.isEmpty()) {
+                    QAWarning issue = new QAWarning(constraintItemType, pid.getId(), QAWarning.Severity.IMPORTANT, 1);
+                    issue.setProperty("property_entity", pid);
+                    issue.setProperty("example_entity", update.getItemId());
+                    addIssue(issue);
+                } else {
+                    warning(generalType);
+                }
+            }
         }
     }
 

extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/UnsourcedScrutinizerTest.java

@@ -23,20 +23,80 @@
  ******************************************************************************/
 package org.openrefine.wikidata.qa.scrutinizers;
 
+import org.openrefine.wikidata.qa.ConstraintFetcher;
 import org.openrefine.wikidata.testing.TestingData;
+import org.openrefine.wikidata.updates.ItemUpdate;
+import org.openrefine.wikidata.updates.ItemUpdateBuilder;
 import org.testng.annotations.Test;
+import org.wikidata.wdtk.datamodel.helpers.Datamodel;
+import org.wikidata.wdtk.datamodel.implementation.StatementImpl;
+import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
+import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
+import org.wikidata.wdtk.datamodel.interfaces.Snak;
+import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
+import org.wikidata.wdtk.datamodel.interfaces.Statement;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.openrefine.wikidata.qa.scrutinizers.UnsourcedScrutinizer.CITATION_NEEDED_QID;
 
 public class UnsourcedScrutinizerTest extends StatementScrutinizerTest {
 
+    public static PropertyIdValue propertyIdValue = Datamodel.makeWikidataPropertyIdValue("P2302");
+    public static PropertyIdValue referenceProperty = Datamodel.makeWikidataPropertyIdValue("P143");
+    public static ItemIdValue referenceValue = Datamodel.makeWikidataItemIdValue("Q348");
+    public static ItemIdValue entityIdValue = Datamodel.makeWikidataItemIdValue(CITATION_NEEDED_QID);
+
     @Override
     public EditScrutinizer getScrutinizer() {
         return new UnsourcedScrutinizer();
     }
 
     @Test
-    public void testTrigger() {
+    public void testWithoutConstraint() {
+        ConstraintFetcher fetcher = mock(ConstraintFetcher.class);
+        when(fetcher.getConstraintsByType(any(), eq(CITATION_NEEDED_QID))).thenReturn(Collections.emptyList());
+        setFetcher(fetcher);
         scrutinize(TestingData.generateStatement(TestingData.existingId, TestingData.matchedId));
-        assertWarningsRaised(UnsourcedScrutinizer.type);
+        assertWarningsRaised(UnsourcedScrutinizer.generalType);
     }
 
+    @Test
+    public void testTrigger() {
+        ItemIdValue id = TestingData.existingId;
+        Snak mainSnak = Datamodel.makeSomeValueSnak(propertyIdValue);
+        Statement statement = new StatementImpl("P172", mainSnak, id);
+        ItemUpdate update = new ItemUpdateBuilder(id).addStatement(statement).build();
+
+        List<Statement> constraintDefinitions = constraintParameterStatementList(entityIdValue, Collections.emptyList());
+        ConstraintFetcher fetcher = mock(ConstraintFetcher.class);
+        when(fetcher.getConstraintsByType(propertyIdValue, CITATION_NEEDED_QID)).thenReturn(constraintDefinitions);
+        setFetcher(fetcher);
+
+        scrutinize(update);
+        assertWarningsRaised(UnsourcedScrutinizer.constraintItemType);
+    }
+
+    @Test
+    public void testNoIssue() {
+        ItemIdValue id = TestingData.existingId;
+        Snak referenceSnak = Datamodel.makeValueSnak(referenceProperty, referenceValue);
+        List<SnakGroup> constraintQualifiers = makeSnakGroupList(referenceSnak);
+        List<Statement> itemStatementList = constraintParameterStatementList(entityIdValue, constraintQualifiers);
+        Statement statement = itemStatementList.get(0);
+        ItemUpdate update = new ItemUpdateBuilder(id).addStatement(statement).build();
+
+        List<Statement> constraintDefinitions = constraintParameterStatementList(entityIdValue, Collections.emptyList());
+        ConstraintFetcher fetcher = mock(ConstraintFetcher.class);
+        when(fetcher.getConstraintsByType(propertyIdValue, CITATION_NEEDED_QID)).thenReturn(constraintDefinitions);
+        setFetcher(fetcher);
+
+        scrutinize(update);
+        assertNoWarningRaised();
+    }
 }