From 8c1d8cdcb7021e1f9d4cbd0d1cef21ec84d1c238 Mon Sep 17 00:00:00 2001 From: Ekta Mishra Date: Fri, 26 Jun 2020 13:44:34 +0530 Subject: [PATCH] New implementation for Multivalue Scrutinizer (#2807) Created inner class for Multivalue & mocks for unit tests New implementation for multivalue scrutinizer tests updated --- .../wikidata/qa/ConstraintFetcher.java | 2 +- .../scrutinizers/MultiValueScrutinizer.java | 29 +++++++- .../MultiValueScrutinizerTest.java | 66 ++++++++++++++++++- .../qa/scrutinizers/ScrutinizerTest.java | 45 +++++++++++-- 4 files changed, 129 insertions(+), 13 deletions(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java index 43f1d609d..d587c249d 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/ConstraintFetcher.java @@ -26,8 +26,8 @@ package org.openrefine.wikidata.qa; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; -import org.wikidata.wdtk.datamodel.interfaces.Value; import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.Value; import java.util.Set; import java.util.stream.Stream; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizer.java index 32b5de81d..138f76de4 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizer.java @@ -3,15 +3,36 @@ package org.openrefine.wikidata.qa.scrutinizers; import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.Value; import java.util.HashMap; +import java.util.List; import java.util.Map; public class MultiValueScrutinizer extends EditScrutinizer { public static final String new_type = "multi-valued-property-is-required-for-new-item"; public static final String existing_type = "multi-valued-property-is-required-for-existing-item"; + public static String MULTI_VALUE_CONSTRAINT_QID = "Q21510857"; + public static String MULTI_VALUE_CONSTRAINT_STATUS = "P2316"; + + class MultivalueConstraint { + Value constraintStatus; + + MultivalueConstraint(Statement statement) { + List snakGroupList = statement.getClaim().getQualifiers(); + for(SnakGroup group : snakGroupList) { + for (Snak snak : group.getSnaks()) { + if (group.getProperty().getId().equals(MULTI_VALUE_CONSTRAINT_STATUS)){ + constraintStatus = snak.getValue(); + } + } + } + } + } @Override public void scrutinize(ItemUpdate update) { @@ -21,8 +42,12 @@ public class MultiValueScrutinizer extends EditScrutinizer { PropertyIdValue pid = statement.getClaim().getMainSnak().getPropertyId(); if (propertyCount.containsKey(pid)) { propertyCount.put(pid, propertyCount.get(pid) + 1); - } else if (_fetcher.hasMultiValue(pid)) { - propertyCount.put(pid, 1); + } else { + Statement constraintStatement = _fetcher.getConstraintsByType(pid, MULTI_VALUE_CONSTRAINT_QID).findFirst().orElse(null); + if (constraintStatement != null) { + MultivalueConstraint constraint = new MultivalueConstraint(constraintStatement); + propertyCount.put(pid, 1); + } } } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizerTest.java index 152c49a21..7fafbefa1 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizerTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/MultiValueScrutinizerTest.java @@ -1,13 +1,35 @@ package org.openrefine.wikidata.qa.scrutinizers; +import org.openrefine.wikidata.qa.ConstraintFetcher; import org.openrefine.wikidata.testing.TestingData; import org.openrefine.wikidata.updates.ItemUpdate; import org.openrefine.wikidata.updates.ItemUpdateBuilder; import org.testng.annotations.Test; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.implementation.StatementImpl; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.Value; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MultiValueScrutinizerTest extends ScrutinizerTest { + public static PropertyIdValue propertyIdValue = Datamodel.makeWikidataPropertyIdValue("P1963"); + public static Value valueSnak1 = Datamodel.makeWikidataItemIdValue("Q5"); + public static Value valueSnak2 = Datamodel.makeWikidataItemIdValue("Q4"); + public static ItemIdValue entityIdValue = Datamodel.makeWikidataItemIdValue("Q21510857"); + public static PropertyIdValue constraintParameter = Datamodel.makeWikidataPropertyIdValue("P2316"); + public static Value constraintStatus = Datamodel.makeWikidataItemIdValue("Q62026391"); + @Override public EditScrutinizer getScrutinizer() { return new MultiValueScrutinizer(); @@ -17,8 +39,22 @@ public class MultiValueScrutinizerTest extends ScrutinizerTest { public void testNoIssue() { ItemIdValue idA = TestingData.existingId; ItemIdValue idB = TestingData.matchedId; + Snak snakValue1 = Datamodel.makeSomeValueSnak(propertyIdValue); + Snak snakValue2 = Datamodel.makeSomeValueSnak(propertyIdValue); + Statement statement1 = new StatementImpl("P1963", snakValue1, idA); + Statement statement2 = new StatementImpl("P1963", snakValue2, idA); ItemUpdate update = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, idB)) - .addStatement(TestingData.generateStatement(idA, idB)).build(); + .addStatement(TestingData.generateStatement(idA, idB)).addStatement(statement1).addStatement(statement2).build(); + + Snak snak = Datamodel.makeValueSnak(constraintParameter, constraintStatus); + List snakList1 = Collections.singletonList(snak); + SnakGroup snakGroup = Datamodel.makeSnakGroup(snakList1); + List snakGroupList = Collections.singletonList(snakGroup); + Stream statementStream = constraintParameterStatementStream(entityIdValue, snakGroupList); + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, "Q21510857")).thenReturn(statementStream); + setFetcher(fetcher); + scrutinize(update); assertNoWarningRaised(); } @@ -27,8 +63,20 @@ public class MultiValueScrutinizerTest extends ScrutinizerTest { public void testNewItemTrigger() { ItemIdValue idA = TestingData.newIdA; ItemIdValue idB = TestingData.newIdB; - ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, idB)).build(); + Snak mainSnakValue = Datamodel.makeValueSnak(propertyIdValue, valueSnak1); + Statement statement = new StatementImpl("P1963", mainSnakValue, idA); + ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, idB)).addStatement(statement).build(); ItemUpdate updateB = new ItemUpdateBuilder(idB).addStatement(TestingData.generateStatement(idB, idB)).build(); + + Snak snak = Datamodel.makeValueSnak(constraintParameter, constraintStatus); + List snakList1 = Collections.singletonList(snak); + SnakGroup snakGroup = Datamodel.makeSnakGroup(snakList1); + List snakGroupList = Collections.singletonList(snakGroup); + Stream statementStream = constraintParameterStatementStream(entityIdValue, snakGroupList); + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, "Q21510857")).thenReturn(statementStream); + setFetcher(fetcher); + scrutinize(updateA, updateB); assertWarningsRaised(MultiValueScrutinizer.new_type); } @@ -37,8 +85,20 @@ public class MultiValueScrutinizerTest extends ScrutinizerTest { public void testExistingItemTrigger() { ItemIdValue idA = TestingData.existingId; ItemIdValue idB = TestingData.matchedId; - ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, idB)).build(); + Snak mainSnakValue = Datamodel.makeValueSnak(propertyIdValue, valueSnak1); + Statement statement = new StatementImpl("P1963", mainSnakValue, idA); + ItemUpdate updateA = new ItemUpdateBuilder(idA).addStatement(TestingData.generateStatement(idA, idB)).addStatement(statement).build(); ItemUpdate updateB = new ItemUpdateBuilder(idB).addStatement(TestingData.generateStatement(idB, idB)).build(); + + Snak snak = Datamodel.makeValueSnak(constraintParameter, constraintStatus); + List snakList1 = Collections.singletonList(snak); + SnakGroup snakGroup = Datamodel.makeSnakGroup(snakList1); + List snakGroupList = Collections.singletonList(snakGroup); + Stream statementStream = constraintParameterStatementStream(entityIdValue, snakGroupList); + ConstraintFetcher fetcher = mock(ConstraintFetcher.class); + when(fetcher.getConstraintsByType(propertyIdValue, "Q21510857")).thenReturn(statementStream); + setFetcher(fetcher); + scrutinize(updateA, updateB); assertWarningsRaised(MultiValueScrutinizer.existing_type); } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java index 346ffadd5..ac85a584f 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java @@ -23,19 +23,31 @@ ******************************************************************************/ package org.openrefine.wikidata.qa.scrutinizers; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; - -import java.util.Arrays; -import java.util.Set; -import java.util.stream.Collectors; - import org.openrefine.wikidata.qa.ConstraintFetcher; import org.openrefine.wikidata.qa.MockConstraintFetcher; import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.updates.ItemUpdate; import org.testng.annotations.BeforeMethod; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.Claim; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Reference; +import org.wikidata.wdtk.datamodel.interfaces.Snak; +import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.StatementRank; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; public abstract class ScrutinizerTest { @@ -79,4 +91,23 @@ public abstract class ScrutinizerTest { public Set getWarningTypes() { return store.getWarnings().stream().map(w -> w.getType()).collect(Collectors.toSet()); } + + public void setFetcher(ConstraintFetcher fetcher) { + scrutinizer.setFetcher(fetcher); + } + + public Stream constraintParameterStatementStream(ItemIdValue itemIdValue, List listSnakGroup) { + PropertyIdValue propertyIdValue = Datamodel.makeWikidataPropertyIdValue("P2302"); + Snak snakValue = Datamodel.makeValueSnak(propertyIdValue,itemIdValue); + + Claim claim = Datamodel.makeClaim(itemIdValue, snakValue, listSnakGroup); + + Reference reference = Datamodel.makeReference(listSnakGroup); + List referenceList = Collections.singletonList(reference); + + Statement statement = Datamodel.makeStatement(claim, referenceList, StatementRank.NORMAL, "P2302$77BD7FE4-C051-4776-855C-543F0CE697D0"); + List statements = Collections.singletonList(statement); + + return statements.stream(); + } }