diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java index 2a55ffc53..83bfa37e9 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java @@ -100,8 +100,21 @@ public class EditInspector { Map updates = ItemUpdate.groupBySubject(editBatch); List mergedUpdates = updates.values().stream().collect(Collectors.toList()); + for (EditScrutinizer scrutinizer : scrutinizers.values()) { - scrutinizer.scrutinize(mergedUpdates); + scrutinizer.batchIsBeginning(); + } + + for(ItemUpdate update : mergedUpdates) { + if(!update.isNull()) { + for (EditScrutinizer scrutinizer : scrutinizers.values()) { + scrutinizer.scrutinize(update); + } + } + } + + for(EditScrutinizer scrutinizer : scrutinizers.values()) { + scrutinizer.batchIsFinished(); } if (warningStore.getNbWarnings() == 0) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java index f1d1bbcf3..79ac8e032 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java @@ -53,6 +53,13 @@ public abstract class EditScrutinizer { public void setFetcher(ConstraintFetcher fetcher) { _fetcher = fetcher; } + + /** + * Called before an edit batch is scrutinized. + */ + public void batchIsBeginning() { + + } /** * Reads the candidate edits and emits warnings in the store @@ -60,8 +67,22 @@ public abstract class EditScrutinizer { * @param edit: * the list of ItemUpdates to scrutinize */ - public abstract void scrutinize(List edit); - + public abstract void scrutinize(ItemUpdate edit); + + /** + * Method called once the edit batch has been read entirely + */ + public void batchIsFinished() { + + } + + /** + * Emits an issue that will be reported to the user, + * after mergin with other issues of the same kind. + * + * @param warning + * the issue to report + */ protected void addIssue(QAWarning warning) { _store.addWarning(warning); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemUpdateScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemUpdateScrutinizer.java deleted file mode 100644 index 8dae72fbc..000000000 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemUpdateScrutinizer.java +++ /dev/null @@ -1,57 +0,0 @@ -/******************************************************************************* - * MIT License - * - * Copyright (c) 2018 Antonin Delpeuch - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - ******************************************************************************/ -package org.openrefine.wikidata.qa.scrutinizers; - -import java.util.List; - -import org.openrefine.wikidata.updates.ItemUpdate; - -public abstract class ItemUpdateScrutinizer extends EditScrutinizer { - - @Override - public void scrutinize(List edit) { - for (ItemUpdate update : edit) { - if (!update.isNull()) { - scrutinize(update); - } - } - batchIsFinished(); - } - - /** - * Method to be overridden by subclasses to scrutinize an individual item - * update. - * - * @param update - */ - public abstract void scrutinize(ItemUpdate update); - - /** - * Method to be overridden by subclasses to emit warnings once a batch has been - * completely analyzed. - */ - public void batchIsFinished() { - ; - } -} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java index 064964de4..335e00caf 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java @@ -32,7 +32,7 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; * * @author Antonin Delpeuch */ -public class NewItemScrutinizer extends ItemUpdateScrutinizer { +public class NewItemScrutinizer extends EditScrutinizer { public static final String noLabelType = "new-item-without-labels-or-aliases"; public static final String noDescType = "new-item-without-descriptions"; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java index 089105032..4e9e8ae3c 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java @@ -30,13 +30,24 @@ import org.openrefine.wikidata.updates.ItemUpdate; public class NoEditsMadeScrutinizer extends EditScrutinizer { public static final String type = "no-edit-generated"; + + private boolean nonNullUpdateSeen = false; + + @Override + public void batchIsBeginning() { + nonNullUpdateSeen = false; + } @Override - public void scrutinize(List edit) { - if (edit.stream().allMatch(e -> e.isNull())) { + public void scrutinize(ItemUpdate edit) { + nonNullUpdateSeen = true; + } + + @Override + public void batchIsFinished() { + if(!nonNullUpdateSeen) { info(type); } - } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java index 4e6747b5d..bdd7bb3f4 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java @@ -38,7 +38,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement; * @author Antonin Delpeuch * */ -public class SingleValueScrutinizer extends ItemUpdateScrutinizer { +public class SingleValueScrutinizer extends EditScrutinizer { public static final String type = "single-valued-property-added-more-than-once"; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java index 62f856e17..847ccaf07 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java @@ -27,7 +27,7 @@ import org.openrefine.wikidata.updates.ItemUpdate; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.Statement; -public abstract class StatementScrutinizer extends ItemUpdateScrutinizer { +public abstract class StatementScrutinizer extends EditScrutinizer { @Override public void scrutinize(ItemUpdate update) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java index c5b087c40..c86fb540c 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java @@ -26,6 +26,7 @@ package org.openrefine.wikidata.schema; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.refine.model.Cell; @@ -39,6 +40,7 @@ import com.google.refine.model.Cell; * @param * the type of Wikibase value returned by the expression. */ +@JsonIgnoreProperties(ignoreUnknown = true) public abstract class WbVariableExpr implements WbExpression { private String columnName; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index 7e6fbe42a..4eb708037 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -23,21 +23,27 @@ ******************************************************************************/ package org.openrefine.wikidata.schema; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Properties; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; import org.openrefine.wikidata.qa.QAWarningStore; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.openrefine.wikidata.updates.ItemUpdate; -import org.openrefine.wikidata.utils.JacksonJsonizable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; + import com.google.refine.browsing.Engine; import com.google.refine.browsing.FilteredRows; import com.google.refine.browsing.RowVisitor; @@ -52,6 +58,7 @@ import com.google.refine.model.Row; * @author Antonin Delpeuch * */ +@JsonIgnoreProperties(ignoreUnknown = true) public class WikibaseSchema implements OverlayModel { final static Logger logger = LoggerFactory.getLogger("RdfSchema"); @@ -66,6 +73,14 @@ public class WikibaseSchema implements OverlayModel { public WikibaseSchema() { } + + /** + * Constructor for deserialization via Jackson + */ + @JsonCreator + public WikibaseSchema(@JsonProperty("itemDocuments") List exprs) { + this.itemDocumentExprs = exprs; + } /** * @return the site IRI of the Wikibase instance referenced by this schema @@ -80,7 +95,7 @@ public class WikibaseSchema implements OverlayModel { public List getItemDocumentExpressions() { return itemDocumentExprs; } - + public void setItemDocumentExpressions(List exprs) { this.itemDocumentExprs = exprs; } @@ -168,17 +183,22 @@ public class WikibaseSchema implements OverlayModel { static public WikibaseSchema reconstruct(JSONObject o) throws JSONException { - - JSONArray changeArr = o.getJSONArray("itemDocuments"); - WikibaseSchema schema = new WikibaseSchema(); - for (int i = 0; i != changeArr.length(); i++) { - WbItemDocumentExpr changeExpr = JacksonJsonizable.fromJSONClass(changeArr.getJSONObject(i), - WbItemDocumentExpr.class); - schema.itemDocumentExprs.add(changeExpr); + return reconstruct(o.toString()); + } + + static public WikibaseSchema reconstruct(String json) { + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.readValue(json, WikibaseSchema.class); + } catch (JsonParseException e) { + throw new JSONException(e.toString()); + } catch (JsonMappingException e) { + throw new JSONException(e.toString()); + } catch (IOException e) { + throw new JSONException(e.toString()); } - return schema; } - + @Override public void write(JSONWriter writer, Properties options) throws JSONException { diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java index 2bc8e6e48..2978bd02c 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java @@ -55,7 +55,13 @@ public abstract class ScrutinizerTest { } public void scrutinize(ItemUpdate... updates) { - scrutinizer.scrutinize(Arrays.asList(updates)); + scrutinizer.batchIsBeginning(); + for(ItemUpdate update : Arrays.asList(updates)) { + if(!update.isNull()) { + scrutinizer.scrutinize(update); + } + } + scrutinizer.batchIsFinished(); } public void assertWarningsRaised(String... types) {