Flatten scrutinizer hierarchy, optimize schema deserialization

extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java

@@ -100,8 +100,21 @@ public class EditInspector {
 
         Map<EntityIdValue, ItemUpdate> updates = ItemUpdate.groupBySubject(editBatch);
         List<ItemUpdate> mergedUpdates = updates.values().stream().collect(Collectors.toList());
+        
         for (EditScrutinizer scrutinizer : scrutinizers.values()) {
-            scrutinizer.scrutinize(mergedUpdates);
+            scrutinizer.batchIsBeginning();
+        }
+        
+        for(ItemUpdate update : mergedUpdates) {
+            if(!update.isNull()) {
+                for (EditScrutinizer scrutinizer : scrutinizers.values()) {
+                    scrutinizer.scrutinize(update);
+                }
+            }
+        }
+        
+        for(EditScrutinizer scrutinizer : scrutinizers.values()) {
+            scrutinizer.batchIsFinished();
         }
 
         if (warningStore.getNbWarnings() == 0) {

extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java

@@ -53,6 +53,13 @@ public abstract class EditScrutinizer {
     public void setFetcher(ConstraintFetcher fetcher) {
         _fetcher = fetcher;
     }
+    
+    /**
+     * Called before an edit batch is scrutinized.
+     */
+    public void batchIsBeginning() {
+        
+    }
 
     /**
      * Reads the candidate edits and emits warnings in the store
@@ -60,8 +67,22 @@ public abstract class EditScrutinizer {
      * @param edit:
      *            the list of ItemUpdates to scrutinize
      */
-    public abstract void scrutinize(List<ItemUpdate> edit);
+    public abstract void scrutinize(ItemUpdate edit);
-
+    
+    /**
+     * Method called once the edit batch has been read entirely
+     */
+    public void batchIsFinished() {
+        
+    }
+    
+    /**
+     * Emits an issue that will be reported to the user,
+     * after mergin with other issues of the same kind.
+     * 
+     * @param warning
+     *    the issue to report
+     */
     protected void addIssue(QAWarning warning) {
         _store.addWarning(warning);
     }

extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemUpdateScrutinizer.java

@@ -1,57 +0,0 @@
-/*******************************************************************************
- * MIT License
- * 
- * Copyright (c) 2018 Antonin Delpeuch
- * 
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- * 
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- * 
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- ******************************************************************************/
-package org.openrefine.wikidata.qa.scrutinizers;
-
-import java.util.List;
-
-import org.openrefine.wikidata.updates.ItemUpdate;
-
-public abstract class ItemUpdateScrutinizer extends EditScrutinizer {
-
-    @Override
-    public void scrutinize(List<ItemUpdate> edit) {
-        for (ItemUpdate update : edit) {
-            if (!update.isNull()) {
-                scrutinize(update);
-            }
-        }
-        batchIsFinished();
-    }
-
-    /**
-     * Method to be overridden by subclasses to scrutinize an individual item
-     * update.
-     * 
-     * @param update
-     */
-    public abstract void scrutinize(ItemUpdate update);
-
-    /**
-     * Method to be overridden by subclasses to emit warnings once a batch has been
-     * completely analyzed.
-     */
-    public void batchIsFinished() {
-        ;
-    }
-}

extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java

@@ -32,7 +32,7 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
  * 
  * @author Antonin Delpeuch
  */
-public class NewItemScrutinizer extends ItemUpdateScrutinizer {
+public class NewItemScrutinizer extends EditScrutinizer {
 
     public static final String noLabelType = "new-item-without-labels-or-aliases";
     public static final String noDescType = "new-item-without-descriptions";

extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java

@@ -30,13 +30,24 @@ import org.openrefine.wikidata.updates.ItemUpdate;
 public class NoEditsMadeScrutinizer extends EditScrutinizer {
 
     public static final String type = "no-edit-generated";
+    
+    private boolean nonNullUpdateSeen = false;
+    
+    @Override
+    public void batchIsBeginning() {
+        nonNullUpdateSeen = false;
+    }
 
     @Override
-    public void scrutinize(List<ItemUpdate> edit) {
+    public void scrutinize(ItemUpdate edit) {
-        if (edit.stream().allMatch(e -> e.isNull())) {
+        nonNullUpdateSeen = true;
+    }
+    
+    @Override
+    public void batchIsFinished() {
+        if(!nonNullUpdateSeen) {
             info(type);
         }
-
     }
 
 }

extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java

@@ -38,7 +38,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
  * @author Antonin Delpeuch
  *
  */
-public class SingleValueScrutinizer extends ItemUpdateScrutinizer {
+public class SingleValueScrutinizer extends EditScrutinizer {
 
     public static final String type = "single-valued-property-added-more-than-once";
 

extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java

@@ -27,7 +27,7 @@ import org.openrefine.wikidata.updates.ItemUpdate;
 import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
 import org.wikidata.wdtk.datamodel.interfaces.Statement;
 
-public abstract class StatementScrutinizer extends ItemUpdateScrutinizer {
+public abstract class StatementScrutinizer extends EditScrutinizer {
 
     @Override
     public void scrutinize(ItemUpdate update) {

extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java

@@ -26,6 +26,7 @@ package org.openrefine.wikidata.schema;
 import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import com.google.refine.model.Cell;
@@ -39,6 +40,7 @@ import com.google.refine.model.Cell;
  * @param <T>
  *            the type of Wikibase value returned by the expression.
  */
+@JsonIgnoreProperties(ignoreUnknown = true)
 public abstract class WbVariableExpr<T> implements WbExpression<T> {
 
     private String columnName;

extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java

@@ -23,21 +23,27 @@
  ******************************************************************************/
 package org.openrefine.wikidata.schema;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
-import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONWriter;
 import org.openrefine.wikidata.qa.QAWarningStore;
 import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
 import org.openrefine.wikidata.updates.ItemUpdate;
-import org.openrefine.wikidata.utils.JacksonJsonizable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import com.google.refine.browsing.Engine;
 import com.google.refine.browsing.FilteredRows;
 import com.google.refine.browsing.RowVisitor;
@@ -52,6 +58,7 @@ import com.google.refine.model.Row;
  * @author Antonin Delpeuch
  *
  */
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class WikibaseSchema implements OverlayModel {
 
     final static Logger logger = LoggerFactory.getLogger("RdfSchema");
@@ -66,6 +73,14 @@ public class WikibaseSchema implements OverlayModel {
     public WikibaseSchema() {
 
     }
+    
+    /**
+     * Constructor for deserialization via Jackson
+     */
+    @JsonCreator
+    public WikibaseSchema(@JsonProperty("itemDocuments") List<WbItemDocumentExpr> exprs) {
+        this.itemDocumentExprs = exprs;
+    }
 
     /**
      * @return the site IRI of the Wikibase instance referenced by this schema
@@ -80,7 +95,7 @@ public class WikibaseSchema implements OverlayModel {
     public List<WbItemDocumentExpr> getItemDocumentExpressions() {
         return itemDocumentExprs;
     }
-
+    
     public void setItemDocumentExpressions(List<WbItemDocumentExpr> exprs) {
         this.itemDocumentExprs = exprs;
     }
@@ -168,17 +183,22 @@ public class WikibaseSchema implements OverlayModel {
 
     static public WikibaseSchema reconstruct(JSONObject o)
             throws JSONException {
-
+        return reconstruct(o.toString());
-        JSONArray changeArr = o.getJSONArray("itemDocuments");
+    }    
-        WikibaseSchema schema = new WikibaseSchema();
+    
-        for (int i = 0; i != changeArr.length(); i++) {
+    static public WikibaseSchema reconstruct(String json) {
-            WbItemDocumentExpr changeExpr = JacksonJsonizable.fromJSONClass(changeArr.getJSONObject(i),
+        ObjectMapper mapper = new ObjectMapper();
-                    WbItemDocumentExpr.class);
+        try {
-            schema.itemDocumentExprs.add(changeExpr);
+            return mapper.readValue(json, WikibaseSchema.class);
+        } catch (JsonParseException e) {
+            throw new JSONException(e.toString());
+        } catch (JsonMappingException e) {
+            throw new JSONException(e.toString());
+        } catch (IOException e) {
+            throw new JSONException(e.toString());
         }
-        return schema;
     }
-
+    
     @Override
     public void write(JSONWriter writer, Properties options)
             throws JSONException {

extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java

@@ -55,7 +55,13 @@ public abstract class ScrutinizerTest {
     }
 
     public void scrutinize(ItemUpdate... updates) {
-        scrutinizer.scrutinize(Arrays.asList(updates));
+        scrutinizer.batchIsBeginning();
+        for(ItemUpdate update : Arrays.asList(updates)) {
+            if(!update.isNull()) {
+                scrutinizer.scrutinize(update);
+            }
+        }
+        scrutinizer.batchIsFinished();
     }
 
     public void assertWarningsRaised(String... types) {