s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Flatten scrutinizer hierarchy, optimize schema deserialization

Browse Source
This commit is contained in:
Antonin Delpeuch 2018-03-06 16:48:28 +00:00
parent 46964862bc
commit e9ce0fa59c
10 changed files with 95 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
extensions/wikidata/src/org/openrefine/wikidata/qa/EditInspector.java
View File

@ -100,8 +100,21 @@ public class EditInspector {
Map<EntityIdValue, ItemUpdate> updates = ItemUpdate.groupBySubject(editBatch);
List<ItemUpdate> mergedUpdates = updates.values().stream().collect(Collectors.toList());
for (EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.scrutinize(mergedUpdates);
scrutinizer.batchIsBeginning();
}
for(ItemUpdate update : mergedUpdates) {
if(!update.isNull()) {
for (EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.scrutinize(update);
}
}
}
for(EditScrutinizer scrutinizer : scrutinizers.values()) {
scrutinizer.batchIsFinished();
}
if (warningStore.getNbWarnings() == 0) {

25
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/EditScrutinizer.java
View File

@ -53,6 +53,13 @@ public abstract class EditScrutinizer {
public void setFetcher(ConstraintFetcher fetcher) {
_fetcher = fetcher;
}
/**
* Called before an edit batch is scrutinized.
*/
public void batchIsBeginning() {
}
/**
* Reads the candidate edits and emits warnings in the store
@ -60,8 +67,22 @@ public abstract class EditScrutinizer {
* @param edit:
* the list of ItemUpdates to scrutinize
*/
public abstract void scrutinize(List<ItemUpdate> edit);
public abstract void scrutinize(ItemUpdate edit);
/**
* Method called once the edit batch has been read entirely
*/
public void batchIsFinished() {
}
/**
* Emits an issue that will be reported to the user,
* after mergin with other issues of the same kind.
*
* @param warning
* the issue to report
*/
protected void addIssue(QAWarning warning) {
_store.addWarning(warning);
}

57
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/ItemUpdateScrutinizer.java
View File

@ -1,57 +0,0 @@
/*******************************************************************************
* MIT License
*
* Copyright (c) 2018 Antonin Delpeuch
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
******************************************************************************/
package org.openrefine.wikidata.qa.scrutinizers;
import java.util.List;
import org.openrefine.wikidata.updates.ItemUpdate;
public abstract class ItemUpdateScrutinizer extends EditScrutinizer {
@Override
public void scrutinize(List<ItemUpdate> edit) {
for (ItemUpdate update : edit) {
if (!update.isNull()) {
scrutinize(update);
}
}
batchIsFinished();
}
/**
* Method to be overridden by subclasses to scrutinize an individual item
* update.
*
* @param update
*/
public abstract void scrutinize(ItemUpdate update);
/**
* Method to be overridden by subclasses to emit warnings once a batch has been
* completely analyzed.
*/
public void batchIsFinished() {
;
}
}

2
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NewItemScrutinizer.java
View File

@ -32,7 +32,7 @@ import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
*
* @author Antonin Delpeuch
*/
public class NewItemScrutinizer extends ItemUpdateScrutinizer {
public class NewItemScrutinizer extends EditScrutinizer {
public static final String noLabelType = "new-item-without-labels-or-aliases";
public static final String noDescType = "new-item-without-descriptions";

17
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/NoEditsMadeScrutinizer.java
View File

@ -30,13 +30,24 @@ import org.openrefine.wikidata.updates.ItemUpdate;
public class NoEditsMadeScrutinizer extends EditScrutinizer {
public static final String type = "no-edit-generated";
private boolean nonNullUpdateSeen = false;
@Override
public void batchIsBeginning() {
nonNullUpdateSeen = false;
}
@Override
public void scrutinize(List<ItemUpdate> edit) {
if (edit.stream().allMatch(e -> e.isNull())) {
public void scrutinize(ItemUpdate edit) {
nonNullUpdateSeen = true;
}
@Override
public void batchIsFinished() {
if(!nonNullUpdateSeen) {
info(type);
}
}
}

2
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/SingleValueScrutinizer.java
View File

@ -38,7 +38,7 @@ import org.wikidata.wdtk.datamodel.interfaces.Statement;
* @author Antonin Delpeuch
*
*/
public class SingleValueScrutinizer extends ItemUpdateScrutinizer {
public class SingleValueScrutinizer extends EditScrutinizer {
public static final String type = "single-valued-property-added-more-than-once";

2
extensions/wikidata/src/org/openrefine/wikidata/qa/scrutinizers/StatementScrutinizer.java
View File

@ -27,7 +27,7 @@ import org.openrefine.wikidata.updates.ItemUpdate;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
public abstract class StatementScrutinizer extends ItemUpdateScrutinizer {
public abstract class StatementScrutinizer extends EditScrutinizer {
@Override
public void scrutinize(ItemUpdate update) {

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbVariableExpr.java
View File

@ -26,6 +26,7 @@ package org.openrefine.wikidata.schema;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.refine.model.Cell;
@ -39,6 +40,7 @@ import com.google.refine.model.Cell;
* @param <T>
* the type of Wikibase value returned by the expression.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public abstract class WbVariableExpr<T> implements WbExpression<T> {
private String columnName;

44
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -23,21 +23,27 @@
******************************************************************************/
package org.openrefine.wikidata.schema;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.qa.QAWarningStore;
import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException;
import org.openrefine.wikidata.updates.ItemUpdate;
import org.openrefine.wikidata.utils.JacksonJsonizable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
@ -52,6 +58,7 @@ import com.google.refine.model.Row;
* @author Antonin Delpeuch
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class WikibaseSchema implements OverlayModel {
final static Logger logger = LoggerFactory.getLogger("RdfSchema");
@ -66,6 +73,14 @@ public class WikibaseSchema implements OverlayModel {
public WikibaseSchema() {
}
/**
* Constructor for deserialization via Jackson
*/
@JsonCreator
public WikibaseSchema(@JsonProperty("itemDocuments") List<WbItemDocumentExpr> exprs) {
this.itemDocumentExprs = exprs;
}
/**
* @return the site IRI of the Wikibase instance referenced by this schema
@ -80,7 +95,7 @@ public class WikibaseSchema implements OverlayModel {
public List<WbItemDocumentExpr> getItemDocumentExpressions() {
return itemDocumentExprs;
}
public void setItemDocumentExpressions(List<WbItemDocumentExpr> exprs) {
this.itemDocumentExprs = exprs;
}
@ -168,17 +183,22 @@ public class WikibaseSchema implements OverlayModel {
static public WikibaseSchema reconstruct(JSONObject o)
throws JSONException {
JSONArray changeArr = o.getJSONArray("itemDocuments");
WikibaseSchema schema = new WikibaseSchema();
for (int i = 0; i != changeArr.length(); i++) {
WbItemDocumentExpr changeExpr = JacksonJsonizable.fromJSONClass(changeArr.getJSONObject(i),
WbItemDocumentExpr.class);
schema.itemDocumentExprs.add(changeExpr);
return reconstruct(o.toString());
}
static public WikibaseSchema reconstruct(String json) {
ObjectMapper mapper = new ObjectMapper();
try {
return mapper.readValue(json, WikibaseSchema.class);
} catch (JsonParseException e) {
throw new JSONException(e.toString());
} catch (JsonMappingException e) {
throw new JSONException(e.toString());
} catch (IOException e) {
throw new JSONException(e.toString());
}
return schema;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {

8
extensions/wikidata/tests/src/org/openrefine/wikidata/qa/scrutinizers/ScrutinizerTest.java
View File

@ -55,7 +55,13 @@ public abstract class ScrutinizerTest {
}
public void scrutinize(ItemUpdate... updates) {
scrutinizer.scrutinize(Arrays.asList(updates));
scrutinizer.batchIsBeginning();
for(ItemUpdate update : Arrays.asList(updates)) {
if(!update.isNull()) {
scrutinizer.scrutinize(update);
}
}
scrutinizer.batchIsFinished();
}
public void assertWarningsRaised(String... types) {