From 5853cdf10d4448ec8e687755089e80c4ed57bca3 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Mon, 28 Aug 2017 10:08:24 +0100 Subject: [PATCH] Wikibase schema serialization and related operation --- .../SaveWikibaseSchemaOperation.java | 138 ++++++++++++++++++ .../wikidata/schema/BiJsonizable.java | 28 ++++ .../wikidata/schema/WbChangeExpr.java | 6 +- .../wikidata/schema/WbClaimExpr.java | 31 +++- .../wikidata/schema/WbItemConstant.java | 11 +- .../wikidata/schema/WbItemExpr.java | 16 +- .../wikidata/schema/WbItemVariable.java | 12 +- .../wikidata/schema/WbPropConstant.java | 13 +- .../wikidata/schema/WbPropExpr.java | 5 +- .../wikidata/schema/WbSnakExpr.java | 19 +++ .../wikidata/schema/WbStringConstant.java | 14 +- .../wikidata/schema/WbValueExpr.java | 12 +- .../wikidata/schema/WikibaseSchema.java | 84 +++++++++++ 13 files changed, 354 insertions(+), 35 deletions(-) create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/BiJsonizable.java create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java diff --git a/extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java b/extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java new file mode 100644 index 000000000..99a42e6dc --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java @@ -0,0 +1,138 @@ +package org.openrefine.wikidata.operations; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Writer; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.openrefine.wikidata.schema.WikibaseSchema; + +import com.google.refine.history.Change; +import com.google.refine.history.HistoryEntry; +import com.google.refine.model.AbstractOperation; +import com.google.refine.model.Project; +import com.google.refine.operations.OperationRegistry; +import com.google.refine.util.ParsingUtilities; +import com.google.refine.util.Pool; + +public class SaveWikibaseSchemaOperation extends AbstractOperation { + + final protected WikibaseSchema _schema; + + public SaveWikibaseSchemaOperation(WikibaseSchema schema) { + this._schema = schema; + } + + static public AbstractOperation reconstruct(Project project, JSONObject obj) + throws Exception { + return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj + .getJSONObject("schema"))); + } + + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.object(); + writer.key("op"); + writer.value(OperationRegistry.s_opClassToName.get(this.getClass())); + writer.key("description"); + writer.value("Save RDF schema skeleton"); + writer.key("schema"); + _schema.write(writer, options); + writer.endObject(); + + } + + @Override + protected String getBriefDescription(Project project) { + return "Save Wikibase schema skelton"; + } + + @Override + protected HistoryEntry createHistoryEntry(Project project, + long historyEntryID) throws Exception { + String description = "Save Wikibase schema skeleton"; + + Change change = new WikibaseSchemaChange(_schema); + + return new HistoryEntry(historyEntryID, project, description, + SaveWikibaseSchemaOperation.this, change); + } + + static public class WikibaseSchemaChange implements Change { + final protected WikibaseSchema _newSchema; + protected WikibaseSchema _oldSchema; + public final static String overlayModelKey = "wikibaseSchema"; + + public WikibaseSchemaChange(WikibaseSchema schema) { + _newSchema = schema; + } + + public void apply(Project project) { + synchronized (project) { + _oldSchema = (WikibaseSchema) project.overlayModels.get(overlayModelKey); + project.overlayModels.put(overlayModelKey, _newSchema); + } + } + + public void revert(Project project) { + synchronized (project) { + if (_oldSchema == null) { + project.overlayModels.remove(overlayModelKey); + } else { + project.overlayModels.put(overlayModelKey, _oldSchema); + } + } + } + + public void save(Writer writer, Properties options) throws IOException { + writer.write("newSchema="); + writeWikibaseSchema(_newSchema, writer); + writer.write('\n'); + writer.write("oldSchema="); + writeWikibaseSchema(_oldSchema, writer); + writer.write('\n'); + writer.write("/ec/\n"); // end of change marker + } + + static public Change load(LineNumberReader reader, Pool pool) + throws Exception { + WikibaseSchema oldSchema = null; + WikibaseSchema newSchema = null; + + String line; + while ((line = reader.readLine()) != null && !"/ec/".equals(line)) { + int equal = line.indexOf('='); + CharSequence field = line.subSequence(0, equal); + String value = line.substring(equal + 1); + + if ("oldSchema".equals(field) && value.length() > 0) { + oldSchema = WikibaseSchema.reconstruct(ParsingUtilities + .evaluateJsonStringToObject(value)); + } else if ("newSchema".equals(field) && value.length() > 0) { + newSchema = WikibaseSchema.reconstruct(ParsingUtilities + .evaluateJsonStringToObject(value)); + } + } + + WikibaseSchemaChange change = new WikibaseSchemaChange(newSchema); + change._oldSchema = oldSchema; + + return change; + } + + static protected void writeWikibaseSchema(WikibaseSchema s, Writer writer) + throws IOException { + if (s != null) { + JSONWriter jsonWriter = new JSONWriter(writer); + try { + s.write(jsonWriter, new Properties()); + } catch (JSONException e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/BiJsonizable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/BiJsonizable.java new file mode 100644 index 000000000..17d11ed71 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/BiJsonizable.java @@ -0,0 +1,28 @@ +package org.openrefine.wikidata.schema; + +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONWriter; + +import com.google.refine.Jsonizable; + + +public abstract class BiJsonizable implements Jsonizable { + + public static String jsonType; + public static final String jsonTypeKey = "@type"; + + public abstract void writeFields(JSONWriter writer, Properties options) + throws JSONException; + + @Override + public void write(JSONWriter writer, Properties options) throws JSONException { + writer.object(); + writer.key(jsonTypeKey); + writer.value(jsonType); + writeFields(writer, options); + writer.endObject(); + } + +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java index a3eb4b587..2f9ae8eb3 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java @@ -1,10 +1,10 @@ package org.openrefine.wikidata.schema; -import com.google.refine.Jsonizable; - -public interface WbChangeExpr extends Jsonizable { +public abstract class WbChangeExpr extends BiJsonizable { /* Represents a change on an item: adding a statement, * adding a label, adding a sitelink… */ + + } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbClaimExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbClaimExpr.java index f4e41dea0..5f4b080b8 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbClaimExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbClaimExpr.java @@ -6,7 +6,9 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; +import org.json.JSONObject; import org.json.JSONWriter; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.Claim; @@ -16,18 +18,26 @@ import org.wikidata.wdtk.datamodel.interfaces.SnakGroup; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; -public class WbClaimExpr implements WbChangeExpr { +public class WbClaimExpr extends WbChangeExpr { + + public static String jsonType = "wbclaimexpr"; private WbItemExpr subjectExpr; private WbSnakExpr mainSnakExpr; private List qualifierExprs; // TODO: references + public WbClaimExpr(WbItemExpr subjectExpr, + WbSnakExpr mainSnakExpr, + List qualifierExprs) { + this.subjectExpr = subjectExpr; + this.mainSnakExpr = mainSnakExpr; + this.qualifierExprs = qualifierExprs; + } @Override - public void write(JSONWriter writer, Properties options) + public void writeFields(JSONWriter writer, Properties options) throws JSONException { - writer.object(); writer.key("subject"); subjectExpr.write(writer, options); writer.key("mainsnak"); @@ -38,7 +48,20 @@ public class WbClaimExpr implements WbChangeExpr { expr.write(writer, options); } writer.endArray(); - writer.endObject(); + } + + public static WbClaimExpr fromJSON(JSONObject obj) throws JSONException { + JSONObject subjObj = obj.getJSONObject("subject"); + JSONObject mainSnakObj = obj.getJSONObject("mainsnak"); + JSONArray qualifiersArr = obj.getJSONArray("qualifiers"); + List qualifierExprs = new ArrayList(); + for (int i = 0; i != qualifiersArr.length(); i++) { + qualifierExprs.add(WbSnakExpr.fromJSON(qualifiersArr.getJSONObject(i))); + } + return new WbClaimExpr( + WbItemExpr.fromJSON(subjObj), + WbSnakExpr.fromJSON(mainSnakObj), + qualifierExprs); } public static List groupSnaks(List snaks) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java index 5efe60ab2..da883dde6 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java @@ -3,12 +3,13 @@ package org.openrefine.wikidata.schema; import java.util.Properties; import org.json.JSONException; +import org.json.JSONObject; import org.json.JSONWriter; import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; -public class WbItemConstant implements WbItemExpr { +public class WbItemConstant extends WbItemExpr { /* Represents an item that does not vary, * it is independent of the row. */ @@ -21,14 +22,16 @@ public class WbItemConstant implements WbItemExpr { } @Override - public void write(JSONWriter writer, Properties options) + public void writeFields(JSONWriter writer, Properties options) throws JSONException { - writer.object(); writer.key("qid"); writer.value(qid); writer.key("label"); writer.value(label); - writer.endObject(); + } + + public static WbItemConstant fromJSON(JSONObject obj) throws JSONException { + return new WbItemConstant(obj.getString("qid"), obj.getString("label")); } @Override diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java index 5c9e02341..8b04c1905 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java @@ -1,9 +1,21 @@ package org.openrefine.wikidata.schema; +import org.json.JSONException; +import org.json.JSONObject; import org.openrefine.wikidata.schema.ExpressionContext; import org.openrefine.wikidata.schema.WbValueExpr; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; -public interface WbItemExpr extends WbValueExpr { - public ItemIdValue evaluate(ExpressionContext ctxt); +public abstract class WbItemExpr extends WbValueExpr { + public abstract ItemIdValue evaluate(ExpressionContext ctxt); + + public static WbItemExpr fromJSON(JSONObject obj) throws JSONException { + String type = obj.getString(jsonTypeKey); + if (WbItemConstant.jsonType.equals(type)) { + return WbItemConstant.fromJSON(obj); + } else if (WbItemVariable.jsonType.equals(type)) { + return WbItemVariable.fromJSON(obj); + } + return null; + } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java index e6831108a..d10d33d3e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java @@ -3,6 +3,7 @@ package org.openrefine.wikidata.schema; import java.util.Properties; import org.json.JSONException; +import org.json.JSONObject; import org.json.JSONWriter; import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; @@ -10,7 +11,7 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import com.google.refine.model.Cell; import com.google.refine.model.ReconCandidate; -public class WbItemVariable implements WbItemExpr { +public class WbItemVariable extends WbItemExpr { /* An item that depends on a reconciled value in a column */ private String columnName; @@ -20,13 +21,14 @@ public class WbItemVariable implements WbItemExpr { } @Override - public void write(JSONWriter writer, Properties options) + public void writeFields(JSONWriter writer, Properties options) throws JSONException { - writer.object(); writer.key("columnName"); writer.value(columnName); - writer.endObject(); - + } + + public static WbItemVariable fromJSON(JSONObject obj) throws JSONException { + return new WbItemVariable(obj.getString("columnName")); } @Override diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java index 2a3724428..1047cb2c1 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java @@ -3,14 +3,17 @@ package org.openrefine.wikidata.schema; import java.util.Properties; import org.json.JSONException; +import org.json.JSONObject; import org.json.JSONWriter; import org.wikidata.wdtk.datamodel.implementation.PropertyIdValueImpl; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; -public class WbPropConstant implements WbPropExpr { +public class WbPropConstant extends WbPropExpr { /* A constant property, that does not change depending on the row */ + public static final String jsonType = "wbpropconstant"; + private String pid; public WbPropConstant(String pid) { @@ -18,12 +21,14 @@ public class WbPropConstant implements WbPropExpr { } @Override - public void write(JSONWriter writer, Properties options) + public void writeFields(JSONWriter writer, Properties options) throws JSONException { - writer.object(); writer.key("pid"); writer.value(pid); - writer.endObject(); + } + + public static WbPropConstant fromJSON(JSONObject obj) throws JSONException { + return new WbPropConstant(obj.getString("pid")); } @Override diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropExpr.java index fdf7a7300..698695819 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropExpr.java @@ -3,9 +3,8 @@ package org.openrefine.wikidata.schema; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; -public interface WbPropExpr extends WbValueExpr { +public abstract class WbPropExpr extends WbValueExpr { /* An expression that represents a property */ - public PropertyIdValue evaluate(ExpressionContext ctxt); - + public abstract PropertyIdValue evaluate(ExpressionContext ctxt); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java index d282589ac..b9d485528 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java @@ -3,6 +3,7 @@ package org.openrefine.wikidata.schema; import java.util.Properties; import org.json.JSONException; +import org.json.JSONObject; import org.json.JSONWriter; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue; @@ -33,6 +34,24 @@ public class WbSnakExpr implements Jsonizable { writer.endObject(); } + public static WbSnakExpr fromJSON(JSONObject obj) throws JSONException { + JSONObject propObj = obj.getJSONObject("prop"); + WbPropExpr propExpr = WbPropConstant.fromJSON(propObj); + JSONObject valueObj = obj.getJSONObject("value"); + String type = valueObj.getString(WbValueExpr.jsonTypeKey); + WbValueExpr valueExpr = null; + if (WbPropConstant.jsonType.equals(type)) { + valueExpr = WbPropConstant.fromJSON(valueObj); + } else if (WbItemConstant.jsonType.equals(type)) { + valueExpr = WbItemConstant.fromJSON(valueObj); + } else if (WbItemVariable.jsonType.equals(type)) { + valueExpr = WbItemVariable.fromJSON(valueObj); + } else if (WbStringConstant.jsonType.equals(type)) { + valueExpr = WbStringConstant.fromJSON(valueObj); + } + return new WbSnakExpr(propExpr, valueExpr); + } + public Snak evaluate(ExpressionContext ctxt) { PropertyIdValue propertyId = propExpr.evaluate(ctxt); Value value = valueExpr.evaluate(ctxt); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java index 7844564c5..f021f51c3 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java @@ -3,13 +3,16 @@ package org.openrefine.wikidata.schema; import java.util.Properties; import org.json.JSONException; +import org.json.JSONObject; import org.json.JSONWriter; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.implementation.StringValueImpl; import org.wikidata.wdtk.datamodel.interfaces.Value; -public class WbStringConstant implements WbValueExpr { +public class WbStringConstant extends WbValueExpr { + + public static final String jsonType = "wbstringconstant"; private String value; @@ -17,13 +20,14 @@ public class WbStringConstant implements WbValueExpr { this.value = value; } - @Override - public void write(JSONWriter writer, Properties options) + public void writeFields(JSONWriter writer, Properties options) throws JSONException { - writer.object(); writer.key("value"); writer.value(value); - writer.endObject(); + } + + public static WbStringConstant fromJSON(JSONObject obj) throws JSONException { + return new WbStringConstant(obj.getString("value")); } @Override diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java index 1e3970fec..f91a8b2b5 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java @@ -2,15 +2,17 @@ package org.openrefine.wikidata.schema; import org.wikidata.wdtk.datamodel.interfaces.Value; -import com.google.refine.Jsonizable; - -public interface WbValueExpr extends Jsonizable { +public abstract class WbValueExpr extends BiJsonizable { /* An expression that represents a Wikibase value, * i.e. anything that can be on the right-hand side * of a statement. */ - public Value evaluate(ExpressionContext ctxt); - + + /* + * Evaluates the value expression in a given context, + * returns a wikibase value suitable to be the target of a claim. + */ + public abstract Value evaluate(ExpressionContext ctxt); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java new file mode 100644 index 000000000..8ef0e8c08 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -0,0 +1,84 @@ +package org.openrefine.wikidata.schema; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/* +import org.deri.grefine.rdf.ResourceNode.RdfType; +import org.deri.grefine.rdf.app.ApplicationContext; +import org.deri.grefine.rdf.vocab.PrefixExistException; +import org.deri.grefine.rdf.vocab.Vocabulary; +import org.deri.grefine.rdf.vocab.VocabularyIndexException; */ +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.refine.model.OverlayModel; +import com.google.refine.model.Project; + +public class WikibaseSchema implements OverlayModel { + + final static Logger logger = LoggerFactory.getLogger("RdfSchema"); + + final protected List changeExprs = new ArrayList(); + + protected String baseUri; + + @Override + public void onBeforeSave(Project project) { + } + + @Override + public void onAfterSave(Project project) { + } + + @Override + public void dispose(Project project) { + + } + + public void setBaseUri(String baseUri) { + this.baseUri = baseUri; + } + + public WikibaseSchema(){ + + } + + + public String getBaseUri() { + return baseUri; + } + + public List getChangeExpressions() { + return changeExprs; + } + + static public WikibaseSchema reconstruct(JSONObject o) throws JSONException { + JSONArray changeArr = o.getJSONArray("changes"); + WikibaseSchema schema = new WikibaseSchema(); + for (int i = 0; i != changeArr.length(); i++) { + WbChangeExpr changeExpr = WbClaimExpr.fromJSON(changeArr.getJSONObject(i)); + schema.changeExprs.add(changeExpr); + } + return schema; + } + + @Override + public void write(JSONWriter writer, Properties options) + throws JSONException { + writer.array(); + for (WbChangeExpr changeExpr : changeExprs) { + changeExpr.write(writer, options); + } + writer.endArray(); + } + + static public WikibaseSchema load(Project project, JSONObject obj) throws Exception { + return reconstruct(obj); + } +} \ No newline at end of file