s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Wikibase schema serialization and related operation

Browse Source
This commit is contained in:
Antonin Delpeuch 2017-08-28 10:08:24 +01:00
parent ad475849b9
commit 5853cdf10d
13 changed files with 354 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
extensions/wikidata/src/org/openrefine/wikidata/operations/SaveWikibaseSchemaOperation.java Normal file
View File

@ -0,0 +1,138 @@
package org.openrefine.wikidata.operations;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.google.refine.history.Change;
import com.google.refine.history.HistoryEntry;
import com.google.refine.model.AbstractOperation;
import com.google.refine.model.Project;
import com.google.refine.operations.OperationRegistry;
import com.google.refine.util.ParsingUtilities;
import com.google.refine.util.Pool;
public class SaveWikibaseSchemaOperation extends AbstractOperation {
final protected WikibaseSchema _schema;
public SaveWikibaseSchemaOperation(WikibaseSchema schema) {
this._schema = schema;
}
static public AbstractOperation reconstruct(Project project, JSONObject obj)
throws Exception {
return new SaveWikibaseSchemaOperation(WikibaseSchema.reconstruct(obj
.getJSONObject("schema")));
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op");
writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description");
writer.value("Save RDF schema skeleton");
writer.key("schema");
_schema.write(writer, options);
writer.endObject();
}
@Override
protected String getBriefDescription(Project project) {
return "Save Wikibase schema skelton";
}
@Override
protected HistoryEntry createHistoryEntry(Project project,
long historyEntryID) throws Exception {
String description = "Save Wikibase schema skeleton";
Change change = new WikibaseSchemaChange(_schema);
return new HistoryEntry(historyEntryID, project, description,
SaveWikibaseSchemaOperation.this, change);
}
static public class WikibaseSchemaChange implements Change {
final protected WikibaseSchema _newSchema;
protected WikibaseSchema _oldSchema;
public final static String overlayModelKey = "wikibaseSchema";
public WikibaseSchemaChange(WikibaseSchema schema) {
_newSchema = schema;
}
public void apply(Project project) {
synchronized (project) {
_oldSchema = (WikibaseSchema) project.overlayModels.get(overlayModelKey);
project.overlayModels.put(overlayModelKey, _newSchema);
}
}
public void revert(Project project) {
synchronized (project) {
if (_oldSchema == null) {
project.overlayModels.remove(overlayModelKey);
} else {
project.overlayModels.put(overlayModelKey, _oldSchema);
}
}
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("newSchema=");
writeWikibaseSchema(_newSchema, writer);
writer.write('\n');
writer.write("oldSchema=");
writeWikibaseSchema(_oldSchema, writer);
writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool)
throws Exception {
WikibaseSchema oldSchema = null;
WikibaseSchema newSchema = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldSchema".equals(field) && value.length() > 0) {
oldSchema = WikibaseSchema.reconstruct(ParsingUtilities
.evaluateJsonStringToObject(value));
} else if ("newSchema".equals(field) && value.length() > 0) {
newSchema = WikibaseSchema.reconstruct(ParsingUtilities
.evaluateJsonStringToObject(value));
}
}
WikibaseSchemaChange change = new WikibaseSchemaChange(newSchema);
change._oldSchema = oldSchema;
return change;
}
static protected void writeWikibaseSchema(WikibaseSchema s, Writer writer)
throws IOException {
if (s != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
s.write(jsonWriter, new Properties());
} catch (JSONException e) {
e.printStackTrace();
}
}
}
}
}

28
extensions/wikidata/src/org/openrefine/wikidata/schema/BiJsonizable.java Normal file
View File

@ -0,0 +1,28 @@
package org.openrefine.wikidata.schema;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONWriter;
import com.google.refine.Jsonizable;
public abstract class BiJsonizable implements Jsonizable {
public static String jsonType;
public static final String jsonTypeKey = "@type";
public abstract void writeFields(JSONWriter writer, Properties options)
throws JSONException;
@Override
public void write(JSONWriter writer, Properties options) throws JSONException {
writer.object();
writer.key(jsonTypeKey);
writer.value(jsonType);
writeFields(writer, options);
writer.endObject();
}
}

6
extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java
View File

@ -1,10 +1,10 @@
package org.openrefine.wikidata.schema;
import com.google.refine.Jsonizable;
public interface WbChangeExpr extends Jsonizable {
public abstract class WbChangeExpr extends BiJsonizable {
/* Represents a change on an item: adding a statement,
* adding a label, adding a sitelink…
*/
}

31
extensions/wikidata/src/org/openrefine/wikidata/schema/WbClaimExpr.java
View File

@ -6,7 +6,9 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
@ -16,18 +18,26 @@ import org.wikidata.wdtk.datamodel.interfaces.SnakGroup;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public class WbClaimExpr implements WbChangeExpr {
public class WbClaimExpr extends WbChangeExpr {
public static String jsonType = "wbclaimexpr";
private WbItemExpr subjectExpr;
private WbSnakExpr mainSnakExpr;
private List<WbSnakExpr> qualifierExprs;
// TODO: references
public WbClaimExpr(WbItemExpr subjectExpr,
WbSnakExpr mainSnakExpr,
List<WbSnakExpr> qualifierExprs) {
this.subjectExpr = subjectExpr;
this.mainSnakExpr = mainSnakExpr;
this.qualifierExprs = qualifierExprs;
}
@Override
public void write(JSONWriter writer, Properties options)
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("subject");
subjectExpr.write(writer, options);
writer.key("mainsnak");
@ -38,7 +48,20 @@ public class WbClaimExpr implements WbChangeExpr {
expr.write(writer, options);
}
writer.endArray();
writer.endObject();
}
public static WbClaimExpr fromJSON(JSONObject obj) throws JSONException {
JSONObject subjObj = obj.getJSONObject("subject");
JSONObject mainSnakObj = obj.getJSONObject("mainsnak");
JSONArray qualifiersArr = obj.getJSONArray("qualifiers");
List<WbSnakExpr> qualifierExprs = new ArrayList<WbSnakExpr>();
for (int i = 0; i != qualifiersArr.length(); i++) {
qualifierExprs.add(WbSnakExpr.fromJSON(qualifiersArr.getJSONObject(i)));
}
return new WbClaimExpr(
WbItemExpr.fromJSON(subjObj),
WbSnakExpr.fromJSON(mainSnakObj),
qualifierExprs);
}
public static List<SnakGroup> groupSnaks(List<Snak> snaks) {

11
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemConstant.java
View File

@ -3,12 +3,13 @@ package org.openrefine.wikidata.schema;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
public class WbItemConstant implements WbItemExpr {
public class WbItemConstant extends WbItemExpr {
/* Represents an item that does not vary,
* it is independent of the row. */
@ -21,14 +22,16 @@ public class WbItemConstant implements WbItemExpr {
}
@Override
public void write(JSONWriter writer, Properties options)
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("qid");
writer.value(qid);
writer.key("label");
writer.value(label);
writer.endObject();
}
public static WbItemConstant fromJSON(JSONObject obj) throws JSONException {
return new WbItemConstant(obj.getString("qid"), obj.getString("label"));
}
@Override

16
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java
View File

@ -1,9 +1,21 @@
package org.openrefine.wikidata.schema;
import org.json.JSONException;
import org.json.JSONObject;
import org.openrefine.wikidata.schema.ExpressionContext;
import org.openrefine.wikidata.schema.WbValueExpr;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
public interface WbItemExpr extends WbValueExpr {
public ItemIdValue evaluate(ExpressionContext ctxt);
public abstract class WbItemExpr extends WbValueExpr {
public abstract ItemIdValue evaluate(ExpressionContext ctxt);
public static WbItemExpr fromJSON(JSONObject obj) throws JSONException {
String type = obj.getString(jsonTypeKey);
if (WbItemConstant.jsonType.equals(type)) {
return WbItemConstant.fromJSON(obj);
} else if (WbItemVariable.jsonType.equals(type)) {
return WbItemVariable.fromJSON(obj);
}
return null;
}
}

12
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java
View File

@ -3,6 +3,7 @@ package org.openrefine.wikidata.schema;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
@ -10,7 +11,7 @@ import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import com.google.refine.model.Cell;
import com.google.refine.model.ReconCandidate;
public class WbItemVariable implements WbItemExpr {
public class WbItemVariable extends WbItemExpr {
/* An item that depends on a reconciled value in a column */
private String columnName;
@ -20,13 +21,14 @@ public class WbItemVariable implements WbItemExpr {
}
@Override
public void write(JSONWriter writer, Properties options)
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("columnName");
writer.value(columnName);
writer.endObject();
}
public static WbItemVariable fromJSON(JSONObject obj) throws JSONException {
return new WbItemVariable(obj.getString("columnName"));
}
@Override

13
extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropConstant.java
View File

@ -3,14 +3,17 @@ package org.openrefine.wikidata.schema;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.implementation.PropertyIdValueImpl;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public class WbPropConstant implements WbPropExpr {
public class WbPropConstant extends WbPropExpr {
/* A constant property, that does not change depending on the row */
public static final String jsonType = "wbpropconstant";
private String pid;
public WbPropConstant(String pid) {
@ -18,12 +21,14 @@ public class WbPropConstant implements WbPropExpr {
}
@Override
public void write(JSONWriter writer, Properties options)
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("pid");
writer.value(pid);
writer.endObject();
}
public static WbPropConstant fromJSON(JSONObject obj) throws JSONException {
return new WbPropConstant(obj.getString("pid"));
}
@Override

5
extensions/wikidata/src/org/openrefine/wikidata/schema/WbPropExpr.java
View File

@ -3,9 +3,8 @@ package org.openrefine.wikidata.schema;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
public interface WbPropExpr extends WbValueExpr {
public abstract class WbPropExpr extends WbValueExpr {
/* An expression that represents a property */
public PropertyIdValue evaluate(ExpressionContext ctxt);
public abstract PropertyIdValue evaluate(ExpressionContext ctxt);
}

19
extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java
View File

@ -3,6 +3,7 @@ package org.openrefine.wikidata.schema;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
@ -33,6 +34,24 @@ public class WbSnakExpr implements Jsonizable {
writer.endObject();
}
public static WbSnakExpr fromJSON(JSONObject obj) throws JSONException {
JSONObject propObj = obj.getJSONObject("prop");
WbPropExpr propExpr = WbPropConstant.fromJSON(propObj);
JSONObject valueObj = obj.getJSONObject("value");
String type = valueObj.getString(WbValueExpr.jsonTypeKey);
WbValueExpr valueExpr = null;
if (WbPropConstant.jsonType.equals(type)) {
valueExpr = WbPropConstant.fromJSON(valueObj);
} else if (WbItemConstant.jsonType.equals(type)) {
valueExpr = WbItemConstant.fromJSON(valueObj);
} else if (WbItemVariable.jsonType.equals(type)) {
valueExpr = WbItemVariable.fromJSON(valueObj);
} else if (WbStringConstant.jsonType.equals(type)) {
valueExpr = WbStringConstant.fromJSON(valueObj);
}
return new WbSnakExpr(propExpr, valueExpr);
}
public Snak evaluate(ExpressionContext ctxt) {
PropertyIdValue propertyId = propExpr.evaluate(ctxt);
Value value = valueExpr.evaluate(ctxt);

14
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringConstant.java
View File

@ -3,13 +3,16 @@ package org.openrefine.wikidata.schema;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.implementation.StringValueImpl;
import org.wikidata.wdtk.datamodel.interfaces.Value;
public class WbStringConstant implements WbValueExpr {
public class WbStringConstant extends WbValueExpr {
public static final String jsonType = "wbstringconstant";
private String value;
@ -17,13 +20,14 @@ public class WbStringConstant implements WbValueExpr {
this.value = value;
}
@Override
public void write(JSONWriter writer, Properties options)
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("value");
writer.value(value);
writer.endObject();
}
public static WbStringConstant fromJSON(JSONObject obj) throws JSONException {
return new WbStringConstant(obj.getString("value"));
}
@Override

12
extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java
View File

@ -2,15 +2,17 @@ package org.openrefine.wikidata.schema;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import com.google.refine.Jsonizable;
public interface WbValueExpr extends Jsonizable {
public abstract class WbValueExpr extends BiJsonizable {
/* An expression that represents a Wikibase value,
* i.e. anything that can be on the right-hand side
* of a statement.
*/
public Value evaluate(ExpressionContext ctxt);
/*
* Evaluates the value expression in a given context,
* returns a wikibase value suitable to be the target of a claim.
*/
public abstract Value evaluate(ExpressionContext ctxt);
}

84
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java Normal file
View File

@ -0,0 +1,84 @@
package org.openrefine.wikidata.schema;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
/*
import org.deri.grefine.rdf.ResourceNode.RdfType;
import org.deri.grefine.rdf.app.ApplicationContext;
import org.deri.grefine.rdf.vocab.PrefixExistException;
import org.deri.grefine.rdf.vocab.Vocabulary;
import org.deri.grefine.rdf.vocab.VocabularyIndexException; */
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.refine.model.OverlayModel;
import com.google.refine.model.Project;
public class WikibaseSchema implements OverlayModel {
final static Logger logger = LoggerFactory.getLogger("RdfSchema");
final protected List<WbChangeExpr> changeExprs = new ArrayList<WbChangeExpr>();
protected String baseUri;
@Override
public void onBeforeSave(Project project) {
}
@Override
public void onAfterSave(Project project) {
}
@Override
public void dispose(Project project) {
}
public void setBaseUri(String baseUri) {
this.baseUri = baseUri;
}
public WikibaseSchema(){
}
public String getBaseUri() {
return baseUri;
}
public List<WbChangeExpr> getChangeExpressions() {
return changeExprs;
}
static public WikibaseSchema reconstruct(JSONObject o) throws JSONException {
JSONArray changeArr = o.getJSONArray("changes");
WikibaseSchema schema = new WikibaseSchema();
for (int i = 0; i != changeArr.length(); i++) {
WbChangeExpr changeExpr = WbClaimExpr.fromJSON(changeArr.getJSONObject(i));
schema.changeExprs.add(changeExpr);
}
return schema;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.array();
for (WbChangeExpr changeExpr : changeExprs) {
changeExpr.write(writer, options);
}
writer.endArray();
}
static public WikibaseSchema load(Project project, JSONObject obj) throws Exception {
return reconstruct(obj);
}
}