s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial support for item creations

Browse Source
This commit is contained in:
Antonin Delpeuch 2017-09-06 11:58:30 +01:00
parent f763e8c6ad
commit 9fcdbc31e9
7 changed files with 68 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
extensions/wikidata/build.xml
View File

@ -33,7 +33,6 @@
<property name="server.lib.dir" value="${server.dir}/lib" /> <property name="server.lib.dir" value="${server.dir}/lib" />
<property name="src.dir" value="${basedir}/src" /> <property name="src.dir" value="${basedir}/src" />
<property name="rsrc.dir" value="${basedir}/resources" />
<property name="module.dir" value="${basedir}/module" /> <property name="module.dir" value="${basedir}/module" />
<property name="modinf.dir" value="${module.dir}/MOD-INF" /> <property name="modinf.dir" value="${module.dir}/MOD-INF" />
<property name="lib.dir" value="${modinf.dir}/lib" /> <property name="lib.dir" value="${modinf.dir}/lib" />
@ -52,7 +51,7 @@
<pathelement path="${refine.classes.dir}"/> <pathelement path="${refine.classes.dir}"/>
</path> </path>
<target name="build_java" depends="copyfiles,copyresources"> <target name="build_java" depends="copyfiles">
<mkdir dir="${classes.dir}" /> <mkdir dir="${classes.dir}" />
<javac destdir="${classes.dir}" debug="true" includeAntRuntime="no"> <javac destdir="${classes.dir}" debug="true" includeAntRuntime="no">
<src path="${src.dir}"/> <src path="${src.dir}"/>
@ -71,14 +70,6 @@
</copy> </copy>
</target> </target>
<target name="copyresources">
<copy todir="${classes.dir}">
<fileset dir="${src.dir}">
<include name="**/*.tsv"/>
</fileset>
</copy>
</target>
<target name="clean"> <target name="clean">
<delete dir="${classes.dir}" /> <delete dir="${classes.dir}" />
</target> </target>

12
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js
View File

@ -97,18 +97,18 @@ SchemaAlignmentDialog.launch = function(onDone) {
} }
SchemaAlignmentDialog._reset = function(schema, initial) { SchemaAlignmentDialog._reset = function(schema, initial) {
this._originalSchema = schema || { changes: [] }; this._originalSchema = schema || { itemDocuments: [] };
this._schema = cloneDeep(this._originalSchema); // this is what can be munched on this._schema = cloneDeep(this._originalSchema); // this is what can be munched on
$('#schema-alignment-statements-container').empty(); $('#schema-alignment-statements-container').empty();
if (this._schema && this._schema.changes) { if (this._schema && this._schema.itemDocuments) {
for(var i = 0; i != this._schema.changes.length; i++) { for(var i = 0; i != this._schema.itemDocuments.length; i++) {
this._addItem(this._schema.changes[i]); this._addItem(this._schema.itemDocuments[i]);
} }
} }
if (!this._schema.changes.length) { if (!this._schema.itemDocuments.length) {
// this._addItem(); // this._addItem();
} }
this.preview(initial); this.preview(initial);
@ -501,7 +501,7 @@ SchemaAlignmentDialog.getJSON = function() {
list.push(SchemaAlignmentDialog._itemToJSON($(this))); list.push(SchemaAlignmentDialog._itemToJSON($(this)));
}); });
return { return {
'changes': list, 'itemDocuments': list,
'wikibasePrefix': this._wikibasePrefix, 'wikibasePrefix': this._wikibasePrefix,
}; };
}; };

53
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java
View File

@ -1,42 +1,14 @@
package org.openrefine.wikidata.exporters; package org.openrefine.wikidata.exporters;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer; import java.io.Writer;
import java.net.URI;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
/*
import org.deri.grefine.rdf.Node;
import org.deri.grefine.rdf.RdfSchema;
import org.deri.grefine.rdf.Util;
import org.deri.grefine.rdf.app.ApplicationContext;
import org.deri.grefine.rdf.vocab.Vocabulary;
import org.deri.grefine.rdf.vocab.VocabularyIndexException;
import org.openrdf.model.*;
import org.openrdf.repository.Repository;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandlerException;
import org.openrdf.rio.RDFWriter;
import org.openrdf.rio.Rio;
import org.openrdf.sail.memory.MemoryStore;
import info.aduna.iteration.CloseableIteration;
*/
import com.google.refine.browsing.Engine; import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.exporters.WriterExporter; import com.google.refine.exporters.WriterExporter;
import com.google.refine.model.Project; import com.google.refine.model.Project;
import com.google.refine.model.Row;
import org.openrefine.wikidata.schema.WbChangeExpr;
import org.openrefine.wikidata.schema.WbItemStatementsExpr;
import org.openrefine.wikidata.schema.WikibaseSchema; import org.openrefine.wikidata.schema.WikibaseSchema;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -44,6 +16,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue; import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.Statement;
@ -77,8 +51,17 @@ public class QuickStatementsExporter implements WriterExporter {
} }
public void translateSchema(Project project, WikibaseSchema schema, Writer writer) throws IOException { public void translateSchema(Project project, WikibaseSchema schema, Writer writer) throws IOException {
List<StatementGroup> statements = schema.evaluate(project); List<ItemDocument> items = schema.evaluate(project);
for (StatementGroup group : statements) { for (ItemDocument item : items) {
translateItem(item, writer);
}
}
protected void translateItem(ItemDocument item, Writer writer) throws IOException {
if (item.getItemId().equals(ItemIdValue.NULL)) {
writer.write("CREATE\n");
}
for (StatementGroup group : item.getStatementGroups()) {
translateStatementGroup(group, writer); translateStatementGroup(group, writer);
} }
} }
@ -93,10 +76,15 @@ public class QuickStatementsExporter implements WriterExporter {
protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException { protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException {
Claim claim = statement.getClaim(); Claim claim = statement.getClaim();
String qid = claim.getSubject().getId(); String qid = claim.getSubject().getId();
if (claim.getSubject().equals(ItemIdValue.NULL)) {
qid = "LAST";
}
Value val = claim.getValue(); Value val = claim.getValue();
ValueVisitor<String> vv = new ValuePrinter(); ValueVisitor<String> vv = new ValuePrinter();
String targetValue = val.accept(vv); String targetValue = val.accept(vv);
writer.write(qid + "\t" + pid + "\t" + targetValue + "\n"); if (targetValue != null) {
writer.write(qid + "\t" + pid + "\t" + targetValue + "\n");
}
} }
class ValuePrinter implements ValueVisitor<String> { class ValuePrinter implements ValueVisitor<String> {
@ -109,6 +97,9 @@ public class QuickStatementsExporter implements WriterExporter {
@Override @Override
public String visit(EntityIdValue value) { public String visit(EntityIdValue value) {
if (value.equals(ItemIdValue.NULL)) {
return null;
}
return value.getId(); return value.getId();
} }

10
extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java
View File

@ -1,10 +0,0 @@
package org.openrefine.wikidata.schema;
public abstract class WbChangeExpr extends BiJsonizable {
/* Represents a change on an item: adding a statement,
* adding a label, adding a sitelink…
*/
}

23
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java → extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java
View File

@ -8,18 +8,21 @@ import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.ItemDocumentBuilder;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
public class WbItemStatementsExpr extends WbChangeExpr { public class WbItemDocumentExpr extends BiJsonizable {
public static final String jsonType = "wbitemstatements"; public static final String jsonType = "wbitemdocument";
private WbItemExpr subjectExpr; private WbItemExpr subjectExpr;
private List<WbStatementGroupExpr> statementGroupExprs; private List<WbStatementGroupExpr> statementGroupExprs;
public WbItemStatementsExpr(WbItemExpr subjectExpr, List<WbStatementGroupExpr> statementGroupExprs) { public WbItemDocumentExpr(WbItemExpr subjectExpr, List<WbStatementGroupExpr> statementGroupExprs) {
this.subjectExpr = subjectExpr; this.subjectExpr = subjectExpr;
this.statementGroupExprs = statementGroupExprs; this.statementGroupExprs = statementGroupExprs;
} }
@ -37,25 +40,27 @@ public class WbItemStatementsExpr extends WbChangeExpr {
writer.endArray(); writer.endArray();
} }
public static WbItemStatementsExpr fromJSON(JSONObject obj) throws JSONException { public static WbItemDocumentExpr fromJSON(JSONObject obj) throws JSONException {
JSONObject subjectObj = obj.getJSONObject("subject"); JSONObject subjectObj = obj.getJSONObject("subject");
JSONArray statementsArr = obj.getJSONArray("statementGroups"); JSONArray statementsArr = obj.getJSONArray("statementGroups");
List<WbStatementGroupExpr> statementExprs = new ArrayList<WbStatementGroupExpr>(); List<WbStatementGroupExpr> statementExprs = new ArrayList<WbStatementGroupExpr>();
for (int i = 0; i != statementsArr.length(); i++) { for (int i = 0; i != statementsArr.length(); i++) {
statementExprs.add(WbStatementGroupExpr.fromJSON(statementsArr.getJSONObject(i))); statementExprs.add(WbStatementGroupExpr.fromJSON(statementsArr.getJSONObject(i)));
} }
return new WbItemStatementsExpr( return new WbItemDocumentExpr(
WbItemExpr.fromJSON(subjectObj), WbItemExpr.fromJSON(subjectObj),
statementExprs); statementExprs);
} }
public List<StatementGroup> evaluate(ExpressionContext ctxt) throws SkipStatementException { public ItemDocument evaluate(ExpressionContext ctxt) throws SkipStatementException {
List<StatementGroup> results = new ArrayList<StatementGroup>(statementGroupExprs.size());
ItemIdValue subjectId = subjectExpr.evaluate(ctxt); ItemIdValue subjectId = subjectExpr.evaluate(ctxt);
ItemDocumentBuilder builder = ItemDocumentBuilder.forItemId(subjectId);
for(WbStatementGroupExpr expr : statementGroupExprs) { for(WbStatementGroupExpr expr : statementGroupExprs) {
results.add(expr.evaluate(ctxt, subjectId)); for(Statement s : expr.evaluate(ctxt, subjectId).getStatements()) {
builder.withStatement(s);
}
} }
return results; return builder.build();
} }
public String getJsonType() { public String getJsonType() {

13
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java
View File

@ -5,10 +5,12 @@ import java.util.Properties;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl; import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import com.google.refine.model.Cell; import com.google.refine.model.Cell;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate; import com.google.refine.model.ReconCandidate;
public class WbItemVariable extends WbItemExpr { public class WbItemVariable extends WbItemExpr {
@ -36,9 +38,14 @@ public class WbItemVariable extends WbItemExpr {
@Override @Override
public ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException { public ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException {
Cell cell = ctxt.getCellByName(columnName); Cell cell = ctxt.getCellByName(columnName);
if (cell != null && cell.recon != null && cell.recon.match != null) { if (cell != null && cell.recon != null) {
ReconCandidate match = cell.recon.match; Recon recon = cell.recon;
return ItemIdValueImpl.create(match.id, ctxt.getBaseIRI()); if (recon.judgment == Recon.Judgment.Matched && cell.recon.match != null) {
ReconCandidate match = cell.recon.match;
return Datamodel.makeItemIdValue(match.id, ctxt.getBaseIRI());
} else if (recon.judgment == Recon.Judgment.New) {
return ItemIdValue.NULL;
}
} }
throw new SkipStatementException(); throw new SkipStatementException();
} }

30
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -10,6 +10,7 @@ import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import com.google.refine.model.OverlayModel; import com.google.refine.model.OverlayModel;
@ -21,7 +22,7 @@ public class WikibaseSchema implements OverlayModel {
final static Logger logger = LoggerFactory.getLogger("RdfSchema"); final static Logger logger = LoggerFactory.getLogger("RdfSchema");
final protected List<WbChangeExpr> changeExprs = new ArrayList<WbChangeExpr>(); final protected List<WbItemDocumentExpr> itemDocumentExprs = new ArrayList<WbItemDocumentExpr>();
protected String baseUri = "http://www.wikidata.org/entity/"; protected String baseUri = "http://www.wikidata.org/entity/";
@ -51,17 +52,16 @@ public class WikibaseSchema implements OverlayModel {
return baseUri; return baseUri;
} }
public List<WbChangeExpr> getChangeExpressions() { public List<WbItemDocumentExpr> getItemDocumentExpressions() {
return changeExprs; return itemDocumentExprs;
} }
public List<StatementGroup> evaluate(ExpressionContext ctxt) { public List<ItemDocument> evaluate(ExpressionContext ctxt) {
List<StatementGroup> result = new ArrayList<StatementGroup>(); List<ItemDocument> result = new ArrayList<ItemDocument>();
for (WbChangeExpr changeExpr : changeExprs) { for (WbItemDocumentExpr expr : itemDocumentExprs) {
WbItemStatementsExpr expr = (WbItemStatementsExpr)changeExpr;
try { try {
result.addAll(expr.evaluate(ctxt)); result.add(expr.evaluate(ctxt));
} catch (SkipStatementException e) { } catch (SkipStatementException e) {
continue; continue;
} }
@ -69,8 +69,8 @@ public class WikibaseSchema implements OverlayModel {
return result; return result;
} }
public List<StatementGroup> evaluate(Project project) { public List<ItemDocument> evaluate(Project project) {
List<StatementGroup> result = new ArrayList<StatementGroup>(); List<ItemDocument> result = new ArrayList<ItemDocument>();
for (Row row : project.rows) { for (Row row : project.rows) {
ExpressionContext ctxt = new ExpressionContext(baseUri, row, project.columnModel); ExpressionContext ctxt = new ExpressionContext(baseUri, row, project.columnModel);
result.addAll(evaluate(ctxt)); result.addAll(evaluate(ctxt));
@ -79,11 +79,11 @@ public class WikibaseSchema implements OverlayModel {
} }
static public WikibaseSchema reconstruct(JSONObject o) throws JSONException { static public WikibaseSchema reconstruct(JSONObject o) throws JSONException {
JSONArray changeArr = o.getJSONArray("changes"); JSONArray changeArr = o.getJSONArray("itemDocuments");
WikibaseSchema schema = new WikibaseSchema(); WikibaseSchema schema = new WikibaseSchema();
for (int i = 0; i != changeArr.length(); i++) { for (int i = 0; i != changeArr.length(); i++) {
WbChangeExpr changeExpr = WbItemStatementsExpr.fromJSON(changeArr.getJSONObject(i)); WbItemDocumentExpr changeExpr = WbItemDocumentExpr.fromJSON(changeArr.getJSONObject(i));
schema.changeExprs.add(changeExpr); schema.itemDocumentExprs.add(changeExpr);
} }
return schema; return schema;
} }
@ -92,9 +92,9 @@ public class WikibaseSchema implements OverlayModel {
public void write(JSONWriter writer, Properties options) public void write(JSONWriter writer, Properties options)
throws JSONException { throws JSONException {
writer.object(); writer.object();
writer.key("changes"); writer.key("itemDocuments");
writer.array(); writer.array();
for (WbChangeExpr changeExpr : changeExprs) { for (WbItemDocumentExpr changeExpr : itemDocumentExprs) {
changeExpr.write(writer, options); changeExpr.write(writer, options);
} }
writer.endArray(); writer.endArray();