s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial support for item creations

Browse Source
This commit is contained in:
Antonin Delpeuch 2017-09-06 11:58:30 +01:00
parent f763e8c6ad
commit 9fcdbc31e9
7 changed files with 68 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
extensions/wikidata/build.xml
View File

@ -33,7 +33,6 @@
<property name="server.lib.dir" value="${server.dir}/lib" />
<property name="src.dir" value="${basedir}/src" />
<property name="rsrc.dir" value="${basedir}/resources" />
<property name="module.dir" value="${basedir}/module" />
<property name="modinf.dir" value="${module.dir}/MOD-INF" />
<property name="lib.dir" value="${modinf.dir}/lib" />
@ -52,7 +51,7 @@
<pathelement path="${refine.classes.dir}"/>
</path>
<target name="build_java" depends="copyfiles,copyresources">
<target name="build_java" depends="copyfiles">
<mkdir dir="${classes.dir}" />
<javac destdir="${classes.dir}" debug="true" includeAntRuntime="no">
<src path="${src.dir}"/>
@ -71,14 +70,6 @@
</copy>
</target>
<target name="copyresources">
<copy todir="${classes.dir}">
<fileset dir="${src.dir}">
<include name="**/*.tsv"/>
</fileset>
</copy>
</target>
<target name="clean">
<delete dir="${classes.dir}" />
</target>

12
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js
View File

@ -97,18 +97,18 @@ SchemaAlignmentDialog.launch = function(onDone) {
}
SchemaAlignmentDialog._reset = function(schema, initial) {
this._originalSchema = schema || { changes: [] };
this._originalSchema = schema || { itemDocuments: [] };
this._schema = cloneDeep(this._originalSchema); // this is what can be munched on
$('#schema-alignment-statements-container').empty();
if (this._schema && this._schema.changes) {
for(var i = 0; i != this._schema.changes.length; i++) {
this._addItem(this._schema.changes[i]);
if (this._schema && this._schema.itemDocuments) {
for(var i = 0; i != this._schema.itemDocuments.length; i++) {
this._addItem(this._schema.itemDocuments[i]);
}
}
if (!this._schema.changes.length) {
if (!this._schema.itemDocuments.length) {
// this._addItem();
}
this.preview(initial);
@ -501,7 +501,7 @@ SchemaAlignmentDialog.getJSON = function() {
list.push(SchemaAlignmentDialog._itemToJSON($(this)));
});
return {
'changes': list,
'itemDocuments': list,
'wikibasePrefix': this._wikibasePrefix,
};
};

53
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java
View File

@ -1,42 +1,14 @@
package org.openrefine.wikidata.exporters;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.net.URI;
import java.util.List;
import java.util.Properties;
/*
import org.deri.grefine.rdf.Node;
import org.deri.grefine.rdf.RdfSchema;
import org.deri.grefine.rdf.Util;
import org.deri.grefine.rdf.app.ApplicationContext;
import org.deri.grefine.rdf.vocab.Vocabulary;
import org.deri.grefine.rdf.vocab.VocabularyIndexException;
import org.openrdf.model.*;
import org.openrdf.repository.Repository;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandlerException;
import org.openrdf.rio.RDFWriter;
import org.openrdf.rio.Rio;
import org.openrdf.sail.memory.MemoryStore;
import info.aduna.iteration.CloseableIteration;
*/
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import org.openrefine.wikidata.schema.WbChangeExpr;
import org.openrefine.wikidata.schema.WbItemStatementsExpr;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -44,6 +16,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
@ -77,8 +51,17 @@ public class QuickStatementsExporter implements WriterExporter {
}
public void translateSchema(Project project, WikibaseSchema schema, Writer writer) throws IOException {
List<StatementGroup> statements = schema.evaluate(project);
for (StatementGroup group : statements) {
List<ItemDocument> items = schema.evaluate(project);
for (ItemDocument item : items) {
translateItem(item, writer);
}
}
protected void translateItem(ItemDocument item, Writer writer) throws IOException {
if (item.getItemId().equals(ItemIdValue.NULL)) {
writer.write("CREATE\n");
}
for (StatementGroup group : item.getStatementGroups()) {
translateStatementGroup(group, writer);
}
}
@ -93,10 +76,15 @@ public class QuickStatementsExporter implements WriterExporter {
protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException {
Claim claim = statement.getClaim();
String qid = claim.getSubject().getId();
if (claim.getSubject().equals(ItemIdValue.NULL)) {
qid = "LAST";
}
Value val = claim.getValue();
ValueVisitor<String> vv = new ValuePrinter();
String targetValue = val.accept(vv);
writer.write(qid + "\t" + pid + "\t" + targetValue + "\n");
if (targetValue != null) {
writer.write(qid + "\t" + pid + "\t" + targetValue + "\n");
}
}
class ValuePrinter implements ValueVisitor<String> {
@ -109,6 +97,9 @@ public class QuickStatementsExporter implements WriterExporter {
@Override
public String visit(EntityIdValue value) {
if (value.equals(ItemIdValue.NULL)) {
return null;
}
return value.getId();
}

10
extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java
View File

@ -1,10 +0,0 @@
package org.openrefine.wikidata.schema;
public abstract class WbChangeExpr extends BiJsonizable {
/* Represents a change on an item: adding a statement,
* adding a label, adding a sitelink…
*/
}

23
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java → extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java
View File

@ -8,18 +8,21 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.ItemDocumentBuilder;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
public class WbItemStatementsExpr extends WbChangeExpr {
public class WbItemDocumentExpr extends BiJsonizable {
public static final String jsonType = "wbitemstatements";
public static final String jsonType = "wbitemdocument";
private WbItemExpr subjectExpr;
private List<WbStatementGroupExpr> statementGroupExprs;
public WbItemStatementsExpr(WbItemExpr subjectExpr, List<WbStatementGroupExpr> statementGroupExprs) {
public WbItemDocumentExpr(WbItemExpr subjectExpr, List<WbStatementGroupExpr> statementGroupExprs) {
this.subjectExpr = subjectExpr;
this.statementGroupExprs = statementGroupExprs;
}
@ -37,25 +40,27 @@ public class WbItemStatementsExpr extends WbChangeExpr {
writer.endArray();
}
public static WbItemStatementsExpr fromJSON(JSONObject obj) throws JSONException {
public static WbItemDocumentExpr fromJSON(JSONObject obj) throws JSONException {
JSONObject subjectObj = obj.getJSONObject("subject");
JSONArray statementsArr = obj.getJSONArray("statementGroups");
List<WbStatementGroupExpr> statementExprs = new ArrayList<WbStatementGroupExpr>();
for (int i = 0; i != statementsArr.length(); i++) {
statementExprs.add(WbStatementGroupExpr.fromJSON(statementsArr.getJSONObject(i)));
}
return new WbItemStatementsExpr(
return new WbItemDocumentExpr(
WbItemExpr.fromJSON(subjectObj),
statementExprs);
}
public List<StatementGroup> evaluate(ExpressionContext ctxt) throws SkipStatementException {
List<StatementGroup> results = new ArrayList<StatementGroup>(statementGroupExprs.size());
public ItemDocument evaluate(ExpressionContext ctxt) throws SkipStatementException {
ItemIdValue subjectId = subjectExpr.evaluate(ctxt);
ItemDocumentBuilder builder = ItemDocumentBuilder.forItemId(subjectId);
for(WbStatementGroupExpr expr : statementGroupExprs) {
results.add(expr.evaluate(ctxt, subjectId));
for(Statement s : expr.evaluate(ctxt, subjectId).getStatements()) {
builder.withStatement(s);
}
}
return results;
return builder.build();
}
public String getJsonType() {

13
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java
View File

@ -5,10 +5,12 @@ import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import com.google.refine.model.Cell;
import com.google.refine.model.Recon;
import com.google.refine.model.ReconCandidate;
public class WbItemVariable extends WbItemExpr {
@ -36,9 +38,14 @@ public class WbItemVariable extends WbItemExpr {
@Override
public ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException {
Cell cell = ctxt.getCellByName(columnName);
if (cell != null && cell.recon != null && cell.recon.match != null) {
ReconCandidate match = cell.recon.match;
return ItemIdValueImpl.create(match.id, ctxt.getBaseIRI());
if (cell != null && cell.recon != null) {
Recon recon = cell.recon;
if (recon.judgment == Recon.Judgment.Matched && cell.recon.match != null) {
ReconCandidate match = cell.recon.match;
return Datamodel.makeItemIdValue(match.id, ctxt.getBaseIRI());
} else if (recon.judgment == Recon.Judgment.New) {
return ItemIdValue.NULL;
}
}
throw new SkipStatementException();
}

30
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -10,6 +10,7 @@ import org.json.JSONObject;
import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import com.google.refine.model.OverlayModel;
@ -21,7 +22,7 @@ public class WikibaseSchema implements OverlayModel {
final static Logger logger = LoggerFactory.getLogger("RdfSchema");
final protected List<WbChangeExpr> changeExprs = new ArrayList<WbChangeExpr>();
final protected List<WbItemDocumentExpr> itemDocumentExprs = new ArrayList<WbItemDocumentExpr>();
protected String baseUri = "http://www.wikidata.org/entity/";
@ -51,17 +52,16 @@ public class WikibaseSchema implements OverlayModel {
return baseUri;
}
public List<WbChangeExpr> getChangeExpressions() {
return changeExprs;
public List<WbItemDocumentExpr> getItemDocumentExpressions() {
return itemDocumentExprs;
}
public List<StatementGroup> evaluate(ExpressionContext ctxt) {
List<StatementGroup> result = new ArrayList<StatementGroup>();
for (WbChangeExpr changeExpr : changeExprs) {
WbItemStatementsExpr expr = (WbItemStatementsExpr)changeExpr;
public List<ItemDocument> evaluate(ExpressionContext ctxt) {
List<ItemDocument> result = new ArrayList<ItemDocument>();
for (WbItemDocumentExpr expr : itemDocumentExprs) {
try {
result.addAll(expr.evaluate(ctxt));
result.add(expr.evaluate(ctxt));
} catch (SkipStatementException e) {
continue;
}
@ -69,8 +69,8 @@ public class WikibaseSchema implements OverlayModel {
return result;
}
public List<StatementGroup> evaluate(Project project) {
List<StatementGroup> result = new ArrayList<StatementGroup>();
public List<ItemDocument> evaluate(Project project) {
List<ItemDocument> result = new ArrayList<ItemDocument>();
for (Row row : project.rows) {
ExpressionContext ctxt = new ExpressionContext(baseUri, row, project.columnModel);
result.addAll(evaluate(ctxt));
@ -79,11 +79,11 @@ public class WikibaseSchema implements OverlayModel {
}
static public WikibaseSchema reconstruct(JSONObject o) throws JSONException {
JSONArray changeArr = o.getJSONArray("changes");
JSONArray changeArr = o.getJSONArray("itemDocuments");
WikibaseSchema schema = new WikibaseSchema();
for (int i = 0; i != changeArr.length(); i++) {
WbChangeExpr changeExpr = WbItemStatementsExpr.fromJSON(changeArr.getJSONObject(i));
schema.changeExprs.add(changeExpr);
WbItemDocumentExpr changeExpr = WbItemDocumentExpr.fromJSON(changeArr.getJSONObject(i));
schema.itemDocumentExprs.add(changeExpr);
}
return schema;
}
@ -92,9 +92,9 @@ public class WikibaseSchema implements OverlayModel {
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("changes");
writer.key("itemDocuments");
writer.array();
for (WbChangeExpr changeExpr : changeExprs) {
for (WbItemDocumentExpr changeExpr : itemDocumentExprs) {
changeExpr.write(writer, options);
}
writer.endArray();