From 9fcdbc31e93b0dda34c3033e08348ee8d1e2addc Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Wed, 6 Sep 2017 11:58:30 +0100 Subject: [PATCH] Initial support for item creations --- extensions/wikidata/build.xml | 11 +--- .../dialogs/schema-alignment-dialog.js | 12 ++--- .../exporters/QuickStatementsExporter.java | 53 ++++++++----------- .../wikidata/schema/WbChangeExpr.java | 10 ---- ...mentsExpr.java => WbItemDocumentExpr.java} | 23 ++++---- .../wikidata/schema/WbItemVariable.java | 13 +++-- .../wikidata/schema/WikibaseSchema.java | 30 +++++------ 7 files changed, 68 insertions(+), 84 deletions(-) delete mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java rename extensions/wikidata/src/org/openrefine/wikidata/schema/{WbItemStatementsExpr.java => WbItemDocumentExpr.java} (66%) diff --git a/extensions/wikidata/build.xml b/extensions/wikidata/build.xml index fa5f142b1..65e9937ec 100644 --- a/extensions/wikidata/build.xml +++ b/extensions/wikidata/build.xml @@ -33,7 +33,6 @@ - @@ -52,7 +51,7 @@ - + @@ -71,14 +70,6 @@ - - - - - - - - diff --git a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js index 2e6935524..4f73e336e 100644 --- a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js +++ b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js @@ -97,18 +97,18 @@ SchemaAlignmentDialog.launch = function(onDone) { } SchemaAlignmentDialog._reset = function(schema, initial) { - this._originalSchema = schema || { changes: [] }; + this._originalSchema = schema || { itemDocuments: [] }; this._schema = cloneDeep(this._originalSchema); // this is what can be munched on $('#schema-alignment-statements-container').empty(); - if (this._schema && this._schema.changes) { - for(var i = 0; i != this._schema.changes.length; i++) { - this._addItem(this._schema.changes[i]); + if (this._schema && this._schema.itemDocuments) { + for(var i = 0; i != this._schema.itemDocuments.length; i++) { + this._addItem(this._schema.itemDocuments[i]); } } - if (!this._schema.changes.length) { + if (!this._schema.itemDocuments.length) { // this._addItem(); } this.preview(initial); @@ -501,7 +501,7 @@ SchemaAlignmentDialog.getJSON = function() { list.push(SchemaAlignmentDialog._itemToJSON($(this))); }); return { - 'changes': list, + 'itemDocuments': list, 'wikibasePrefix': this._wikibasePrefix, }; }; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java index b7b2a9f64..ebc5890ef 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java @@ -1,42 +1,14 @@ package org.openrefine.wikidata.exporters; import java.io.IOException; -import java.io.OutputStream; import java.io.Writer; -import java.net.URI; import java.util.List; import java.util.Properties; -/* -import org.deri.grefine.rdf.Node; -import org.deri.grefine.rdf.RdfSchema; -import org.deri.grefine.rdf.Util; -import org.deri.grefine.rdf.app.ApplicationContext; -import org.deri.grefine.rdf.vocab.Vocabulary; -import org.deri.grefine.rdf.vocab.VocabularyIndexException; -import org.openrdf.model.*; -import org.openrdf.repository.Repository; -import org.openrdf.repository.RepositoryConnection; -import org.openrdf.repository.RepositoryException; -import org.openrdf.repository.sail.SailRepository; -import org.openrdf.rio.RDFFormat; -import org.openrdf.rio.RDFHandlerException; -import org.openrdf.rio.RDFWriter; -import org.openrdf.rio.Rio; -import org.openrdf.sail.memory.MemoryStore; - -import info.aduna.iteration.CloseableIteration; -*/ - import com.google.refine.browsing.Engine; -import com.google.refine.browsing.FilteredRows; -import com.google.refine.browsing.RowVisitor; import com.google.refine.exporters.WriterExporter; import com.google.refine.model.Project; -import com.google.refine.model.Row; -import org.openrefine.wikidata.schema.WbChangeExpr; -import org.openrefine.wikidata.schema.WbItemStatementsExpr; import org.openrefine.wikidata.schema.WikibaseSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +16,8 @@ import org.wikidata.wdtk.datamodel.interfaces.Claim; import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue; import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; +import org.wikidata.wdtk.datamodel.interfaces.ItemDocument; +import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; import org.wikidata.wdtk.datamodel.interfaces.Statement; @@ -77,8 +51,17 @@ public class QuickStatementsExporter implements WriterExporter { } public void translateSchema(Project project, WikibaseSchema schema, Writer writer) throws IOException { - List statements = schema.evaluate(project); - for (StatementGroup group : statements) { + List items = schema.evaluate(project); + for (ItemDocument item : items) { + translateItem(item, writer); + } + } + + protected void translateItem(ItemDocument item, Writer writer) throws IOException { + if (item.getItemId().equals(ItemIdValue.NULL)) { + writer.write("CREATE\n"); + } + for (StatementGroup group : item.getStatementGroups()) { translateStatementGroup(group, writer); } } @@ -93,10 +76,15 @@ public class QuickStatementsExporter implements WriterExporter { protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException { Claim claim = statement.getClaim(); String qid = claim.getSubject().getId(); + if (claim.getSubject().equals(ItemIdValue.NULL)) { + qid = "LAST"; + } Value val = claim.getValue(); ValueVisitor vv = new ValuePrinter(); String targetValue = val.accept(vv); - writer.write(qid + "\t" + pid + "\t" + targetValue + "\n"); + if (targetValue != null) { + writer.write(qid + "\t" + pid + "\t" + targetValue + "\n"); + } } class ValuePrinter implements ValueVisitor { @@ -109,6 +97,9 @@ public class QuickStatementsExporter implements WriterExporter { @Override public String visit(EntityIdValue value) { + if (value.equals(ItemIdValue.NULL)) { + return null; + } return value.getId(); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java deleted file mode 100644 index 2f9ae8eb3..000000000 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbChangeExpr.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.openrefine.wikidata.schema; - - -public abstract class WbChangeExpr extends BiJsonizable { - /* Represents a change on an item: adding a statement, - * adding a label, adding a sitelink… - */ - - -} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java similarity index 66% rename from extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java rename to extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java index 88d66b5ee..80df0151a 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemDocumentExpr.java @@ -8,18 +8,21 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import org.wikidata.wdtk.datamodel.helpers.ItemDocumentBuilder; +import org.wikidata.wdtk.datamodel.interfaces.ItemDocument; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; +import org.wikidata.wdtk.datamodel.interfaces.Statement; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; -public class WbItemStatementsExpr extends WbChangeExpr { +public class WbItemDocumentExpr extends BiJsonizable { - public static final String jsonType = "wbitemstatements"; + public static final String jsonType = "wbitemdocument"; private WbItemExpr subjectExpr; private List statementGroupExprs; - public WbItemStatementsExpr(WbItemExpr subjectExpr, List statementGroupExprs) { + public WbItemDocumentExpr(WbItemExpr subjectExpr, List statementGroupExprs) { this.subjectExpr = subjectExpr; this.statementGroupExprs = statementGroupExprs; } @@ -37,25 +40,27 @@ public class WbItemStatementsExpr extends WbChangeExpr { writer.endArray(); } - public static WbItemStatementsExpr fromJSON(JSONObject obj) throws JSONException { + public static WbItemDocumentExpr fromJSON(JSONObject obj) throws JSONException { JSONObject subjectObj = obj.getJSONObject("subject"); JSONArray statementsArr = obj.getJSONArray("statementGroups"); List statementExprs = new ArrayList(); for (int i = 0; i != statementsArr.length(); i++) { statementExprs.add(WbStatementGroupExpr.fromJSON(statementsArr.getJSONObject(i))); } - return new WbItemStatementsExpr( + return new WbItemDocumentExpr( WbItemExpr.fromJSON(subjectObj), statementExprs); } - public List evaluate(ExpressionContext ctxt) throws SkipStatementException { - List results = new ArrayList(statementGroupExprs.size()); + public ItemDocument evaluate(ExpressionContext ctxt) throws SkipStatementException { ItemIdValue subjectId = subjectExpr.evaluate(ctxt); + ItemDocumentBuilder builder = ItemDocumentBuilder.forItemId(subjectId); for(WbStatementGroupExpr expr : statementGroupExprs) { - results.add(expr.evaluate(ctxt, subjectId)); + for(Statement s : expr.evaluate(ctxt, subjectId).getStatements()) { + builder.withStatement(s); + } } - return results; + return builder.build(); } public String getJsonType() { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java index 1f8d9fbb8..9741afe8d 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java @@ -5,10 +5,12 @@ import java.util.Properties; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.implementation.ItemIdValueImpl; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; import com.google.refine.model.Cell; +import com.google.refine.model.Recon; import com.google.refine.model.ReconCandidate; public class WbItemVariable extends WbItemExpr { @@ -36,9 +38,14 @@ public class WbItemVariable extends WbItemExpr { @Override public ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException { Cell cell = ctxt.getCellByName(columnName); - if (cell != null && cell.recon != null && cell.recon.match != null) { - ReconCandidate match = cell.recon.match; - return ItemIdValueImpl.create(match.id, ctxt.getBaseIRI()); + if (cell != null && cell.recon != null) { + Recon recon = cell.recon; + if (recon.judgment == Recon.Judgment.Matched && cell.recon.match != null) { + ReconCandidate match = cell.recon.match; + return Datamodel.makeItemIdValue(match.id, ctxt.getBaseIRI()); + } else if (recon.judgment == Recon.Judgment.New) { + return ItemIdValue.NULL; + } } throw new SkipStatementException(); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index 798a1de71..ace646174 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -10,6 +10,7 @@ import org.json.JSONObject; import org.json.JSONWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.wikidata.wdtk.datamodel.interfaces.ItemDocument; import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; import com.google.refine.model.OverlayModel; @@ -21,7 +22,7 @@ public class WikibaseSchema implements OverlayModel { final static Logger logger = LoggerFactory.getLogger("RdfSchema"); - final protected List changeExprs = new ArrayList(); + final protected List itemDocumentExprs = new ArrayList(); protected String baseUri = "http://www.wikidata.org/entity/"; @@ -51,17 +52,16 @@ public class WikibaseSchema implements OverlayModel { return baseUri; } - public List getChangeExpressions() { - return changeExprs; + public List getItemDocumentExpressions() { + return itemDocumentExprs; } - public List evaluate(ExpressionContext ctxt) { - List result = new ArrayList(); - for (WbChangeExpr changeExpr : changeExprs) { - WbItemStatementsExpr expr = (WbItemStatementsExpr)changeExpr; + public List evaluate(ExpressionContext ctxt) { + List result = new ArrayList(); + for (WbItemDocumentExpr expr : itemDocumentExprs) { try { - result.addAll(expr.evaluate(ctxt)); + result.add(expr.evaluate(ctxt)); } catch (SkipStatementException e) { continue; } @@ -69,8 +69,8 @@ public class WikibaseSchema implements OverlayModel { return result; } - public List evaluate(Project project) { - List result = new ArrayList(); + public List evaluate(Project project) { + List result = new ArrayList(); for (Row row : project.rows) { ExpressionContext ctxt = new ExpressionContext(baseUri, row, project.columnModel); result.addAll(evaluate(ctxt)); @@ -79,11 +79,11 @@ public class WikibaseSchema implements OverlayModel { } static public WikibaseSchema reconstruct(JSONObject o) throws JSONException { - JSONArray changeArr = o.getJSONArray("changes"); + JSONArray changeArr = o.getJSONArray("itemDocuments"); WikibaseSchema schema = new WikibaseSchema(); for (int i = 0; i != changeArr.length(); i++) { - WbChangeExpr changeExpr = WbItemStatementsExpr.fromJSON(changeArr.getJSONObject(i)); - schema.changeExprs.add(changeExpr); + WbItemDocumentExpr changeExpr = WbItemDocumentExpr.fromJSON(changeArr.getJSONObject(i)); + schema.itemDocumentExprs.add(changeExpr); } return schema; } @@ -92,9 +92,9 @@ public class WikibaseSchema implements OverlayModel { public void write(JSONWriter writer, Properties options) throws JSONException { writer.object(); - writer.key("changes"); + writer.key("itemDocuments"); writer.array(); - for (WbChangeExpr changeExpr : changeExprs) { + for (WbItemDocumentExpr changeExpr : itemDocumentExprs) { changeExpr.write(writer, options); } writer.endArray();