diff --git a/extensions/wikidata/module/MOD-INF/controller.js b/extensions/wikidata/module/MOD-INF/controller.js index afe80b673..85aece3db 100644 --- a/extensions/wikidata/module/MOD-INF/controller.js +++ b/extensions/wikidata/module/MOD-INF/controller.js @@ -48,6 +48,7 @@ function init() { * Commands */ RefineServlet.registerCommand(module, "save-wikibase-schema", new SaveWikibaseSchemaCommand()); + RefineServlet.registerCommand(module, "preview-wikibase-schema", new PreviewWikibaseSchemaCommand()); /* * Resources diff --git a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html index 9cd431c2e..0545f78f8 100644 --- a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html +++ b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html @@ -24,6 +24,7 @@ diff --git a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js index b1881df6d..141f8f064 100644 --- a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js +++ b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js @@ -111,7 +111,7 @@ SchemaAlignmentDialog._reset = function(schema, initial) { if (!this._schema.changes.length) { // this._addItem(); } - // this.preview(initial); + this.preview(initial); }; SchemaAlignmentDialog._save = function(onDone) { @@ -130,6 +130,7 @@ SchemaAlignmentDialog._save = function(onDone) { theProject.overlayModels.wikibaseSchema = schema; self._elmts.statusIndicator.hide(); + $('.invalid-schema-warning').hide(); self._hasUnsavedChanges = false; if (onDone) onDone(); @@ -208,6 +209,7 @@ SchemaAlignmentDialog._createDialog = function() { var url = ReconciliationManager.ensureDefaultServicePresent(); SchemaAlignmentDialog._reconService = ReconciliationManager.getServiceFromUrl(url); + this.preview(); }; SchemaAlignmentDialog._addItem = function(json) { @@ -342,7 +344,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue) input.suggestP(suggestConfig).bind("fb-select", function(evt, data) { - if (mode === "item") { + if (mode === "item" || mode === "target") { inputContainer.data("jsonValue", { type : "wbitemconstant", qid : data.id, @@ -355,6 +357,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue) label: data.name, }); } + SchemaAlignmentDialog._hasChanged(); }); } @@ -367,6 +370,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue) deleteButton.click(function () { columnDiv.remove(); input.show(); + SchemaAlignmentDialog._hasChanged(); }); }; @@ -387,6 +391,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue) type : "wbitemvariable", columnName: ui.draggable.text(), }); + SchemaAlignmentDialog._hasChanged(); return true; }).on("dropactivate", function(evt, ui) { input.addClass("wbs-accepting-input"); @@ -421,6 +426,7 @@ SchemaAlignmentDialog._removeStatement = function(statement) { if (remainingStatements === 0) { statementGroup.remove(); } + SchemaAlignmentDialog._hasChanged(); } /* SchemaAlignmentDialog._addStatement = function() { @@ -445,25 +451,33 @@ SchemaAlignmentDialog.getJSON = function() { }; }; +SchemaAlignmentDialog._hasChanged = function() { + this._hasUnsavedChanges = true; + SchemaAlignmentDialog.preview(false); +} + SchemaAlignmentDialog.preview = function(initial) { var self = this; + $('.invalid-schema-warning').hide(); this._previewPanes.empty(); +/* if (!(initial)) { this._elmts.statusIndicator.show().text("There are unsaved changes."); this._hasUnsavedChanges = true; } +*/ - var protograph = this.getJSON(); + var schema = this.getJSON(); $.post( - "command/freebase/preview-protograph?" + $.param({ project: theProject.id }), - { protograph: JSON.stringify(protograph), engine: JSON.stringify(ui.browsingEngine.getJSON()) }, + "command/wikidata/preview-wikibase-schema?" + $.param({ project: theProject.id }), + { schema: JSON.stringify(schema), engine: JSON.stringify(ui.browsingEngine.getJSON()) }, function(data) { - if ("mqllike" in data) { - $(self._previewPanes[0]).text(JSON.stringify(data.mqllike, null, 2)); + if ("quickstatements" in data) { + $(self._previewPanes[0]).text(data.quickstatements); } - if ("tripleloader" in data) { - $(self._previewPanes[1]).text(data.tripleloader); + if ("code" in data && data.code === "error") { + $('.invalid-schema-warning').show(); } }, "json" @@ -472,7 +486,7 @@ SchemaAlignmentDialog.preview = function(initial) { SchemaAlignmentDialog._findColumn = function(cellIndex) { var columns = theProject.columnModel.columns; - for (var i = 0; i < columns.length; i++) { + for (var i = 0; i < columns.length; i++) { var column = columns[i]; if (column.cellIndex == cellIndex) { return column; diff --git a/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java b/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java new file mode 100644 index 000000000..d4e77180f --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java @@ -0,0 +1,93 @@ +/* + +Copyright 2010, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +package org.openrefine.wikidata.commands; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Properties; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.json.JSONObject; +import org.json.JSONWriter; + +import com.google.refine.browsing.Engine; +import com.google.refine.browsing.FilteredRows; +import com.google.refine.commands.Command; + +import org.openrefine.wikidata.exporters.QuickStatementsExporter; +import org.openrefine.wikidata.schema.WikibaseSchema; +import com.google.refine.model.Project; +import com.google.refine.util.ParsingUtilities; + +public class PreviewWikibaseSchemaCommand extends Command { + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + try { + Project project = getProject(request); + Engine engine = getEngine(request, project); + FilteredRows filteredRows = engine.getAllFilteredRows(); + + response.setCharacterEncoding("UTF-8"); + response.setHeader("Content-Type", "application/json"); + + String jsonString = request.getParameter("schema"); + JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString); + WikibaseSchema schema = WikibaseSchema.reconstruct(json); + + StringWriter sb = new StringWriter(2048); + JSONWriter writer = new JSONWriter(sb, 32); + writer.object(); + + { + StringWriter stringWriter = new StringWriter(); + QuickStatementsExporter exporter = new QuickStatementsExporter(); + exporter.translateSchema(project, schema, stringWriter); + + writer.key("quickstatements"); + writer.value(stringWriter.toString()); + } + + writer.endObject(); + + respond(response, sb.toString()); + } catch (Exception e) { + respondException(response, e); + } + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java new file mode 100644 index 000000000..b7b2a9f64 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java @@ -0,0 +1,145 @@ +package org.openrefine.wikidata.exporters; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.net.URI; +import java.util.List; +import java.util.Properties; + +/* +import org.deri.grefine.rdf.Node; +import org.deri.grefine.rdf.RdfSchema; +import org.deri.grefine.rdf.Util; +import org.deri.grefine.rdf.app.ApplicationContext; +import org.deri.grefine.rdf.vocab.Vocabulary; +import org.deri.grefine.rdf.vocab.VocabularyIndexException; +import org.openrdf.model.*; +import org.openrdf.repository.Repository; +import org.openrdf.repository.RepositoryConnection; +import org.openrdf.repository.RepositoryException; +import org.openrdf.repository.sail.SailRepository; +import org.openrdf.rio.RDFFormat; +import org.openrdf.rio.RDFHandlerException; +import org.openrdf.rio.RDFWriter; +import org.openrdf.rio.Rio; +import org.openrdf.sail.memory.MemoryStore; + +import info.aduna.iteration.CloseableIteration; +*/ + +import com.google.refine.browsing.Engine; +import com.google.refine.browsing.FilteredRows; +import com.google.refine.browsing.RowVisitor; +import com.google.refine.exporters.WriterExporter; +import com.google.refine.model.Project; +import com.google.refine.model.Row; + +import org.openrefine.wikidata.schema.WbChangeExpr; +import org.openrefine.wikidata.schema.WbItemStatementsExpr; +import org.openrefine.wikidata.schema.WikibaseSchema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.wikidata.wdtk.datamodel.interfaces.Claim; +import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue; +import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue; +import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; +import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue; +import org.wikidata.wdtk.datamodel.interfaces.QuantityValue; +import org.wikidata.wdtk.datamodel.interfaces.Statement; +import org.wikidata.wdtk.datamodel.interfaces.StatementGroup; +import org.wikidata.wdtk.datamodel.interfaces.StringValue; +import org.wikidata.wdtk.datamodel.interfaces.TimeValue; +import org.wikidata.wdtk.datamodel.interfaces.Value; +import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor; + +public class QuickStatementsExporter implements WriterExporter { + + final static Logger logger = LoggerFactory.getLogger("QuickStatementsExporter"); + + public QuickStatementsExporter(){ + } + + @Override + public String getContentType() { + return "text"; + } + + + @Override + public void export(Project project, Properties options, Engine engine, Writer writer) + throws IOException { + WikibaseSchema schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema"); + if (schema == null) { + return; + } + translateSchema(project, schema, writer); + } + + public void translateSchema(Project project, WikibaseSchema schema, Writer writer) throws IOException { + List statements = schema.evaluate(project); + for (StatementGroup group : statements) { + translateStatementGroup(group, writer); + } + } + + protected void translateStatementGroup(StatementGroup group, Writer writer) throws IOException { + String pid = group.getProperty().getId(); + for(Statement statement : group.getStatements()) { + translateStatement(statement, pid, writer); + } + } + + protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException { + Claim claim = statement.getClaim(); + String qid = claim.getSubject().getId(); + Value val = claim.getValue(); + ValueVisitor vv = new ValuePrinter(); + String targetValue = val.accept(vv); + writer.write(qid + "\t" + pid + "\t" + targetValue + "\n"); + } + + class ValuePrinter implements ValueVisitor { + + @Override + public String visit(DatatypeIdValue value) { + // TODO Auto-generated method stub + return null; + } + + @Override + public String visit(EntityIdValue value) { + return value.getId(); + } + + @Override + public String visit(GlobeCoordinatesValue value) { + // TODO Auto-generated method stub + return null; + } + + @Override + public String visit(MonolingualTextValue value) { + // TODO Auto-generated method stub + return null; + } + + @Override + public String visit(QuantityValue value) { + // TODO Auto-generated method stub + return null; + } + + @Override + public String visit(StringValue value) { + return "\"" + value.getString() + "\""; + } + + @Override + public String visit(TimeValue value) { + // TODO Auto-generated method stub + return null; + } + + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/InvalidSchemaException.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/InvalidSchemaException.java new file mode 100644 index 000000000..65cbab64b --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/InvalidSchemaException.java @@ -0,0 +1,6 @@ +package org.openrefine.wikidata.schema; + + +public class InvalidSchemaException extends Exception { + static final long serialVersionUID = 494837587034L; +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/SkipStatementException.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/SkipStatementException.java new file mode 100644 index 000000000..4640a500c --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/SkipStatementException.java @@ -0,0 +1,6 @@ +package org.openrefine.wikidata.schema; + + +public class SkipStatementException extends Exception { + static final long serialVersionUID = 738592057L; +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java index 3d8b36620..47a9d24c9 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java @@ -7,7 +7,7 @@ import org.openrefine.wikidata.schema.WbValueExpr; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; public abstract class WbItemExpr extends WbValueExpr { - public abstract ItemIdValue evaluate(ExpressionContext ctxt); + public abstract ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException; public static WbItemExpr fromJSON(JSONObject obj) throws JSONException { String type = obj.getString(jsonTypeKey); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java index da40fafbe..88d66b5ee 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java @@ -49,7 +49,7 @@ public class WbItemStatementsExpr extends WbChangeExpr { statementExprs); } - public List evaluate(ExpressionContext ctxt) { + public List evaluate(ExpressionContext ctxt) throws SkipStatementException { List results = new ArrayList(statementGroupExprs.size()); ItemIdValue subjectId = subjectExpr.evaluate(ctxt); for(WbStatementGroupExpr expr : statementGroupExprs) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java index 401625b45..1f8d9fbb8 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java @@ -34,13 +34,13 @@ public class WbItemVariable extends WbItemExpr { } @Override - public ItemIdValue evaluate(ExpressionContext ctxt) { + public ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException { Cell cell = ctxt.getCellByName(columnName); if (cell != null && cell.recon != null && cell.recon.match != null) { ReconCandidate match = cell.recon.match; return ItemIdValueImpl.create(match.id, ctxt.getBaseIRI()); } - return null; + throw new SkipStatementException(); } public String getJsonType() { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java index b480fcc19..38ca9e580 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java @@ -43,7 +43,7 @@ public class WbReferenceExpr extends BiJsonizable { return new WbReferenceExpr(lst); } - public Reference evaluate(ExpressionContext ctxt) { + public Reference evaluate(ExpressionContext ctxt) throws SkipStatementException { List snakGroups = new ArrayList(); for (WbSnakExpr expr : snakExprs) { List snakList = new ArrayList(1); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java index 4938994f6..0087b94d0 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java @@ -42,7 +42,7 @@ public class WbSnakExpr implements Jsonizable { return new WbSnakExpr(propExpr, valueExpr); } - public Snak evaluate(ExpressionContext ctxt) { + public Snak evaluate(ExpressionContext ctxt) throws SkipStatementException { PropertyIdValue propertyId = propExpr.evaluate(ctxt); Value value = valueExpr.evaluate(ctxt); return Datamodel.makeValueSnak(propertyId, value); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java index d53bf0276..08ac07819 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java @@ -92,7 +92,7 @@ public class WbStatementExpr extends BiJsonizable { return snakGroups; } - public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId) { + public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId) throws SkipStatementException { Value mainSnakValue = mainSnakValueExpr.evaluate(ctxt); Snak mainSnak = Datamodel.makeValueSnak(propertyId, mainSnakValue); List qualifiers = new ArrayList(qualifierExprs.size()); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java index 6070cca8a..362605d21 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java @@ -56,7 +56,7 @@ public class WbStatementGroupExpr extends BiJsonizable { claimExprs); } - public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) { + public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) throws SkipStatementException { PropertyIdValue propertyId = propertyExpr.evaluate(ctxt); List statements = new ArrayList(claimExprs.size()); for(WbStatementExpr expr : claimExprs) { diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java index 84334b4e0..da538b454 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java @@ -11,12 +11,11 @@ public abstract class WbValueExpr extends BiJsonizable { * of a statement. */ - /* * Evaluates the value expression in a given context, * returns a wikibase value suitable to be the target of a claim. */ - public abstract Value evaluate(ExpressionContext ctxt); + public abstract Value evaluate(ExpressionContext ctxt) throws SkipStatementException; public static WbValueExpr fromJSON(JSONObject obj) throws JSONException { String type = obj.getString(WbValueExpr.jsonTypeKey); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index fd88c89ce..798a1de71 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -59,7 +59,12 @@ public class WikibaseSchema implements OverlayModel { List result = new ArrayList(); for (WbChangeExpr changeExpr : changeExprs) { WbItemStatementsExpr expr = (WbItemStatementsExpr)changeExpr; - result.addAll(expr.evaluate(ctxt)); + + try { + result.addAll(expr.evaluate(ctxt)); + } catch (SkipStatementException e) { + continue; + } } return result; }