Add QuickStatements export and preview

extensions/wikidata/module/MOD-INF/controller.js

@@ -48,6 +48,7 @@ function init() {
      * Commands
      */
     RefineServlet.registerCommand(module, "save-wikibase-schema", new SaveWikibaseSchemaCommand());
+    RefineServlet.registerCommand(module, "preview-wikibase-schema", new PreviewWikibaseSchemaCommand());
      
     /*
      * Resources

extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html

@@ -24,6 +24,7 @@
           </div>
       </div>
       <div id="schema-alignment-tabs-preview-qs" style="display: none;">
+          <div class="invalid-schema-warning">Your schema is incomplete, fix it to see the preview.</div>
           <div class="schema-alignment-dialog-preview"></div>
       </div>
     </div>

extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js

@@ -111,7 +111,7 @@ SchemaAlignmentDialog._reset = function(schema, initial) {
   if (!this._schema.changes.length) {
     // this._addItem();
   }
-  // this.preview(initial);
+  this.preview(initial);
 };
 
 SchemaAlignmentDialog._save = function(onDone) {
@@ -130,6 +130,7 @@ SchemaAlignmentDialog._save = function(onDone) {
         theProject.overlayModels.wikibaseSchema = schema;
 
         self._elmts.statusIndicator.hide();
+        $('.invalid-schema-warning').hide();
         self._hasUnsavedChanges = false;
 
         if (onDone) onDone();
@@ -208,6 +209,7 @@ SchemaAlignmentDialog._createDialog = function() {
 
   var url = ReconciliationManager.ensureDefaultServicePresent();
   SchemaAlignmentDialog._reconService = ReconciliationManager.getServiceFromUrl(url);
+  this.preview();
 };
 
 SchemaAlignmentDialog._addItem = function(json) {
@@ -342,7 +344,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
     
 
     input.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
-        if (mode ===  "item") {
+        if (mode ===  "item" || mode === "target") {
             inputContainer.data("jsonValue", {
                 type : "wbitemconstant",
                 qid : data.id,
@@ -355,6 +357,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
                 label: data.name,
             });
         }
+        SchemaAlignmentDialog._hasChanged();
     });
   }
 
@@ -367,6 +370,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
     deleteButton.click(function () {
         columnDiv.remove();
         input.show();
+        SchemaAlignmentDialog._hasChanged();
     });
   };
 
@@ -387,6 +391,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
             type : "wbitemvariable",
             columnName: ui.draggable.text(),
         });
+        SchemaAlignmentDialog._hasChanged();
         return true; 
     }).on("dropactivate", function(evt, ui) {
         input.addClass("wbs-accepting-input");
@@ -421,6 +426,7 @@ SchemaAlignmentDialog._removeStatement = function(statement) {
   if (remainingStatements === 0) {
       statementGroup.remove();
   }
+  SchemaAlignmentDialog._hasChanged();
 }
 /*
 SchemaAlignmentDialog._addStatement = function() {
@@ -445,25 +451,33 @@ SchemaAlignmentDialog.getJSON = function() {
   };
 };
 
+SchemaAlignmentDialog._hasChanged = function() {
+  this._hasUnsavedChanges = true;
+  SchemaAlignmentDialog.preview(false);
+}
+
 SchemaAlignmentDialog.preview = function(initial) {
   var self = this;
 
+  $('.invalid-schema-warning').hide();
   this._previewPanes.empty();
+/*
   if (!(initial)) {
     this._elmts.statusIndicator.show().text("There are unsaved changes.");
     this._hasUnsavedChanges = true;
   }
+*/
 
-  var protograph = this.getJSON();
+  var schema = this.getJSON();
   $.post(
-    "command/freebase/preview-protograph?" + $.param({ project: theProject.id }),
+    "command/wikidata/preview-wikibase-schema?" + $.param({ project: theProject.id }),
-    { protograph: JSON.stringify(protograph), engine: JSON.stringify(ui.browsingEngine.getJSON()) },
+    { schema: JSON.stringify(schema), engine: JSON.stringify(ui.browsingEngine.getJSON()) },
     function(data) {
-      if ("mqllike" in data) {
+      if ("quickstatements" in data) {
-        $(self._previewPanes[0]).text(JSON.stringify(data.mqllike, null, 2));
+        $(self._previewPanes[0]).text(data.quickstatements);
       }
-      if ("tripleloader" in data) {
+      if ("code" in data && data.code === "error") {
-        $(self._previewPanes[1]).text(data.tripleloader);
+         $('.invalid-schema-warning').show();
       }
     },
     "json"
@@ -472,7 +486,7 @@ SchemaAlignmentDialog.preview = function(initial) {
 
 SchemaAlignmentDialog._findColumn = function(cellIndex) {
   var columns = theProject.columnModel.columns;
-  for (var i = 0; i < columns.length; i++) {
+    for (var i = 0; i < columns.length; i++) {
     var column = columns[i];
     if (column.cellIndex == cellIndex) {
       return column;

extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java

@@ -0,0 +1,93 @@
+/*
+
+Copyright 2010, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+package org.openrefine.wikidata.commands;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Properties;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.refine.browsing.Engine;
+import com.google.refine.browsing.FilteredRows;
+import com.google.refine.commands.Command;
+
+import org.openrefine.wikidata.exporters.QuickStatementsExporter;
+import org.openrefine.wikidata.schema.WikibaseSchema;
+import com.google.refine.model.Project;
+import com.google.refine.util.ParsingUtilities;
+
+public class PreviewWikibaseSchemaCommand extends Command {
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws ServletException, IOException {
+        
+        try {
+            Project project = getProject(request);
+            Engine engine = getEngine(request, project);
+            FilteredRows filteredRows = engine.getAllFilteredRows();
+            
+            response.setCharacterEncoding("UTF-8");
+            response.setHeader("Content-Type", "application/json");
+            
+            String jsonString = request.getParameter("schema");
+            JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
+            WikibaseSchema schema = WikibaseSchema.reconstruct(json);
+            
+            StringWriter sb = new StringWriter(2048);
+            JSONWriter writer = new JSONWriter(sb, 32);
+            writer.object();
+            
+            {
+                StringWriter stringWriter = new StringWriter();
+                QuickStatementsExporter exporter = new QuickStatementsExporter();
+                exporter.translateSchema(project, schema, stringWriter);
+                
+                writer.key("quickstatements");
+                writer.value(stringWriter.toString());
+            }
+            
+            writer.endObject();
+            
+            respond(response, sb.toString());
+        } catch (Exception e) {
+            respondException(response, e);
+        }
+    }
+}

extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java

@@ -0,0 +1,145 @@
+package org.openrefine.wikidata.exporters;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.net.URI;
+import java.util.List;
+import java.util.Properties;
+
+/*
+import org.deri.grefine.rdf.Node;
+import org.deri.grefine.rdf.RdfSchema;
+import org.deri.grefine.rdf.Util;
+import org.deri.grefine.rdf.app.ApplicationContext;
+import org.deri.grefine.rdf.vocab.Vocabulary;
+import org.deri.grefine.rdf.vocab.VocabularyIndexException;
+import org.openrdf.model.*;
+import org.openrdf.repository.Repository;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+import org.openrdf.repository.sail.SailRepository;
+import org.openrdf.rio.RDFFormat;
+import org.openrdf.rio.RDFHandlerException;
+import org.openrdf.rio.RDFWriter;
+import org.openrdf.rio.Rio;
+import org.openrdf.sail.memory.MemoryStore;
+
+import info.aduna.iteration.CloseableIteration;
+*/
+
+import com.google.refine.browsing.Engine;
+import com.google.refine.browsing.FilteredRows;
+import com.google.refine.browsing.RowVisitor;
+import com.google.refine.exporters.WriterExporter;
+import com.google.refine.model.Project;
+import com.google.refine.model.Row;
+
+import org.openrefine.wikidata.schema.WbChangeExpr;
+import org.openrefine.wikidata.schema.WbItemStatementsExpr;
+import org.openrefine.wikidata.schema.WikibaseSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.wikidata.wdtk.datamodel.interfaces.Claim;
+import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
+import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
+import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
+import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
+import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
+import org.wikidata.wdtk.datamodel.interfaces.Statement;
+import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
+import org.wikidata.wdtk.datamodel.interfaces.StringValue;
+import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
+import org.wikidata.wdtk.datamodel.interfaces.Value;
+import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
+
+public class QuickStatementsExporter implements WriterExporter {
+
+    final static Logger logger = LoggerFactory.getLogger("QuickStatementsExporter");
+
+    public QuickStatementsExporter(){
+    }
+     
+    @Override
+    public String getContentType() {
+        return "text";
+    }
+
+
+    @Override
+    public void export(Project project, Properties options, Engine engine, Writer writer)
+            throws IOException {
+        WikibaseSchema schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
+        if (schema == null) {
+            return;
+        }
+        translateSchema(project, schema, writer);
+    }
+    
+    public void translateSchema(Project project, WikibaseSchema schema, Writer writer) throws IOException {
+        List<StatementGroup> statements = schema.evaluate(project);
+        for (StatementGroup group : statements) {
+            translateStatementGroup(group, writer);
+        }
+    }
+    
+    protected void translateStatementGroup(StatementGroup group, Writer writer) throws IOException {
+        String pid = group.getProperty().getId();
+        for(Statement statement : group.getStatements()) {
+            translateStatement(statement, pid, writer);
+        }
+    }
+    
+    protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException {
+        Claim claim = statement.getClaim();
+        String qid = claim.getSubject().getId();
+        Value val = claim.getValue();
+        ValueVisitor<String> vv = new ValuePrinter();
+        String targetValue = val.accept(vv);
+        writer.write(qid + "\t" + pid + "\t" + targetValue + "\n");
+    }
+   
+    class ValuePrinter implements ValueVisitor<String> {
+
+        @Override
+        public String visit(DatatypeIdValue value) {
+            // TODO Auto-generated method stub
+            return null;
+        }
+
+        @Override
+        public String visit(EntityIdValue value) {
+            return value.getId();
+        }
+
+        @Override
+        public String visit(GlobeCoordinatesValue value) {
+            // TODO Auto-generated method stub
+            return null;
+        }
+
+        @Override
+        public String visit(MonolingualTextValue value) {
+            // TODO Auto-generated method stub
+            return null;
+        }
+
+        @Override
+        public String visit(QuantityValue value) {
+            // TODO Auto-generated method stub
+            return null;
+        }
+
+        @Override
+        public String visit(StringValue value) {
+            return "\"" + value.getString() + "\"";
+        }
+
+        @Override
+        public String visit(TimeValue value) {
+            // TODO Auto-generated method stub
+            return null;
+        }
+        
+    }
+}

extensions/wikidata/src/org/openrefine/wikidata/schema/InvalidSchemaException.java

@@ -0,0 +1,6 @@
+package org.openrefine.wikidata.schema;
+
+
+public class InvalidSchemaException extends Exception {
+    static final long serialVersionUID = 494837587034L;
+}

extensions/wikidata/src/org/openrefine/wikidata/schema/SkipStatementException.java

@@ -0,0 +1,6 @@
+package org.openrefine.wikidata.schema;
+
+
+public class SkipStatementException extends Exception {
+    static final long serialVersionUID = 738592057L;
+}

extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java

@@ -7,7 +7,7 @@ import org.openrefine.wikidata.schema.WbValueExpr;
 import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
 
 public abstract class WbItemExpr extends WbValueExpr {
-    public abstract ItemIdValue evaluate(ExpressionContext ctxt);
+    public abstract ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException;
     
     public static WbItemExpr fromJSON(JSONObject obj) throws JSONException {
         String type = obj.getString(jsonTypeKey);

extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java

@@ -49,7 +49,7 @@ public class WbItemStatementsExpr extends WbChangeExpr {
                 statementExprs);
     }
     
-    public List<StatementGroup> evaluate(ExpressionContext ctxt) {
+    public List<StatementGroup> evaluate(ExpressionContext ctxt) throws SkipStatementException {
         List<StatementGroup> results = new ArrayList<StatementGroup>(statementGroupExprs.size());
         ItemIdValue subjectId = subjectExpr.evaluate(ctxt);
         for(WbStatementGroupExpr expr : statementGroupExprs) {

extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java

@@ -34,13 +34,13 @@ public class WbItemVariable extends WbItemExpr {
     }
 
     @Override
-    public ItemIdValue evaluate(ExpressionContext ctxt) {
+    public ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException {
         Cell cell = ctxt.getCellByName(columnName);
         if (cell != null && cell.recon != null && cell.recon.match != null) {
             ReconCandidate match = cell.recon.match;
             return ItemIdValueImpl.create(match.id, ctxt.getBaseIRI());
         }
-        return null;
+        throw new SkipStatementException();
     }
 
     public String getJsonType() {

extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java

@@ -43,7 +43,7 @@ public class WbReferenceExpr extends BiJsonizable {
         return new WbReferenceExpr(lst);
     }
     
-    public Reference evaluate(ExpressionContext ctxt) {
+    public Reference evaluate(ExpressionContext ctxt) throws SkipStatementException {
         List<SnakGroup> snakGroups = new ArrayList<SnakGroup>();
         for (WbSnakExpr expr : snakExprs) {
             List<Snak> snakList = new ArrayList<Snak>(1);

extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java

@@ -42,7 +42,7 @@ public class WbSnakExpr implements Jsonizable {
         return new WbSnakExpr(propExpr, valueExpr);
     }
    
-    public Snak evaluate(ExpressionContext ctxt) {
+    public Snak evaluate(ExpressionContext ctxt) throws SkipStatementException {
         PropertyIdValue propertyId = propExpr.evaluate(ctxt);
         Value value = valueExpr.evaluate(ctxt);
         return Datamodel.makeValueSnak(propertyId, value);

extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java

@@ -92,7 +92,7 @@ public class WbStatementExpr extends BiJsonizable {
         return snakGroups;
     }
     
-    public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId) {
+    public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId) throws SkipStatementException {
         Value mainSnakValue = mainSnakValueExpr.evaluate(ctxt);
         Snak mainSnak = Datamodel.makeValueSnak(propertyId, mainSnakValue);
         List<Snak> qualifiers = new ArrayList<Snak>(qualifierExprs.size());

extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java

@@ -56,7 +56,7 @@ public class WbStatementGroupExpr extends BiJsonizable {
                 claimExprs);
     }
 
-    public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) {
+    public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) throws SkipStatementException {
         PropertyIdValue propertyId = propertyExpr.evaluate(ctxt);
         List<Statement> statements = new ArrayList<Statement>(claimExprs.size());
         for(WbStatementExpr expr : claimExprs) {

extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java

@@ -11,12 +11,11 @@ public abstract class WbValueExpr extends BiJsonizable {
      * of a statement.
      */
     
- 
     /*
      * Evaluates the value expression in a given context,
      * returns a wikibase value suitable to be the target of a claim.
      */
-    public abstract Value evaluate(ExpressionContext ctxt);
+    public abstract Value evaluate(ExpressionContext ctxt) throws SkipStatementException;
     
     public static WbValueExpr fromJSON(JSONObject obj) throws JSONException {
         String type = obj.getString(WbValueExpr.jsonTypeKey);

extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java

@@ -59,7 +59,12 @@ public class WikibaseSchema implements OverlayModel {
         List<StatementGroup> result = new ArrayList<StatementGroup>();
         for (WbChangeExpr changeExpr : changeExprs) {
             WbItemStatementsExpr expr = (WbItemStatementsExpr)changeExpr;
-            result.addAll(expr.evaluate(ctxt));
+            
+            try {
+                result.addAll(expr.evaluate(ctxt));
+            } catch (SkipStatementException e) {
+                continue;
+            }
         }
         return result;
     }