s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add QuickStatements export and preview

Browse Source
This commit is contained in:
Antonin Delpeuch 2017-09-04 21:00:58 +01:00
parent 975542bff1
commit fcde3d2c7d
16 changed files with 291 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/wikidata/module/MOD-INF/controller.js
View File

@ -48,6 +48,7 @@ function init() {
* Commands
*/
RefineServlet.registerCommand(module, "save-wikibase-schema", new SaveWikibaseSchemaCommand());
RefineServlet.registerCommand(module, "preview-wikibase-schema", new PreviewWikibaseSchemaCommand());
/*
* Resources

1
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.html
View File

@ -24,6 +24,7 @@
</div>
</div>
<div id="schema-alignment-tabs-preview-qs" style="display: none;">
<div class="invalid-schema-warning">Your schema is incomplete, fix it to see the preview.</div>
<div class="schema-alignment-dialog-preview"></div>
</div>
</div>

34
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js
View File

@ -111,7 +111,7 @@ SchemaAlignmentDialog._reset = function(schema, initial) {
if (!this._schema.changes.length) {
// this._addItem();
}
// this.preview(initial);
this.preview(initial);
};
SchemaAlignmentDialog._save = function(onDone) {
@ -130,6 +130,7 @@ SchemaAlignmentDialog._save = function(onDone) {
theProject.overlayModels.wikibaseSchema = schema;
self._elmts.statusIndicator.hide();
$('.invalid-schema-warning').hide();
self._hasUnsavedChanges = false;
if (onDone) onDone();
@ -208,6 +209,7 @@ SchemaAlignmentDialog._createDialog = function() {
var url = ReconciliationManager.ensureDefaultServicePresent();
SchemaAlignmentDialog._reconService = ReconciliationManager.getServiceFromUrl(url);
this.preview();
};
SchemaAlignmentDialog._addItem = function(json) {
@ -342,7 +344,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
input.suggestP(suggestConfig).bind("fb-select", function(evt, data) {
if (mode === "item") {
if (mode === "item" || mode === "target") {
inputContainer.data("jsonValue", {
type : "wbitemconstant",
qid : data.id,
@ -355,6 +357,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
label: data.name,
});
}
SchemaAlignmentDialog._hasChanged();
});
}
@ -367,6 +370,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
deleteButton.click(function () {
columnDiv.remove();
input.show();
SchemaAlignmentDialog._hasChanged();
});
};
@ -387,6 +391,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
type : "wbitemvariable",
columnName: ui.draggable.text(),
});
SchemaAlignmentDialog._hasChanged();
return true;
}).on("dropactivate", function(evt, ui) {
input.addClass("wbs-accepting-input");
@ -421,6 +426,7 @@ SchemaAlignmentDialog._removeStatement = function(statement) {
if (remainingStatements === 0) {
statementGroup.remove();
}
SchemaAlignmentDialog._hasChanged();
}
/*
SchemaAlignmentDialog._addStatement = function() {
@ -445,25 +451,33 @@ SchemaAlignmentDialog.getJSON = function() {
};
};
SchemaAlignmentDialog._hasChanged = function() {
this._hasUnsavedChanges = true;
SchemaAlignmentDialog.preview(false);
}
SchemaAlignmentDialog.preview = function(initial) {
var self = this;
$('.invalid-schema-warning').hide();
this._previewPanes.empty();
/*
if (!(initial)) {
this._elmts.statusIndicator.show().text("There are unsaved changes.");
this._hasUnsavedChanges = true;
}
*/
var protograph = this.getJSON();
var schema = this.getJSON();
$.post(
"command/freebase/preview-protograph?" + $.param({ project: theProject.id }),
{ protograph: JSON.stringify(protograph), engine: JSON.stringify(ui.browsingEngine.getJSON()) },
"command/wikidata/preview-wikibase-schema?" + $.param({ project: theProject.id }),
{ schema: JSON.stringify(schema), engine: JSON.stringify(ui.browsingEngine.getJSON()) },
function(data) {
if ("mqllike" in data) {
$(self._previewPanes[0]).text(JSON.stringify(data.mqllike, null, 2));
if ("quickstatements" in data) {
$(self._previewPanes[0]).text(data.quickstatements);
}
if ("tripleloader" in data) {
$(self._previewPanes[1]).text(data.tripleloader);
if ("code" in data && data.code === "error") {
$('.invalid-schema-warning').show();
}
},
"json"
@ -472,7 +486,7 @@ SchemaAlignmentDialog.preview = function(initial) {
SchemaAlignmentDialog._findColumn = function(cellIndex) {
var columns = theProject.columnModel.columns;
for (var i = 0; i < columns.length; i++) {
for (var i = 0; i < columns.length; i++) {
var column = columns[i];
if (column.cellIndex == cellIndex) {
return column;

93
extensions/wikidata/src/org/openrefine/wikidata/commands/PreviewWikibaseSchemaCommand.java Normal file
View File

@ -0,0 +1,93 @@
/*
Copyright 2010, Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.openrefine.wikidata.commands;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.commands.Command;
import org.openrefine.wikidata.exporters.QuickStatementsExporter;
import org.openrefine.wikidata.schema.WikibaseSchema;
import com.google.refine.model.Project;
import com.google.refine.util.ParsingUtilities;
public class PreviewWikibaseSchemaCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
Engine engine = getEngine(request, project);
FilteredRows filteredRows = engine.getAllFilteredRows();
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
String jsonString = request.getParameter("schema");
JSONObject json = ParsingUtilities.evaluateJsonStringToObject(jsonString);
WikibaseSchema schema = WikibaseSchema.reconstruct(json);
StringWriter sb = new StringWriter(2048);
JSONWriter writer = new JSONWriter(sb, 32);
writer.object();
{
StringWriter stringWriter = new StringWriter();
QuickStatementsExporter exporter = new QuickStatementsExporter();
exporter.translateSchema(project, schema, stringWriter);
writer.key("quickstatements");
writer.value(stringWriter.toString());
}
writer.endObject();
respond(response, sb.toString());
} catch (Exception e) {
respondException(response, e);
}
}
}

145
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java Normal file
View File

@ -0,0 +1,145 @@
package org.openrefine.wikidata.exporters;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.net.URI;
import java.util.List;
import java.util.Properties;
/*
import org.deri.grefine.rdf.Node;
import org.deri.grefine.rdf.RdfSchema;
import org.deri.grefine.rdf.Util;
import org.deri.grefine.rdf.app.ApplicationContext;
import org.deri.grefine.rdf.vocab.Vocabulary;
import org.deri.grefine.rdf.vocab.VocabularyIndexException;
import org.openrdf.model.*;
import org.openrdf.repository.Repository;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandlerException;
import org.openrdf.rio.RDFWriter;
import org.openrdf.rio.Rio;
import org.openrdf.sail.memory.MemoryStore;
import info.aduna.iteration.CloseableIteration;
*/
import com.google.refine.browsing.Engine;
import com.google.refine.browsing.FilteredRows;
import com.google.refine.browsing.RowVisitor;
import com.google.refine.exporters.WriterExporter;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import org.openrefine.wikidata.schema.WbChangeExpr;
import org.openrefine.wikidata.schema.WbItemStatementsExpr;
import org.openrefine.wikidata.schema.WikibaseSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueVisitor;
public class QuickStatementsExporter implements WriterExporter {
final static Logger logger = LoggerFactory.getLogger("QuickStatementsExporter");
public QuickStatementsExporter(){
}
@Override
public String getContentType() {
return "text";
}
@Override
public void export(Project project, Properties options, Engine engine, Writer writer)
throws IOException {
WikibaseSchema schema = (WikibaseSchema) project.overlayModels.get("wikibaseSchema");
if (schema == null) {
return;
}
translateSchema(project, schema, writer);
}
public void translateSchema(Project project, WikibaseSchema schema, Writer writer) throws IOException {
List<StatementGroup> statements = schema.evaluate(project);
for (StatementGroup group : statements) {
translateStatementGroup(group, writer);
}
}
protected void translateStatementGroup(StatementGroup group, Writer writer) throws IOException {
String pid = group.getProperty().getId();
for(Statement statement : group.getStatements()) {
translateStatement(statement, pid, writer);
}
}
protected void translateStatement(Statement statement, String pid, Writer writer) throws IOException {
Claim claim = statement.getClaim();
String qid = claim.getSubject().getId();
Value val = claim.getValue();
ValueVisitor<String> vv = new ValuePrinter();
String targetValue = val.accept(vv);
writer.write(qid + "\t" + pid + "\t" + targetValue + "\n");
}
class ValuePrinter implements ValueVisitor<String> {
@Override
public String visit(DatatypeIdValue value) {
// TODO Auto-generated method stub
return null;
}
@Override
public String visit(EntityIdValue value) {
return value.getId();
}
@Override
public String visit(GlobeCoordinatesValue value) {
// TODO Auto-generated method stub
return null;
}
@Override
public String visit(MonolingualTextValue value) {
// TODO Auto-generated method stub
return null;
}
@Override
public String visit(QuantityValue value) {
// TODO Auto-generated method stub
return null;
}
@Override
public String visit(StringValue value) {
return "\"" + value.getString() + "\"";
}
@Override
public String visit(TimeValue value) {
// TODO Auto-generated method stub
return null;
}
}
}

6
extensions/wikidata/src/org/openrefine/wikidata/schema/InvalidSchemaException.java Normal file
View File

@ -0,0 +1,6 @@
package org.openrefine.wikidata.schema;
public class InvalidSchemaException extends Exception {
static final long serialVersionUID = 494837587034L;
}

6
extensions/wikidata/src/org/openrefine/wikidata/schema/SkipStatementException.java Normal file
View File

@ -0,0 +1,6 @@
package org.openrefine.wikidata.schema;
public class SkipStatementException extends Exception {
static final long serialVersionUID = 738592057L;
}

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemExpr.java
View File

@ -7,7 +7,7 @@ import org.openrefine.wikidata.schema.WbValueExpr;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
public abstract class WbItemExpr extends WbValueExpr {
public abstract ItemIdValue evaluate(ExpressionContext ctxt);
public abstract ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException;
public static WbItemExpr fromJSON(JSONObject obj) throws JSONException {
String type = obj.getString(jsonTypeKey);

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemStatementsExpr.java
View File

@ -49,7 +49,7 @@ public class WbItemStatementsExpr extends WbChangeExpr {
statementExprs);
}
public List<StatementGroup> evaluate(ExpressionContext ctxt) {
public List<StatementGroup> evaluate(ExpressionContext ctxt) throws SkipStatementException {
List<StatementGroup> results = new ArrayList<StatementGroup>(statementGroupExprs.size());
ItemIdValue subjectId = subjectExpr.evaluate(ctxt);
for(WbStatementGroupExpr expr : statementGroupExprs) {

4
extensions/wikidata/src/org/openrefine/wikidata/schema/WbItemVariable.java
View File

@ -34,13 +34,13 @@ public class WbItemVariable extends WbItemExpr {
}
@Override
public ItemIdValue evaluate(ExpressionContext ctxt) {
public ItemIdValue evaluate(ExpressionContext ctxt) throws SkipStatementException {
Cell cell = ctxt.getCellByName(columnName);
if (cell != null && cell.recon != null && cell.recon.match != null) {
ReconCandidate match = cell.recon.match;
return ItemIdValueImpl.create(match.id, ctxt.getBaseIRI());
}
return null;
throw new SkipStatementException();
}
public String getJsonType() {

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbReferenceExpr.java
View File

@ -43,7 +43,7 @@ public class WbReferenceExpr extends BiJsonizable {
return new WbReferenceExpr(lst);
}
public Reference evaluate(ExpressionContext ctxt) {
public Reference evaluate(ExpressionContext ctxt) throws SkipStatementException {
List<SnakGroup> snakGroups = new ArrayList<SnakGroup>();
for (WbSnakExpr expr : snakExprs) {
List<Snak> snakList = new ArrayList<Snak>(1);

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbSnakExpr.java
View File

@ -42,7 +42,7 @@ public class WbSnakExpr implements Jsonizable {
return new WbSnakExpr(propExpr, valueExpr);
}
public Snak evaluate(ExpressionContext ctxt) {
public Snak evaluate(ExpressionContext ctxt) throws SkipStatementException {
PropertyIdValue propertyId = propExpr.evaluate(ctxt);
Value value = valueExpr.evaluate(ctxt);
return Datamodel.makeValueSnak(propertyId, value);

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementExpr.java
View File

@ -92,7 +92,7 @@ public class WbStatementExpr extends BiJsonizable {
return snakGroups;
}
public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId) {
public Statement evaluate(ExpressionContext ctxt, ItemIdValue subject, PropertyIdValue propertyId) throws SkipStatementException {
Value mainSnakValue = mainSnakValueExpr.evaluate(ctxt);
Snak mainSnak = Datamodel.makeValueSnak(propertyId, mainSnakValue);
List<Snak> qualifiers = new ArrayList<Snak>(qualifierExprs.size());

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStatementGroupExpr.java
View File

@ -56,7 +56,7 @@ public class WbStatementGroupExpr extends BiJsonizable {
claimExprs);
}
public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) {
public StatementGroup evaluate(ExpressionContext ctxt, ItemIdValue subject) throws SkipStatementException {
PropertyIdValue propertyId = propertyExpr.evaluate(ctxt);
List<Statement> statements = new ArrayList<Statement>(claimExprs.size());
for(WbStatementExpr expr : claimExprs) {

3
extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java
View File

@ -11,12 +11,11 @@ public abstract class WbValueExpr extends BiJsonizable {
* of a statement.
*/
/*
* Evaluates the value expression in a given context,
* returns a wikibase value suitable to be the target of a claim.
*/
public abstract Value evaluate(ExpressionContext ctxt);
public abstract Value evaluate(ExpressionContext ctxt) throws SkipStatementException;
public static WbValueExpr fromJSON(JSONObject obj) throws JSONException {
String type = obj.getString(WbValueExpr.jsonTypeKey);

7
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -59,7 +59,12 @@ public class WikibaseSchema implements OverlayModel {
List<StatementGroup> result = new ArrayList<StatementGroup>();
for (WbChangeExpr changeExpr : changeExprs) {
WbItemStatementsExpr expr = (WbItemStatementsExpr)changeExpr;
result.addAll(expr.evaluate(ctxt));
try {
result.addAll(expr.evaluate(ctxt));
} catch (SkipStatementException e) {
continue;
}
}
return result;
}