From 165ff41469dc56fbfc17fea2f9a38057c3a0e5de Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Mon, 18 Sep 2017 10:19:38 +0100 Subject: [PATCH] Support for dates and coordinates in Wikibase schema --- .../dialogs/schema-alignment-dialog.js | 43 +++++++- .../exporters/QuickStatementsExporter.java | 38 +++++-- .../wikidata/schema/WbDateConstant.java | 101 ++++++++++++++++++ .../wikidata/schema/WbDateExpr.java | 25 +++++ .../wikidata/schema/WbDateVariable.java | 54 ++++++++++ .../wikidata/schema/WbLocationConstant.java | 55 ++++++++++ .../wikidata/schema/WbLocationExpr.java | 23 ++++ .../wikidata/schema/WbLocationVariable.java | 54 ++++++++++ .../wikidata/schema/WbStringExpr.java | 2 +- .../wikidata/schema/WbValueExpr.java | 8 ++ .../wikidata/schema/WikibaseSchema.java | 11 +- 11 files changed, 396 insertions(+), 18 deletions(-) create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateExpr.java create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationExpr.java create mode 100644 extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java diff --git a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js index b540215fb..693aad5f8 100644 --- a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js +++ b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js @@ -323,6 +323,7 @@ SchemaAlignmentDialog._addStatement = function(container, datatype, json) { SchemaAlignmentDialog._initField(inputContainer, datatype, value); // If we are in a mainsnak... + // (see https://www.mediawiki.org/wiki/Wikibase/DataModel#Snaks) if (container.parents('.wbs-statement').length == 0) { // add qualifiers... @@ -431,6 +432,7 @@ SchemaAlignmentDialog._addReference = function(container, json) { $('').attr('alt', 'remove reference').click(function() { reference.remove(); SchemaAlignmentDialog._updateReferencesNumber(container); + SchemaAlignmentDialog._hasChanged(); }).appendTo(toolbarRef); var right = $('
').addClass('wbs-right').appendTo(reference); var qualifierContainer = $('
').addClass('wbs-qualifier-container').appendTo(right); @@ -460,9 +462,7 @@ SchemaAlignmentDialog._referenceToJSON = function(reference) { SchemaAlignmentDialog._updateReferencesNumber = function(container) { var childrenCount = container.children().length; var statement = container.parents('.wbs-statement'); - console.log(statement); var a = statement.find('.wbs-references-toggle a').first(); - console.log(a); a.html(childrenCount+' references'); } @@ -548,6 +548,32 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue) }); SchemaAlignmentDialog._hasChanged(); }); + } else if (mode === "time") { + var propagateValue = function(val) { + // TODO add validation here + inputContainer.data("jsonValue", { + type: "wbdateconstant", + value: val, + }); + }; + propagateValue(""); + input.change(function() { + propagateValue($(this).val()); + SchemaAlignmentDialog._hasChanged(); + }); + } else if (mode === "globecoordinates") { + var propagateValue = function(val) { + // TODO add validation here + inputContainer.data("jsonValue", { + type: "wblocationconstant", + value: val, + }); + }; + propagateValue(""); + input.change(function() { + propagateValue($(this).val()); + SchemaAlignmentDialog._hasChanged(); + }); } else { /* if (mode === "external-id") { */ var propagateValue = function(val) { inputContainer.data("jsonValue", { @@ -581,8 +607,11 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue) if (mode === "wikibase-item") { acceptClass = ".wbs-reconciled-column"; wbVariableType = "wbitemvariable"; + } else if (mode === "time") { + wbVariableType = "wbdatevariable"; + } else if (mode === "globecoordinates") { + wbVariableType = "wblocationvariable"; } - inputContainer.droppable({ accept: acceptClass, @@ -608,9 +637,13 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue) } else if (initialValue.type == "wbitemvariable") { var cell = SchemaAlignmentDialog._createDraggableColumn(initialValue.columnName, true); acceptDraggableColumn(cell); - } else if (initialValue.type == "wbstringconstant") { + } else if (initialValue.type == "wbstringconstant" || + initialValue.type == "wbdateconstant" || + initialValue.type == "wblocationconstant") { input.val(initialValue.value); - } else if (initialValue.type == "wbstringvariable") { + } else if (initialValue.type == "wbstringvariable" || + initialValue.type == "wbdatevariable" || + initialValue.type == "wblocationvariable") { var cell = SchemaAlignmentDialog._createDraggableColumn(initialValue.columnName, false); acceptDraggableColumn(cell); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java index 3d163a424..18bddcf42 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java @@ -121,7 +121,8 @@ public class QuickStatementsExporter implements WriterExporter { @Override public String visit(DatatypeIdValue value) { - // TODO Auto-generated method stub + // unsupported according to + // https://tools.wmflabs.org/wikidata-todo/quick_statements.php? return null; } @@ -135,20 +136,32 @@ public class QuickStatementsExporter implements WriterExporter { @Override public String visit(GlobeCoordinatesValue value) { - // TODO Auto-generated method stub - return null; + return String.format( + "@%f/%f", + value.getLatitude(), + value.getLongitude()); } @Override public String visit(MonolingualTextValue value) { - // TODO Auto-generated method stub - return null; + return String.format( + "%s:/\"%s\"", + value.getLanguageCode(), + value.getText()); } @Override public String visit(QuantityValue value) { - // TODO Auto-generated method stub - return null; + String unitPrefix = "http://www.wikidata.org/entity/Q"; + String unit = value.getUnit(); + if (!unit.startsWith(unitPrefix)) + return null; // QuickStatements only accepts Qids as units + String unitID = "U"+unit.substring(unitPrefix.length()); + return String.format( + "[%f,%f]%s", + value.getLowerBound(), + value.getUpperBound(), + unitID); } @Override @@ -158,8 +171,15 @@ public class QuickStatementsExporter implements WriterExporter { @Override public String visit(TimeValue value) { - // TODO Auto-generated method stub - return null; + return String.format( + "+%04d-%02d-%02dT%02d:%02d:%02dZ/%d", + value.getYear(), + value.getMonth(), + value.getDay(), + value.getHour(), + value.getMinute(), + value.getSecond(), + value.getPrecision()); } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java new file mode 100644 index 000000000..6a8125940 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java @@ -0,0 +1,101 @@ +package org.openrefine.wikidata.schema; + +import java.util.Calendar; +import java.util.Date; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.TimeValue; + +import com.google.common.collect.ImmutableMap; + + +public class WbDateConstant extends WbDateExpr { + public static final String jsonType = "wbdateconstant"; + + public static Map acceptedFormats = ImmutableMap.builder() + .put(new SimpleDateFormat("yyyy"), 9) + .put(new SimpleDateFormat("yyyy-MM"), 10) + .put(new SimpleDateFormat("yyyy-MM-dd"), 11) + .put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12) + .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13) + .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14) + .build(); + + private TimeValue _parsed; + private String _origDatestamp; + + public WbDateConstant(String origDatestamp) { + _origDatestamp = origDatestamp; + try { + _parsed = parse(origDatestamp); + } catch(ParseException e) { + _parsed = null; + } + } + + @Override + public TimeValue evaluate(ExpressionContext ctxt) + throws SkipStatementException { + if (_parsed == null) { + throw new SkipStatementException(); + } + return _parsed; + } + + public static TimeValue parse(String datestamp) throws ParseException { + Date date = null; + int precision = 9; // default precision (will be overridden) + for(Entry entry : acceptedFormats.entrySet()) { + try { + date = entry.getKey().parse(datestamp); + precision = entry.getValue(); + } catch (ParseException e) { + continue; + } + } + if (date == null) { + throw new ParseException("Invalid date.", 0); + } else { + Calendar calendar = Calendar.getInstance(); + calendar = Calendar.getInstance(); + calendar.setTime(date); + return Datamodel.makeTimeValue( + calendar.get(Calendar.YEAR), + (byte) (calendar.get(Calendar.MONTH)+1), // java starts at 0 + (byte) calendar.get(Calendar.DAY_OF_MONTH), + (byte) calendar.get(Calendar.HOUR_OF_DAY), + (byte) calendar.get(Calendar.MINUTE), + (byte) calendar.get(Calendar.SECOND), + (byte) precision, + 1, + 1, + calendar.getTimeZone().getRawOffset()/3600000, + TimeValue.CM_GREGORIAN_PRO); + } + } + + @Override + public void writeFields(JSONWriter writer, Properties options) + throws JSONException { + writer.key("value"); + writer.value(_origDatestamp); + } + + public static WbDateConstant fromJSON(JSONObject obj) throws JSONException { + return new WbDateConstant(obj.getString("value")); + } + + @Override + public String getJsonType() { + return jsonType; + } + +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateExpr.java new file mode 100644 index 000000000..e1d82fc1e --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateExpr.java @@ -0,0 +1,25 @@ +package org.openrefine.wikidata.schema; + +import org.json.JSONException; +import org.json.JSONObject; +import org.wikidata.wdtk.datamodel.interfaces.TimeValue; + + +public abstract class WbDateExpr extends WbValueExpr { + + @Override + public abstract TimeValue evaluate(ExpressionContext ctxt) + throws SkipStatementException; + + public static WbDateExpr fromJSON(JSONObject obj) throws JSONException { + String type = obj.getString(jsonTypeKey); + if (WbDateConstant.jsonType.equals(type)) { + return WbDateConstant.fromJSON(obj); + } else if (WbDateVariable.jsonType.equals(type)) { + return WbDateVariable.fromJSON(obj); + } else { + throw new JSONException("unknown type for WbDateExpr"); + } + } + +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java new file mode 100644 index 000000000..a14eb1fde --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java @@ -0,0 +1,54 @@ +package org.openrefine.wikidata.schema; + +import java.text.ParseException; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.StringValue; +import org.wikidata.wdtk.datamodel.interfaces.TimeValue; + +import com.google.refine.model.Cell; + + +public class WbDateVariable extends WbDateExpr { + public static final String jsonType = "wbdatevariable"; + + private String _columnName; + + public WbDateVariable(String columnName) { + _columnName = columnName; + } + + @Override + public TimeValue evaluate(ExpressionContext ctxt) + throws SkipStatementException { + Cell cell = ctxt.getCellByName(_columnName); + if (cell != null) { + try { + // TODO accept parsed dates (without converting them to strings) + return WbDateConstant.parse(cell.value.toString()); + } catch (ParseException e) { + } + } + throw new SkipStatementException(); + } + + @Override + public void writeFields(JSONWriter writer, Properties options) + throws JSONException { + writer.key("columnName"); + writer.value(_columnName); + } + + public static WbDateVariable fromJSON(JSONObject obj) throws JSONException { + return new WbDateVariable(obj.getString("columnName")); + } + + @Override + public String getJsonType() { + return jsonType; + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java new file mode 100644 index 000000000..19aeab529 --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java @@ -0,0 +1,55 @@ +package org.openrefine.wikidata.schema; + +import java.text.ParseException; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; + + +public class WbLocationConstant extends WbLocationExpr { + public static final String jsonType = "wblocationconstant"; + + private String _origValue; + private GlobeCoordinatesValue _parsed; + + public WbLocationConstant(String origValue) { + _origValue = origValue; + _parsed = null; + } + + public static GlobeCoordinatesValue parse(String expr) throws ParseException { + double lat = 0; + double lng = 0; + double precision = 0; + return Datamodel.makeGlobeCoordinatesValue(lat, lng, precision, + GlobeCoordinatesValue.GLOBE_EARTH); + } + + @Override + public GlobeCoordinatesValue evaluate(ExpressionContext ctxt) + throws SkipStatementException { + if (_parsed == null) + throw new SkipStatementException(); + return _parsed; + } + + public static WbLocationConstant fromJSON(JSONObject obj) throws JSONException { + return new WbLocationConstant(obj.getString("value")); + } + + @Override + public void writeFields(JSONWriter writer, Properties options) + throws JSONException { + writer.key("value"); + writer.value(_origValue); + } + + @Override + public String getJsonType() { + return jsonType; + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationExpr.java new file mode 100644 index 000000000..46ad37d6f --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationExpr.java @@ -0,0 +1,23 @@ +package org.openrefine.wikidata.schema; + +import org.json.JSONException; +import org.json.JSONObject; +import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; + + +public abstract class WbLocationExpr extends WbValueExpr { + @Override + public abstract GlobeCoordinatesValue evaluate(ExpressionContext ctxt) + throws SkipStatementException; + + public static WbLocationExpr fromJSON(JSONObject obj) throws JSONException { + String type = obj.getString(jsonTypeKey); + if (WbLocationConstant.jsonType.equals(type)) { + return WbLocationConstant.fromJSON(obj); + } else if (WbLocationVariable.jsonType.equals(type)) { + return WbLocationVariable.fromJSON(obj); + } else { + throw new JSONException("unknown type for WbLocationExpr"); + } + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java new file mode 100644 index 000000000..929d7d3ac --- /dev/null +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java @@ -0,0 +1,54 @@ +package org.openrefine.wikidata.schema; + +import java.text.ParseException; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; +import org.wikidata.wdtk.datamodel.helpers.Datamodel; +import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; +import org.wikidata.wdtk.datamodel.interfaces.StringValue; + +import com.google.refine.model.Cell; + + +public class WbLocationVariable extends WbLocationExpr { + public static final String jsonType = "wblocationvariable"; + + private String columnName; + + public WbLocationVariable(String columnName) { + this.columnName = columnName; + } + + @Override + public GlobeCoordinatesValue evaluate(ExpressionContext ctxt) + throws SkipStatementException { + Cell cell = ctxt.getCellByName(columnName); + if (cell != null) { + String expr = cell.value.toString(); + try { + return WbLocationConstant.parse(expr); + } catch (ParseException e) { + } + } + throw new SkipStatementException(); + } + + @Override + public void writeFields(JSONWriter writer, Properties options) + throws JSONException { + writer.key("columnName"); + writer.value(columnName); + } + + public static WbLocationVariable fromJSON(JSONObject obj) throws JSONException { + return new WbLocationVariable(obj.getString("columnName")); + } + + @Override + public String getJsonType() { + return jsonType; + } +} diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringExpr.java index 2088e026a..ebbc81825 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringExpr.java @@ -15,7 +15,7 @@ public abstract class WbStringExpr extends WbValueExpr { } else if (WbStringVariable.jsonType.equals(type)) { return WbStringVariable.fromJSON(obj); } else { - throw new JSONException("unknown type for WbItemExpr"); + throw new JSONException("unknown type for WbStringExpr"); } } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java index f1208a109..f22a2fd01 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java @@ -30,6 +30,14 @@ public abstract class WbValueExpr extends BiJsonizable { valueExpr = WbStringVariable.fromJSON(obj); } else if (WbStringConstant.jsonType.equals(type)) { valueExpr = WbStringConstant.fromJSON(obj); + } else if (WbDateVariable.jsonType.equals(type)) { + valueExpr = WbDateVariable.fromJSON(obj); + } else if (WbDateConstant.jsonType.equals(type)) { + valueExpr = WbDateConstant.fromJSON(obj); + } else if (WbLocationVariable.jsonType.equals(type)) { + valueExpr = WbLocationVariable.fromJSON(obj); + } else if (WbLocationConstant.jsonType.equals(type)) { + valueExpr = WbLocationConstant.fromJSON(obj); } else { throw new JSONException("unknown type '"+type+"' for WbValueExpr"); } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java index 4d6f2eea5..7ec06fb6a 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java @@ -45,7 +45,7 @@ public class WikibaseSchema implements OverlayModel { this.baseUri = baseUri; } - public WikibaseSchema(){ + public WikibaseSchema() { } @@ -58,7 +58,12 @@ public class WikibaseSchema implements OverlayModel { return itemDocumentExprs; } - public List evaluate(ExpressionContext ctxt) { + /** + * Evaluates all item documents in a particular expression context. + * @param ctxt + * @return + */ + public List evaluateItemDocuments(ExpressionContext ctxt) { List result = new ArrayList(); for (WbItemDocumentExpr expr : itemDocumentExprs) { @@ -92,7 +97,7 @@ public class WikibaseSchema implements OverlayModel { @Override public boolean visit(Project project, int rowIndex, Row row) { ExpressionContext ctxt = new ExpressionContext(baseUri, row, project.columnModel); - result.addAll(evaluate(ctxt)); + result.addAll(evaluateItemDocuments(ctxt)); return false; }