s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support for dates and coordinates in Wikibase schema

Browse Source
This commit is contained in:
Antonin Delpeuch 2017-09-18 10:19:38 +01:00
parent 8f4d998e21
commit 165ff41469
11 changed files with 396 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js
View File

@ -323,6 +323,7 @@ SchemaAlignmentDialog._addStatement = function(container, datatype, json) {
SchemaAlignmentDialog._initField(inputContainer, datatype, value);
// If we are in a mainsnak...
// (see https://www.mediawiki.org/wiki/Wikibase/DataModel#Snaks)
if (container.parents('.wbs-statement').length == 0) {
// add qualifiers...
@ -431,6 +432,7 @@ SchemaAlignmentDialog._addReference = function(container, json) {
$('<img src="images/close.png" />').attr('alt', 'remove reference').click(function() {
reference.remove();
SchemaAlignmentDialog._updateReferencesNumber(container);
SchemaAlignmentDialog._hasChanged();
}).appendTo(toolbarRef);
var right = $('<div></div>').addClass('wbs-right').appendTo(reference);
var qualifierContainer = $('<div></div>').addClass('wbs-qualifier-container').appendTo(right);
@ -460,9 +462,7 @@ SchemaAlignmentDialog._referenceToJSON = function(reference) {
SchemaAlignmentDialog._updateReferencesNumber = function(container) {
var childrenCount = container.children().length;
var statement = container.parents('.wbs-statement');
console.log(statement);
var a = statement.find('.wbs-references-toggle a').first();
console.log(a);
a.html(childrenCount+'&nbsp;references');
}
@ -548,6 +548,32 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
});
SchemaAlignmentDialog._hasChanged();
});
} else if (mode === "time") {
var propagateValue = function(val) {
// TODO add validation here
inputContainer.data("jsonValue", {
type: "wbdateconstant",
value: val,
});
};
propagateValue("");
input.change(function() {
propagateValue($(this).val());
SchemaAlignmentDialog._hasChanged();
});
} else if (mode === "globecoordinates") {
var propagateValue = function(val) {
// TODO add validation here
inputContainer.data("jsonValue", {
type: "wblocationconstant",
value: val,
});
};
propagateValue("");
input.change(function() {
propagateValue($(this).val());
SchemaAlignmentDialog._hasChanged();
});
} else { /* if (mode === "external-id") { */
var propagateValue = function(val) {
inputContainer.data("jsonValue", {
@ -581,9 +607,12 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
if (mode === "wikibase-item") {
acceptClass = ".wbs-reconciled-column";
wbVariableType = "wbitemvariable";
} else if (mode === "time") {
wbVariableType = "wbdatevariable";
} else if (mode === "globecoordinates") {
wbVariableType = "wblocationvariable";
}
inputContainer.droppable({
accept: acceptClass,
}).on("drop", function (evt, ui) {
@ -608,9 +637,13 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue)
} else if (initialValue.type == "wbitemvariable") {
var cell = SchemaAlignmentDialog._createDraggableColumn(initialValue.columnName, true);
acceptDraggableColumn(cell);
} else if (initialValue.type == "wbstringconstant") {
} else if (initialValue.type == "wbstringconstant" ||
initialValue.type == "wbdateconstant" ||
initialValue.type == "wblocationconstant") {
input.val(initialValue.value);
} else if (initialValue.type == "wbstringvariable") {
} else if (initialValue.type == "wbstringvariable" ||
initialValue.type == "wbdatevariable" ||
initialValue.type == "wblocationvariable") {
var cell = SchemaAlignmentDialog._createDraggableColumn(initialValue.columnName, false);
acceptDraggableColumn(cell);
}

38
extensions/wikidata/src/org/openrefine/wikidata/exporters/QuickStatementsExporter.java
View File

@ -121,7 +121,8 @@ public class QuickStatementsExporter implements WriterExporter {
@Override
public String visit(DatatypeIdValue value) {
// TODO Auto-generated method stub
// unsupported according to
// https://tools.wmflabs.org/wikidata-todo/quick_statements.php?
return null;
}
@ -135,20 +136,32 @@ public class QuickStatementsExporter implements WriterExporter {
@Override
public String visit(GlobeCoordinatesValue value) {
// TODO Auto-generated method stub
return null;
return String.format(
"@%f/%f",
value.getLatitude(),
value.getLongitude());
}
@Override
public String visit(MonolingualTextValue value) {
// TODO Auto-generated method stub
return null;
return String.format(
"%s:/\"%s\"",
value.getLanguageCode(),
value.getText());
}
@Override
public String visit(QuantityValue value) {
// TODO Auto-generated method stub
return null;
String unitPrefix = "http://www.wikidata.org/entity/Q";
String unit = value.getUnit();
if (!unit.startsWith(unitPrefix))
return null; // QuickStatements only accepts Qids as units
String unitID = "U"+unit.substring(unitPrefix.length());
return String.format(
"[%f,%f]%s",
value.getLowerBound(),
value.getUpperBound(),
unitID);
}
@Override
@ -158,8 +171,15 @@ public class QuickStatementsExporter implements WriterExporter {
@Override
public String visit(TimeValue value) {
// TODO Auto-generated method stub
return null;
return String.format(
"+%04d-%02d-%02dT%02d:%02d:%02dZ/%d",
value.getYear(),
value.getMonth(),
value.getDay(),
value.getHour(),
value.getMinute(),
value.getSecond(),
value.getPrecision());
}
}

101
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java Normal file
View File

@ -0,0 +1,101 @@
package org.openrefine.wikidata.schema;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.google.common.collect.ImmutableMap;
public class WbDateConstant extends WbDateExpr {
public static final String jsonType = "wbdateconstant";
public static Map<SimpleDateFormat,Integer> acceptedFormats = ImmutableMap.<SimpleDateFormat,Integer>builder()
.put(new SimpleDateFormat("yyyy"), 9)
.put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14)
.build();
private TimeValue _parsed;
private String _origDatestamp;
public WbDateConstant(String origDatestamp) {
_origDatestamp = origDatestamp;
try {
_parsed = parse(origDatestamp);
} catch(ParseException e) {
_parsed = null;
}
}
@Override
public TimeValue evaluate(ExpressionContext ctxt)
throws SkipStatementException {
if (_parsed == null) {
throw new SkipStatementException();
}
return _parsed;
}
public static TimeValue parse(String datestamp) throws ParseException {
Date date = null;
int precision = 9; // default precision (will be overridden)
for(Entry<SimpleDateFormat,Integer> entry : acceptedFormats.entrySet()) {
try {
date = entry.getKey().parse(datestamp);
precision = entry.getValue();
} catch (ParseException e) {
continue;
}
}
if (date == null) {
throw new ParseException("Invalid date.", 0);
} else {
Calendar calendar = Calendar.getInstance();
calendar = Calendar.getInstance();
calendar.setTime(date);
return Datamodel.makeTimeValue(
calendar.get(Calendar.YEAR),
(byte) (calendar.get(Calendar.MONTH)+1), // java starts at 0
(byte) calendar.get(Calendar.DAY_OF_MONTH),
(byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE),
(byte) calendar.get(Calendar.SECOND),
(byte) precision,
1,
1,
calendar.getTimeZone().getRawOffset()/3600000,
TimeValue.CM_GREGORIAN_PRO);
}
}
@Override
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.key("value");
writer.value(_origDatestamp);
}
public static WbDateConstant fromJSON(JSONObject obj) throws JSONException {
return new WbDateConstant(obj.getString("value"));
}
@Override
public String getJsonType() {
return jsonType;
}
}

25
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateExpr.java Normal file
View File

@ -0,0 +1,25 @@
package org.openrefine.wikidata.schema;
import org.json.JSONException;
import org.json.JSONObject;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
public abstract class WbDateExpr extends WbValueExpr {
@Override
public abstract TimeValue evaluate(ExpressionContext ctxt)
throws SkipStatementException;
public static WbDateExpr fromJSON(JSONObject obj) throws JSONException {
String type = obj.getString(jsonTypeKey);
if (WbDateConstant.jsonType.equals(type)) {
return WbDateConstant.fromJSON(obj);
} else if (WbDateVariable.jsonType.equals(type)) {
return WbDateVariable.fromJSON(obj);
} else {
throw new JSONException("unknown type for WbDateExpr");
}
}
}

54
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java Normal file
View File

@ -0,0 +1,54 @@
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import com.google.refine.model.Cell;
public class WbDateVariable extends WbDateExpr {
public static final String jsonType = "wbdatevariable";
private String _columnName;
public WbDateVariable(String columnName) {
_columnName = columnName;
}
@Override
public TimeValue evaluate(ExpressionContext ctxt)
throws SkipStatementException {
Cell cell = ctxt.getCellByName(_columnName);
if (cell != null) {
try {
// TODO accept parsed dates (without converting them to strings)
return WbDateConstant.parse(cell.value.toString());
} catch (ParseException e) {
}
}
throw new SkipStatementException();
}
@Override
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.key("columnName");
writer.value(_columnName);
}
public static WbDateVariable fromJSON(JSONObject obj) throws JSONException {
return new WbDateVariable(obj.getString("columnName"));
}
@Override
public String getJsonType() {
return jsonType;
}
}

55
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationConstant.java Normal file
View File

@ -0,0 +1,55 @@
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
public class WbLocationConstant extends WbLocationExpr {
public static final String jsonType = "wblocationconstant";
private String _origValue;
private GlobeCoordinatesValue _parsed;
public WbLocationConstant(String origValue) {
_origValue = origValue;
_parsed = null;
}
public static GlobeCoordinatesValue parse(String expr) throws ParseException {
double lat = 0;
double lng = 0;
double precision = 0;
return Datamodel.makeGlobeCoordinatesValue(lat, lng, precision,
GlobeCoordinatesValue.GLOBE_EARTH);
}
@Override
public GlobeCoordinatesValue evaluate(ExpressionContext ctxt)
throws SkipStatementException {
if (_parsed == null)
throw new SkipStatementException();
return _parsed;
}
public static WbLocationConstant fromJSON(JSONObject obj) throws JSONException {
return new WbLocationConstant(obj.getString("value"));
}
@Override
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.key("value");
writer.value(_origValue);
}
@Override
public String getJsonType() {
return jsonType;
}
}

23
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationExpr.java Normal file
View File

@ -0,0 +1,23 @@
package org.openrefine.wikidata.schema;
import org.json.JSONException;
import org.json.JSONObject;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
public abstract class WbLocationExpr extends WbValueExpr {
@Override
public abstract GlobeCoordinatesValue evaluate(ExpressionContext ctxt)
throws SkipStatementException;
public static WbLocationExpr fromJSON(JSONObject obj) throws JSONException {
String type = obj.getString(jsonTypeKey);
if (WbLocationConstant.jsonType.equals(type)) {
return WbLocationConstant.fromJSON(obj);
} else if (WbLocationVariable.jsonType.equals(type)) {
return WbLocationVariable.fromJSON(obj);
} else {
throw new JSONException("unknown type for WbLocationExpr");
}
}
}

54
extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java Normal file
View File

@ -0,0 +1,54 @@
package org.openrefine.wikidata.schema;
import java.text.ParseException;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import com.google.refine.model.Cell;
public class WbLocationVariable extends WbLocationExpr {
public static final String jsonType = "wblocationvariable";
private String columnName;
public WbLocationVariable(String columnName) {
this.columnName = columnName;
}
@Override
public GlobeCoordinatesValue evaluate(ExpressionContext ctxt)
throws SkipStatementException {
Cell cell = ctxt.getCellByName(columnName);
if (cell != null) {
String expr = cell.value.toString();
try {
return WbLocationConstant.parse(expr);
} catch (ParseException e) {
}
}
throw new SkipStatementException();
}
@Override
public void writeFields(JSONWriter writer, Properties options)
throws JSONException {
writer.key("columnName");
writer.value(columnName);
}
public static WbLocationVariable fromJSON(JSONObject obj) throws JSONException {
return new WbLocationVariable(obj.getString("columnName"));
}
@Override
public String getJsonType() {
return jsonType;
}
}

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbStringExpr.java
View File

@ -15,7 +15,7 @@ public abstract class WbStringExpr extends WbValueExpr {
} else if (WbStringVariable.jsonType.equals(type)) {
return WbStringVariable.fromJSON(obj);
} else {
throw new JSONException("unknown type for WbItemExpr");
throw new JSONException("unknown type for WbStringExpr");
}
}
}

8
extensions/wikidata/src/org/openrefine/wikidata/schema/WbValueExpr.java
View File

@ -30,6 +30,14 @@ public abstract class WbValueExpr extends BiJsonizable {
valueExpr = WbStringVariable.fromJSON(obj);
} else if (WbStringConstant.jsonType.equals(type)) {
valueExpr = WbStringConstant.fromJSON(obj);
} else if (WbDateVariable.jsonType.equals(type)) {
valueExpr = WbDateVariable.fromJSON(obj);
} else if (WbDateConstant.jsonType.equals(type)) {
valueExpr = WbDateConstant.fromJSON(obj);
} else if (WbLocationVariable.jsonType.equals(type)) {
valueExpr = WbLocationVariable.fromJSON(obj);
} else if (WbLocationConstant.jsonType.equals(type)) {
valueExpr = WbLocationConstant.fromJSON(obj);
} else {
throw new JSONException("unknown type '"+type+"' for WbValueExpr");
}

11
extensions/wikidata/src/org/openrefine/wikidata/schema/WikibaseSchema.java
View File

@ -45,7 +45,7 @@ public class WikibaseSchema implements OverlayModel {
this.baseUri = baseUri;
}
public WikibaseSchema(){
public WikibaseSchema() {
}
@ -58,7 +58,12 @@ public class WikibaseSchema implements OverlayModel {
return itemDocumentExprs;
}
public List<ItemUpdate> evaluate(ExpressionContext ctxt) {
/**
* Evaluates all item documents in a particular expression context.
* @param ctxt
* @return
*/
public List<ItemUpdate> evaluateItemDocuments(ExpressionContext ctxt) {
List<ItemUpdate> result = new ArrayList<ItemUpdate>();
for (WbItemDocumentExpr expr : itemDocumentExprs) {
@ -92,7 +97,7 @@ public class WikibaseSchema implements OverlayModel {
@Override
public boolean visit(Project project, int rowIndex, Row row) {
ExpressionContext ctxt = new ExpressionContext(baseUri, row, project.columnModel);
result.addAll(evaluate(ctxt));
result.addAll(evaluateItemDocuments(ctxt));
return false;
}