s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1946 from OpenRefine/issue1944

Browse Source 
Do not represent time values with precision > 11.
This commit is contained in:
Antonin Delpeuch 2019-02-20 03:09:02 +00:00 committed by GitHub
commit 7b2e2b5894
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 23 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js
View File

@ -834,7 +834,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue,
fixSuggestInput(input); fixSuggestInput(input);
} else if (mode === "time") { } else if (mode === "time") {
input.attr("placeholder", "YYYY(-MM(-DD))..."); input.attr("placeholder", "YYYY(-MM(-DD))");
var propagateValue = function(val) { var propagateValue = function(val) {
// TODO add validation here // TODO add validation here
inputContainer.data("jsonValue", { inputContainer.data("jsonValue", {
@ -848,7 +848,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue,
changedCallback(); changedCallback();
}); });
SchemaAlignmentDialog.setupStringInputValidation(input, /^\d{4}(-[0-1]\d(-[0-3]\d(T[0-2]\d(:[0-5]\d(:[0-5]\dZ)?)?)?)?)?$/); SchemaAlignmentDialog.setupStringInputValidation(input, /^\d{4}(-[0-1]\d(-[0-3]\d)?)?$/);
} else if (mode === "globe-coordinate") { } else if (mode === "globe-coordinate") {
input.attr("placeholder", "lat,lon"); input.attr("placeholder", "lat,lon");
var propagateValue = function(val) { var propagateValue = function(val) {

23
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java
View File

@ -60,10 +60,7 @@ public class WbDateConstant implements WbExpression<TimeValue> {
.put(new SimpleDateFormat("yyyy"), 9) .put(new SimpleDateFormat("yyyy"), 9)
.put(new SimpleDateFormat("yyyy-MM"), 10) .put(new SimpleDateFormat("yyyy-MM"), 10)
.put(new SimpleDateFormat("yyyy-MM-dd"), 11) .put(new SimpleDateFormat("yyyy-MM-dd"), 11)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12) .build();
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"), 13)
.put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14).build();
private TimeValue parsed; private TimeValue parsed;
private String origDatestamp; private String origDatestamp;
@ -100,25 +97,29 @@ public class WbDateConstant implements WbExpression<TimeValue> {
*/ */
public static TimeValue parse(String datestamp) public static TimeValue parse(String datestamp)
throws ParseException { throws ParseException {
Date date = null; Date bestDate = null;
int precision = 9; // default precision (will be overridden) int precision = 0; // default precision (will be overridden if successfully parsed)
int maxLength = 0; // the maximum length parsed
for (Entry<SimpleDateFormat, Integer> entry : acceptedFormats.entrySet()) { for (Entry<SimpleDateFormat, Integer> entry : acceptedFormats.entrySet()) {
ParsePosition position = new ParsePosition(0); ParsePosition position = new ParsePosition(0);
String trimmedDatestamp = datestamp.trim(); String trimmedDatestamp = datestamp.trim();
date = entry.getKey().parse(trimmedDatestamp, position); Date date = entry.getKey().parse(trimmedDatestamp, position);
// Ignore parses which failed or do not consume all the input // Ignore parses which failed or do not consume all the input
if (date != null && position.getIndex() == trimmedDatestamp.length()) { if (date != null && position.getIndex() > maxLength
// only allow to partially consume the input if the precision is more than a year
&& (entry.getValue() > 9 || position.getIndex() == trimmedDatestamp.length())) {
precision = entry.getValue(); precision = entry.getValue();
break; bestDate = date;
maxLength = position.getIndex();
} }
} }
if (date == null) { if (bestDate == null || precision == 0) {
throw new ParseException("Invalid date.", 0); throw new ParseException("Invalid date.", 0);
} else { } else {
Calendar calendar = Calendar.getInstance(); Calendar calendar = Calendar.getInstance();
calendar = Calendar.getInstance(); calendar = Calendar.getInstance();
calendar.setTime(date); calendar.setTime(bestDate);
return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1), return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1),
(byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY), (byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY),
(byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 0, (byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 0,

2
extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java
View File

@ -54,7 +54,7 @@ public class WbDateVariable extends WbVariableExpr<TimeValue> {
public TimeValue fromCell(Cell cell, ExpressionContext ctxt) public TimeValue fromCell(Cell cell, ExpressionContext ctxt)
throws SkipSchemaExpressionException { throws SkipSchemaExpressionException {
try { try {
// TODO accept parsed dates (without converting them to strings) // parsed dates are accepted by converting them to strings
return WbDateConstant.parse(cell.value.toString()); return WbDateConstant.parse(cell.value.toString());
} catch (ParseException e) { } catch (ParseException e) {
if(!cell.value.toString().isEmpty()) { if(!cell.value.toString().isEmpty()) {

9
extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java
View File

@ -37,7 +37,8 @@ public class WbDateConstantTest extends WbExpressionTest<TimeValue> {
private WbDateConstant month = new WbDateConstant("2018-02"); private WbDateConstant month = new WbDateConstant("2018-02");
private WbDateConstant day = new WbDateConstant("2018-02-27"); private WbDateConstant day = new WbDateConstant("2018-02-27");
private WbDateConstant whitespace = new WbDateConstant(" 2018-02-27 "); private WbDateConstant whitespace = new WbDateConstant(" 2018-02-27 ");
private WbDateConstant hour = new WbDateConstant("2018-02-27T13"); private WbDateConstant second = new WbDateConstant("2017-01-03T04:12:45");
private WbDateConstant secondz = new WbDateConstant("2017-01-03T04:12:45Z");
@Test @Test
public void testSerialize() { public void testSerialize() {
@ -62,8 +63,10 @@ public class WbDateConstantTest extends WbExpressionTest<TimeValue> {
TimeValue.CM_GREGORIAN_PRO), month); TimeValue.CM_GREGORIAN_PRO), month);
evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0,
TimeValue.CM_GREGORIAN_PRO), day); TimeValue.CM_GREGORIAN_PRO), day);
evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 13, (byte) 0, (byte) 0, (byte) 12, 0, 0, evaluatesTo(Datamodel.makeTimeValue(2017, (byte) 1, (byte) 3, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0,
0, TimeValue.CM_GREGORIAN_PRO), hour); TimeValue.CM_GREGORIAN_PRO), second);
evaluatesTo(Datamodel.makeTimeValue(2017, (byte) 1, (byte) 3, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0,
TimeValue.CM_GREGORIAN_PRO), secondz);
evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0,
TimeValue.CM_GREGORIAN_PRO), whitespace); TimeValue.CM_GREGORIAN_PRO), whitespace);

3
extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java
View File

@ -36,7 +36,7 @@ public class WbDateVariableTest extends WbVariableTest<TimeValue> {
0, 0, 0, TimeValue.CM_GREGORIAN_PRO); 0, 0, 0, TimeValue.CM_GREGORIAN_PRO);
private TimeValue day = Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, private TimeValue day = Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11,
0, 0, 0, TimeValue.CM_GREGORIAN_PRO); 0, 0, 0, TimeValue.CM_GREGORIAN_PRO);
private TimeValue minute = Datamodel.makeTimeValue(2001, (byte) 2, (byte) 3, (byte)4, (byte)5, (byte)0, (byte)13, (byte)0, (byte)0, (byte)0, TimeValue.CM_GREGORIAN_PRO); private TimeValue minute = Datamodel.makeTimeValue(2001, (byte) 2, (byte) 3, (byte)0, (byte)0, (byte)0, (byte)11, (byte)0, (byte)0, (byte)0, TimeValue.CM_GREGORIAN_PRO);
@Override @Override
@ -71,6 +71,7 @@ public class WbDateVariableTest extends WbVariableTest<TimeValue> {
@Test @Test
public void testMinutesISO() { public void testMinutesISO() {
// Wikidata currently only supports up to day precision
evaluatesTo(minute, "2001-02-03T04:05Z"); evaluatesTo(minute, "2001-02-03T04:05Z");
} }