From 6bc9a6ea8e48c6dbe1dc341ee70fba330057797f Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Sat, 2 Feb 2019 16:25:00 +0000 Subject: [PATCH 1/4] Do not represent time values with precision > 11. Closes #1944. --- .../wikidata/schema/WbDateConstant.java | 21 +++++++++---------- .../wikidata/schema/WbDateConstantTest.java | 11 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java index a6b297601..b1ac73f2e 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java @@ -60,10 +60,7 @@ public class WbDateConstant implements WbExpression { .put(new SimpleDateFormat("yyyy"), 9) .put(new SimpleDateFormat("yyyy-MM"), 10) .put(new SimpleDateFormat("yyyy-MM-dd"), 11) - .put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12) - .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13) - .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"), 13) - .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14).build(); + .build(); private TimeValue parsed; private String origDatestamp; @@ -100,25 +97,27 @@ public class WbDateConstant implements WbExpression { */ public static TimeValue parse(String datestamp) throws ParseException { - Date date = null; - int precision = 9; // default precision (will be overridden) + Date bestDate = null; + int precision = 0; // default precision (will be overridden if successfully parsed) + int maxLength = 0; // the maximum length parsed for (Entry entry : acceptedFormats.entrySet()) { ParsePosition position = new ParsePosition(0); String trimmedDatestamp = datestamp.trim(); - date = entry.getKey().parse(trimmedDatestamp, position); + Date date = entry.getKey().parse(trimmedDatestamp, position); // Ignore parses which failed or do not consume all the input - if (date != null && position.getIndex() == trimmedDatestamp.length()) { + if (date != null && position.getIndex() > maxLength) { precision = entry.getValue(); - break; + bestDate = date; + maxLength = position.getIndex(); } } - if (date == null) { + if (bestDate == null || precision == 0) { throw new ParseException("Invalid date.", 0); } else { Calendar calendar = Calendar.getInstance(); calendar = Calendar.getInstance(); - calendar.setTime(date); + calendar.setTime(bestDate); return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1), (byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY), (byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 0, diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java index d7d31c92b..b455b4f7b 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java @@ -37,7 +37,7 @@ public class WbDateConstantTest extends WbExpressionTest { private WbDateConstant month = new WbDateConstant("2018-02"); private WbDateConstant day = new WbDateConstant("2018-02-27"); private WbDateConstant whitespace = new WbDateConstant(" 2018-02-27 "); - private WbDateConstant hour = new WbDateConstant("2018-02-27T13"); + private WbDateConstant second = new WbDateConstant("2017-01-03T04:12:45"); @Test public void testSerialize() { @@ -62,8 +62,8 @@ public class WbDateConstantTest extends WbExpressionTest { TimeValue.CM_GREGORIAN_PRO), month); evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO), day); - evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 13, (byte) 0, (byte) 0, (byte) 12, 0, 0, - 0, TimeValue.CM_GREGORIAN_PRO), hour); + evaluatesTo(Datamodel.makeTimeValue(2017, (byte) 1, (byte) 3, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, + TimeValue.CM_GREGORIAN_PRO), second); evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO), whitespace); @@ -74,8 +74,9 @@ public class WbDateConstantTest extends WbExpressionTest { new WbDateConstant("invalid format"); } - @Test(expectedExceptions = IllegalArgumentException.class) + @Test public void testPartlyValid() { - new WbDateConstant("2018-partly valid"); + evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 1, (byte) 1, (byte) 0, (byte) 0, (byte) 0, (byte) 9, 0, 0, 0, + TimeValue.CM_GREGORIAN_PRO), new WbDateConstant("2018-partly valid")); } } From 5534e84b29397fb89c1d08f54bda32be6df38ed7 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 19 Feb 2019 22:20:11 +0000 Subject: [PATCH 2/4] Refine implementation thanks to the tests --- .../src/org/openrefine/wikidata/schema/WbDateConstant.java | 4 +++- .../org/openrefine/wikidata/schema/WbDateVariableTest.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java index b1ac73f2e..8c7909bad 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java @@ -106,7 +106,9 @@ public class WbDateConstant implements WbExpression { Date date = entry.getKey().parse(trimmedDatestamp, position); // Ignore parses which failed or do not consume all the input - if (date != null && position.getIndex() > maxLength) { + if (date != null && position.getIndex() > maxLength + // only allow to partially consume the input if the precision is more than a year + && (entry.getValue() > 9 || position.getIndex() == trimmedDatestamp.length())) { precision = entry.getValue(); bestDate = date; maxLength = position.getIndex(); diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java index 9238624ee..2150afc1e 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java @@ -36,7 +36,7 @@ public class WbDateVariableTest extends WbVariableTest { 0, 0, 0, TimeValue.CM_GREGORIAN_PRO); private TimeValue day = Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO); - private TimeValue minute = Datamodel.makeTimeValue(2001, (byte) 2, (byte) 3, (byte)4, (byte)5, (byte)0, (byte)13, (byte)0, (byte)0, (byte)0, TimeValue.CM_GREGORIAN_PRO); + private TimeValue minute = Datamodel.makeTimeValue(2001, (byte) 2, (byte) 3, (byte)0, (byte)0, (byte)0, (byte)11, (byte)0, (byte)0, (byte)0, TimeValue.CM_GREGORIAN_PRO); @Override @@ -71,6 +71,7 @@ public class WbDateVariableTest extends WbVariableTest { @Test public void testMinutesISO() { + // Wikidata currently only supports up to day precision evaluatesTo(minute, "2001-02-03T04:05Z"); } From b157651e9ef59c0cbcfaf5cff08dbc4ee9ad582c Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 19 Feb 2019 22:32:50 +0000 Subject: [PATCH 3/4] Fix Wikidata date parsing tests --- .../src/org/openrefine/wikidata/schema/WbDateVariable.java | 2 +- .../org/openrefine/wikidata/schema/WbDateConstantTest.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java index 355bbd99c..8de86d7ad 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java @@ -54,7 +54,7 @@ public class WbDateVariable extends WbVariableExpr { public TimeValue fromCell(Cell cell, ExpressionContext ctxt) throws SkipSchemaExpressionException { try { - // TODO accept parsed dates (without converting them to strings) + // parsed dates are accepted by converting them to strings return WbDateConstant.parse(cell.value.toString()); } catch (ParseException e) { if(!cell.value.toString().isEmpty()) { diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java index b455b4f7b..d65d3e55a 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java @@ -38,6 +38,7 @@ public class WbDateConstantTest extends WbExpressionTest { private WbDateConstant day = new WbDateConstant("2018-02-27"); private WbDateConstant whitespace = new WbDateConstant(" 2018-02-27 "); private WbDateConstant second = new WbDateConstant("2017-01-03T04:12:45"); + private WbDateConstant secondz = new WbDateConstant("2017-01-03T04:12:45Z"); @Test public void testSerialize() { @@ -64,6 +65,8 @@ public class WbDateConstantTest extends WbExpressionTest { TimeValue.CM_GREGORIAN_PRO), day); evaluatesTo(Datamodel.makeTimeValue(2017, (byte) 1, (byte) 3, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO), second); + evaluatesTo(Datamodel.makeTimeValue(2017, (byte) 1, (byte) 3, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, + TimeValue.CM_GREGORIAN_PRO), secondz); evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, (byte) 0, (byte) 0, (byte) 0, (byte) 11, 0, 0, 0, TimeValue.CM_GREGORIAN_PRO), whitespace); @@ -76,7 +79,6 @@ public class WbDateConstantTest extends WbExpressionTest { @Test public void testPartlyValid() { - evaluatesTo(Datamodel.makeTimeValue(2018, (byte) 1, (byte) 1, (byte) 0, (byte) 0, (byte) 0, (byte) 9, 0, 0, 0, - TimeValue.CM_GREGORIAN_PRO), new WbDateConstant("2018-partly valid")); + isSkipped(new WbDateConstant("2018-partly valid")); } } From 6d820dfea507604b9ce208756cd9ee9122e5b762 Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Tue, 19 Feb 2019 22:46:33 +0000 Subject: [PATCH 4/4] Update frontend date validation and format --- .../module/scripts/dialogs/schema-alignment-dialog.js | 4 ++-- .../org/openrefine/wikidata/schema/WbDateConstantTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js index f9a1ebc8b..66f995dd5 100644 --- a/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js +++ b/extensions/wikidata/module/scripts/dialogs/schema-alignment-dialog.js @@ -834,7 +834,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue, fixSuggestInput(input); } else if (mode === "time") { - input.attr("placeholder", "YYYY(-MM(-DD))..."); + input.attr("placeholder", "YYYY(-MM(-DD))"); var propagateValue = function(val) { // TODO add validation here inputContainer.data("jsonValue", { @@ -848,7 +848,7 @@ SchemaAlignmentDialog._initField = function(inputContainer, mode, initialValue, changedCallback(); }); - SchemaAlignmentDialog.setupStringInputValidation(input, /^\d{4}(-[0-1]\d(-[0-3]\d(T[0-2]\d(:[0-5]\d(:[0-5]\dZ)?)?)?)?)?$/); + SchemaAlignmentDialog.setupStringInputValidation(input, /^\d{4}(-[0-1]\d(-[0-3]\d)?)?$/); } else if (mode === "globe-coordinate") { input.attr("placeholder", "lat,lon"); var propagateValue = function(val) { diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java index d65d3e55a..d7d777034 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateConstantTest.java @@ -77,8 +77,8 @@ public class WbDateConstantTest extends WbExpressionTest { new WbDateConstant("invalid format"); } - @Test + @Test(expectedExceptions = IllegalArgumentException.class) public void testPartlyValid() { - isSkipped(new WbDateConstant("2018-partly valid")); + new WbDateConstant("2018-partly valid"); } }