diff --git a/extensions/wikidata/module/langs/translation-en.json b/extensions/wikidata/module/langs/translation-en.json index 9806d62b4..e3a9cf53c 100644 --- a/extensions/wikidata/module/langs/translation-en.json +++ b/extensions/wikidata/module/langs/translation-en.json @@ -200,6 +200,22 @@ "invalid-identifier-space": { "title": "Invalid identifier space for reconciled cells.", "body": "Some reconciled cells such as {example_cell} were ignored because they are not reconciled to Wikidata." + }, + "ignored-language": { + "title": "Invalid language identifiers.", + "body": "Some language identifiers are invalid, such as {example_value}. See the allowed values." + }, + "ignored-date": { + "title": "Invalid date formats.", + "body": "Some dates are incorrectly formatted, such as {example_value}. See the allowed formats." + }, + "ignored-amount": { + "title": "Invalid amount formats.", + "body": "Some amounts are incorrectly formatted, such as {example_value}. See the allowed formats." + }, + "ignored-coordinates": { + "title": "Invalid geographic coordinates.", + "body": "Some coordinates are incorrectly formatted, such as {example_value}. See the allowed formats." } } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java index 1a5209d39..1f9b41ac2 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateConstant.java @@ -55,9 +55,12 @@ public class WbDateConstant implements WbExpression { * precision it induces (an integer according to Wikibase's data model). */ public static Map acceptedFormats = ImmutableMap. builder() - .put(new SimpleDateFormat("yyyy"), 9).put(new SimpleDateFormat("yyyy-MM"), 10) - .put(new SimpleDateFormat("yyyy-MM-dd"), 11).put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12) + .put(new SimpleDateFormat("yyyy"), 9) + .put(new SimpleDateFormat("yyyy-MM"), 10) + .put(new SimpleDateFormat("yyyy-MM-dd"), 11) + .put(new SimpleDateFormat("yyyy-MM-dd'T'HH"), 12) .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"), 13) + .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"), 13) .put(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), 14).build(); private TimeValue parsed; @@ -114,10 +117,7 @@ public class WbDateConstant implements WbExpression { Calendar calendar = Calendar.getInstance(); calendar = Calendar.getInstance(); calendar.setTime(date); - return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1), // java - // starts - // at - // 0 + return Datamodel.makeTimeValue(calendar.get(Calendar.YEAR), (byte) (calendar.get(Calendar.MONTH) + 1), (byte) calendar.get(Calendar.DAY_OF_MONTH), (byte) calendar.get(Calendar.HOUR_OF_DAY), (byte) calendar.get(Calendar.MINUTE), (byte) calendar.get(Calendar.SECOND), (byte) precision, 0, 1, 0, TimeValue.CM_GREGORIAN_PRO); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java index 3e3675266..5546d5cfb 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbDateVariable.java @@ -25,6 +25,7 @@ package org.openrefine.wikidata.schema; import java.text.ParseException; +import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.interfaces.TimeValue; @@ -57,6 +58,11 @@ public class WbDateVariable extends WbVariableExpr { // TODO accept parsed dates (without converting them to strings) return WbDateConstant.parse(cell.value.toString()); } catch (ParseException e) { + if(!cell.value.toString().isEmpty()) { + QAWarning issue = new QAWarning("ignored-date", null, QAWarning.Severity.WARNING, 1); + issue.setProperty("example_value", cell.value.toString()); + ctxt.addWarning(issue); + } throw new SkipSchemaExpressionException(); } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java index fa46f562a..9107ca193 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLanguageVariable.java @@ -23,6 +23,7 @@ ******************************************************************************/ package org.openrefine.wikidata.schema; +import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import com.fasterxml.jackson.annotation.JsonCreator; @@ -59,6 +60,10 @@ public class WbLanguageVariable extends WbVariableExpr { String normalized = WbLanguageConstant.normalizeLanguageCode(code); if (normalized != null) { return normalized; + } else { + QAWarning issue = new QAWarning("ignored-language", null, QAWarning.Severity.WARNING, 1); + issue.setProperty("example_value", cell.value.toString()); + ctxt.addWarning(issue); } } throw new SkipSchemaExpressionException(); diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java index 7423d6268..c4936df04 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbLocationVariable.java @@ -25,6 +25,7 @@ package org.openrefine.wikidata.schema; import java.text.ParseException; +import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue; @@ -50,6 +51,11 @@ public class WbLocationVariable extends WbVariableExpr { try { return WbLocationConstant.parse(expr); } catch (ParseException e) { + if (!expr.trim().isEmpty()) { + QAWarning issue = new QAWarning("ignored-coordinates", null, QAWarning.Severity.WARNING, 1); + issue.setProperty("example_value", expr); + ctxt.addWarning(issue); + } throw new SkipSchemaExpressionException(); } } diff --git a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java index 94521cbd0..7223a1ffd 100644 --- a/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java +++ b/extensions/wikidata/src/org/openrefine/wikidata/schema/WbQuantityExpr.java @@ -26,6 +26,7 @@ package org.openrefine.wikidata.schema; import java.math.BigDecimal; import org.apache.commons.lang.Validate; +import org.openrefine.wikidata.qa.QAWarning; import org.openrefine.wikidata.schema.exceptions.SkipSchemaExpressionException; import org.wikidata.wdtk.datamodel.helpers.Datamodel; import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue; @@ -66,8 +67,8 @@ public class WbQuantityExpr implements WbExpression { BigDecimal parsedAmount = null; BigDecimal lowerBound = null; BigDecimal upperBound = null; - try { - String originalAmount = amount.getString().toUpperCase(); + String originalAmount = amount.getString().toUpperCase(); + try { parsedAmount = new BigDecimal(originalAmount); @@ -81,6 +82,11 @@ public class WbQuantityExpr implements WbExpression { // workaround for https://github.com/Wikidata/Wikidata-Toolkit/issues/341 parsedAmount = new BigDecimal(parsedAmount.toPlainString()); } catch (NumberFormatException e) { + if (!originalAmount.isEmpty()) { + QAWarning issue = new QAWarning("ignored-amount", null, QAWarning.Severity.WARNING, 1); + issue.setProperty("example_value", originalAmount); + ctxt.addWarning(issue); + } throw new SkipSchemaExpressionException(); } diff --git a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java index f375057ac..5fb1372e4 100644 --- a/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java +++ b/extensions/wikidata/tests/src/org/openrefine/wikidata/schema/WbDateVariableTest.java @@ -35,7 +35,9 @@ public class WbDateVariableTest extends WbVariableTest { private TimeValue year = Datamodel.makeTimeValue(2018, (byte) 1, (byte) 1, (byte) 0, (byte) 0, (byte) 0, (byte) 9, 0, 1, 0, TimeValue.CM_GREGORIAN_PRO); private TimeValue day = Datamodel.makeTimeValue(2018, (byte) 2, (byte) 27, TimeValue.CM_GREGORIAN_PRO); + private TimeValue minute = Datamodel.makeTimeValue(2001, (byte) 2, (byte) 3, (byte)4, (byte)5, (byte)0, (byte)13, (byte)0, (byte)1, (byte)0, TimeValue.CM_GREGORIAN_PRO); + @Override public WbVariableExpr initVariableExpr() { return new WbDateVariable(); @@ -66,7 +68,10 @@ public class WbDateVariableTest extends WbVariableTest { isSkipped(new Cell(1234.56, null)); } - // TODO accept parsed dates with default precision + @Test + public void testMinutesISO() { + evaluatesTo(minute, "2001-02-03T04:05Z"); + } @Test public void testSerialize() {