Make schema evaluation null-proof; avoid adding null values in data extension. (#3723)
Null values in non-null cells should normally not happen, but sometimes they do and we shouldn't fail miserably in those cases. Closes #2880. Co-authored-by: Tom Morris <tfmorris@gmail.com>
This commit is contained in:
parent
787c272fe0
commit
e664712a70
@ -53,6 +53,9 @@ public class WbDateVariable extends WbVariableExpr<TimeValue> {
|
|||||||
@Override
|
@Override
|
||||||
public TimeValue fromCell(Cell cell, ExpressionContext ctxt)
|
public TimeValue fromCell(Cell cell, ExpressionContext ctxt)
|
||||||
throws SkipSchemaExpressionException {
|
throws SkipSchemaExpressionException {
|
||||||
|
if (cell == null || cell.value == null) {
|
||||||
|
throw new SkipSchemaExpressionException();
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
// parsed dates are accepted by converting them to strings
|
// parsed dates are accepted by converting them to strings
|
||||||
return WbDateConstant.parse(cell.value.toString());
|
return WbDateConstant.parse(cell.value.toString());
|
||||||
|
@ -46,6 +46,9 @@ public class WbLocationVariable extends WbVariableExpr<GlobeCoordinatesValue> {
|
|||||||
@Override
|
@Override
|
||||||
public GlobeCoordinatesValue fromCell(Cell cell, ExpressionContext ctxt)
|
public GlobeCoordinatesValue fromCell(Cell cell, ExpressionContext ctxt)
|
||||||
throws SkipSchemaExpressionException {
|
throws SkipSchemaExpressionException {
|
||||||
|
if (cell == null || cell.value == null) {
|
||||||
|
throw new SkipSchemaExpressionException();
|
||||||
|
}
|
||||||
String expr = cell.value.toString();
|
String expr = cell.value.toString();
|
||||||
try {
|
try {
|
||||||
return WbLocationConstant.parse(expr);
|
return WbLocationConstant.parse(expr);
|
||||||
|
@ -56,7 +56,7 @@ public class WbStringVariable extends WbVariableExpr<StringValue> {
|
|||||||
@Override
|
@Override
|
||||||
public StringValue fromCell(Cell cell, ExpressionContext ctxt)
|
public StringValue fromCell(Cell cell, ExpressionContext ctxt)
|
||||||
throws SkipSchemaExpressionException {
|
throws SkipSchemaExpressionException {
|
||||||
if (!cell.value.toString().isEmpty()) {
|
if (cell != null && !cell.value.toString().isEmpty()) {
|
||||||
String stringValue = cell.value.toString();
|
String stringValue = cell.value.toString();
|
||||||
if (cell.value instanceof Double && ((Double)cell.value) % 1 == 0) {
|
if (cell.value instanceof Double && ((Double)cell.value) % 1 == 0) {
|
||||||
stringValue = Long.toString(((Double)cell.value).longValue());
|
stringValue = Long.toString(((Double)cell.value).longValue());
|
||||||
|
@ -61,6 +61,16 @@ public class WbDateVariableTest extends WbVariableTest<TimeValue> {
|
|||||||
isSkipped(" 2018-XX");
|
isSkipped(" 2018-XX");
|
||||||
isSkipped("invalid format");
|
isSkipped("invalid format");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNullStringValue() {
|
||||||
|
isSkipped((String) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNullCell() {
|
||||||
|
isSkipped((Cell) null);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testNumber() {
|
public void testNumber() {
|
||||||
|
@ -82,6 +82,16 @@ public class WbItemVariableTest extends WbVariableTest<ItemIdValue> {
|
|||||||
public void testUnreconciledCell() {
|
public void testUnreconciledCell() {
|
||||||
isSkipped("some value");
|
isSkipped("some value");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNullCell() {
|
||||||
|
isSkipped((Cell) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNullStringValue() {
|
||||||
|
isSkipped((String) null);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSerialize() {
|
public void testSerialize() {
|
||||||
|
@ -26,6 +26,8 @@ package org.openrefine.wikidata.schema;
|
|||||||
import org.openrefine.wikidata.testing.JacksonSerializationTest;
|
import org.openrefine.wikidata.testing.JacksonSerializationTest;
|
||||||
import org.testng.annotations.Test;
|
import org.testng.annotations.Test;
|
||||||
|
|
||||||
|
import com.google.refine.model.Cell;
|
||||||
|
|
||||||
public class WbLanguageVariableTest extends WbVariableTest<String> {
|
public class WbLanguageVariableTest extends WbVariableTest<String> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -46,6 +48,11 @@ public class WbLanguageVariableTest extends WbVariableTest<String> {
|
|||||||
isSkipped((String) null);
|
isSkipped((String) null);
|
||||||
isSkipped("");
|
isSkipped("");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNullCell() {
|
||||||
|
isSkipped((Cell) null);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSerialize() {
|
public void testSerialize() {
|
||||||
|
@ -28,6 +28,8 @@ import org.testng.annotations.Test;
|
|||||||
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
import org.wikidata.wdtk.datamodel.helpers.Datamodel;
|
||||||
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
|
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
|
||||||
|
|
||||||
|
import com.google.refine.model.Cell;
|
||||||
|
|
||||||
public class WbLocationVariableTest extends WbVariableTest<GlobeCoordinatesValue> {
|
public class WbLocationVariableTest extends WbVariableTest<GlobeCoordinatesValue> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -62,6 +64,15 @@ public class WbLocationVariableTest extends WbVariableTest<GlobeCoordinatesValue
|
|||||||
public void testEmpty() {
|
public void testEmpty() {
|
||||||
isSkipped("");
|
isSkipped("");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNullStringValue() {
|
||||||
|
isSkipped((String) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testNullCell() {
|
||||||
|
isSkipped((Cell) null);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSerialize() {
|
public void testSerialize() {
|
||||||
|
@ -40,6 +40,19 @@ public class WbStringVariableTest extends WbVariableTest<StringValue> {
|
|||||||
public void testEmpty() {
|
public void testEmpty() {
|
||||||
isSkipped("");
|
isSkipped("");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This should not normally happen: cell values should
|
||||||
|
* never be null (only whole cells can be null).
|
||||||
|
* But better safe than sorry!
|
||||||
|
*/
|
||||||
|
public void testNullStringValue() {
|
||||||
|
isSkipped((String) null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testNullCell() {
|
||||||
|
isSkipped((Cell) null);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSimpleString() {
|
public void testSimpleString() {
|
||||||
|
@ -270,7 +270,7 @@ public class DataExtensionChange implements Change {
|
|||||||
}
|
}
|
||||||
cell = new Cell(rc.name, recon);
|
cell = new Cell(rc.name, recon);
|
||||||
} else {
|
} else {
|
||||||
cell = new Cell((Serializable) value, null);
|
cell = value == null ? null : new Cell((Serializable) value, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
row.setCell(_firstNewCellIndex + c, cell);
|
row.setCell(_firstNewCellIndex + c, cell);
|
||||||
|
@ -249,7 +249,7 @@ public class ExtendDataOperationTests extends RefineTest {
|
|||||||
"{"
|
"{"
|
||||||
+ "\"rows\": {"
|
+ "\"rows\": {"
|
||||||
+ " \"Q794\": {\"P297\": [{\"str\": \"IR\"}]},"
|
+ " \"Q794\": {\"P297\": [{\"str\": \"IR\"}]},"
|
||||||
+ " \"Q863\": {\"P297\": [{\"str\": \"TJ\"}]},"
|
+ " \"Q863\": {\"P297\": []},"
|
||||||
+ " \"Q30\": {\"P297\": [{\"str\": \"US\"}]},"
|
+ " \"Q30\": {\"P297\": [{\"str\": \"US\"}]},"
|
||||||
+ " \"Q17\": {\"P297\": [{\"str\": \"JP\"}]}"
|
+ " \"Q17\": {\"P297\": [{\"str\": \"JP\"}]}"
|
||||||
+ "},"
|
+ "},"
|
||||||
@ -270,7 +270,7 @@ public class ExtendDataOperationTests extends RefineTest {
|
|||||||
// Inspect rows
|
// Inspect rows
|
||||||
Assert.assertTrue("IR".equals(project.rows.get(0).getCellValue(1)), "Bad country code for Iran.");
|
Assert.assertTrue("IR".equals(project.rows.get(0).getCellValue(1)), "Bad country code for Iran.");
|
||||||
Assert.assertTrue("JP".equals(project.rows.get(1).getCellValue(1)), "Bad country code for Japan.");
|
Assert.assertTrue("JP".equals(project.rows.get(1).getCellValue(1)), "Bad country code for Japan.");
|
||||||
Assert.assertTrue("TJ".equals(project.rows.get(2).getCellValue(1)), "Bad country code for Tajikistan.");
|
Assert.assertNull(project.rows.get(2).getCell(1), "Expected a null country code.");
|
||||||
Assert.assertTrue("US".equals(project.rows.get(3).getCellValue(1)), "Bad country code for United States.");
|
Assert.assertTrue("US".equals(project.rows.get(3).getCellValue(1)), "Bad country code for United States.");
|
||||||
|
|
||||||
// Make sure we did not create any recon stats for that column (no reconciled value)
|
// Make sure we did not create any recon stats for that column (no reconciled value)
|
||||||
|
Loading…
Reference in New Issue
Block a user