s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert illegal characters into legal ones. (#2431)

Browse Source 
* Convert illegal characters into leagal ones.

* Test tab in key & value string

Also fix up test that depended on previous TAB
related error message and clean up logging

Co-authored-by: Tom Morris <tfmorris@gmail.com>
This commit is contained in:
chuhao zeng 2020-06-14 03:47:58 -04:00 committed by GitHub
parent 3a7812aef7
commit 9b03ecae41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 43 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
main/src/com/google/refine/importers/JsonImporter.java
View File

@ -90,6 +90,7 @@ public class JsonImporter extends TreeImportingParserBase {
File file = ImportingUtilities.getFile(job, firstFileRecord);
JsonFactory factory = new JsonFactory();
JsonParser parser = factory.createParser(file);
parser.enable(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS);
PreviewParsingState state = new PreviewParsingState();
JsonNode rootValue = parseForPreview(parser, state);
@ -224,6 +225,7 @@ public class JsonImporter extends TreeImportingParserBase {
public JSONTreeReader(InputStream is) {
try {
parser = factory.createParser(is);
parser.enable(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS);
current = null;
next = parser.nextToken();
} catch (IOException e) {

62
main/tests/server/src/com/google/refine/importers/JsonImporterTests.java
View File

@ -38,11 +38,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.lang.reflect.Method;
import java.util.LinkedList;
import java.util.List;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.ITestResult;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
@ -75,16 +77,16 @@ public class JsonImporterTests extends ImporterTest {
//System Under Test
JsonImporter SUT = null;
@Override
@BeforeMethod
public void setUp(){
public void setUp(Method method){
super.setUp();
SUT = new JsonImporter();
logger.info("About to run test method: " + method.getName());
}
@Override
@AfterMethod
public void tearDown() {
public void tearDown(ITestResult result) {
// logger.info("Finished test method: " + result.getMethod().getMethodName());
SUT = null;
if (inputStream != null) {
try {
@ -100,8 +102,6 @@ public class JsonImporterTests extends ImporterTest {
@Test
public void canParseSample(){
RunTest(getSample());
log(project);
assertProjectCreated(project, 4, 6);
Row row = project.rows.get(0);
@ -122,7 +122,6 @@ public class JsonImporterTests extends ImporterTest {
JSONUtilities.safePut(options, "storeEmptyStrings", true);
JSONUtilities.safePut(options, "guessCellValueTypes", false);
try {
inputStream = new ByteArrayInputStream(errJSON.getBytes( "UTF-8" ) );
} catch (UnsupportedEncodingException e1) {
@ -143,8 +142,8 @@ public class JsonImporterTests extends ImporterTest {
exceptions
);
Assert.assertFalse(exceptions.isEmpty());
Assert.assertEquals(exceptions.get(0).getMessage(), "Illegal unquoted " +
"character ((CTRL-CHAR, code 10)): has to be escaped using backslash to be included in string value");
Assert.assertEquals("Unexpected character (';' (code 59)): was expecting comma to separate Object entries",
exceptions.get(0).getMessage());
}
@Test
@ -159,7 +158,6 @@ public class JsonImporterTests extends ImporterTest {
" }\n" +
"]\n";
RunTest(ScraperwikiOutput, true);
log(project);
assertProjectCreated(project, 4, 1);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
@ -180,7 +178,6 @@ public class JsonImporterTests extends ImporterTest {
" }\n" +
"]\n";
RunTest(ScraperwikiOutput);
log(project);
assertProjectCreated(project, 4, 1);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
@ -192,8 +189,6 @@ public class JsonImporterTests extends ImporterTest {
@Test
public void canParseSampleWithDuplicateNestedElements(){
RunTest(getSampleWithDuplicateNestedElements());
log(project);
assertProjectCreated(project, 4, 12);
Row row = project.rows.get(0);
@ -206,10 +201,7 @@ public class JsonImporterTests extends ImporterTest {
@Test
public void testCanParseLineBreak(){
RunTest(getSampleWithLineBreak());
log(project);
assertProjectCreated(project, 4, 6);
Row row = project.rows.get(3);
@ -222,8 +214,6 @@ public class JsonImporterTests extends ImporterTest {
@Test
public void testElementsWithVaryingStructure(){
RunTest(getSampleWithVaryingStructure());
log(project);
assertProjectCreated(project, 5, 6);
Assert.assertEquals( project.columnModel.getColumnByCellIndex(4).getName(), JsonImporter.ANONYMOUS + " - genre");
@ -240,7 +230,6 @@ public class JsonImporterTests extends ImporterTest {
@Test
public void testElementWithNestedTree(){
RunTest(getSampleWithTreeStructure());
log(project);
assertProjectCreated(project, 5, 6);
Assert.assertEquals(project.columnModel.columnGroups.size(),1);
@ -264,7 +253,6 @@ public class JsonImporterTests extends ImporterTest {
JSONUtilities.safePut(options, "recordPath", path);
RunTest(mqlOutput, options);
log(project);
assertProjectCreated(project,3,16);
}
@ -298,7 +286,6 @@ public class JsonImporterTests extends ImporterTest {
" }\n" +
"]\n";
RunTest(ScraperwikiOutput);
log(project);
assertProjectCreated(project,9,2);
}
@ -379,11 +366,36 @@ public class JsonImporterTests extends ImporterTest {
}
}
@Test
public void testCanParseTab() throws Exception {
// Use un-escaped tabs here.
String sampleJson = "{\"\tfield\":\t\"\tvalue\"}";
JSONTreeReader parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson.getBytes("UTF-8")));
Token token = Token.Ignorable;
int i = 0;
try {
while (token != null) {
token = parser.next();
if (token == null) {
break;
}
i++;
if (i == 3) {
Assert.assertEquals(Token.Value, token);
Assert.assertEquals("\tfield", parser.getFieldName());
Assert.assertEquals("\tvalue", parser.getFieldValue());
}
}
}catch(Exception e){
Assert.fail();
}
}
@Test
public void testJsonDatatypes(){
RunTest(getSampleWithDataTypes());
log(project);
assertProjectCreated(project, 2, 21,4);
Assert.assertEquals( project.columnModel.getColumnByCellIndex(0).getName(), JsonImporter.ANONYMOUS + " - id");
@ -454,12 +466,10 @@ public class JsonImporterTests extends ImporterTest {
String fileName = "grid_small.json";
RunComplexJSONTest(getComplexJSON(fileName));
log(project);
logger.info("************************ columnu number:" + project.columnModel.columns.size() +
logger.debug("************************ columnu number:" + project.columnModel.columns.size() +
". \tcolumn groups number:" + project.columnModel.columnGroups.size() +
".\trow number:" + project.rows.size() + ".\trecord number:" + project.recordModel.getRecordCount()) ;
assertProjectCreated(project, 63, 63, 8);
}