Convert illegal characters into legal ones. (#2431)
* Convert illegal characters into leagal ones. * Test tab in key & value string Also fix up test that depended on previous TAB related error message and clean up logging Co-authored-by: Tom Morris <tfmorris@gmail.com>
This commit is contained in:
chuhao zeng
GitHub
parent
3a7812aef7
commit
9b03ecae41
@ -90,6 +90,7 @@ public class JsonImporter extends TreeImportingParserBase {
|
||||
File file = ImportingUtilities.getFile(job, firstFileRecord);
|
||||
JsonFactory factory = new JsonFactory();
|
||||
JsonParser parser = factory.createParser(file);
|
||||
parser.enable(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS);
|
||||
|
||||
PreviewParsingState state = new PreviewParsingState();
|
||||
JsonNode rootValue = parseForPreview(parser, state);
|
||||
@ -224,6 +225,7 @@ public class JsonImporter extends TreeImportingParserBase {
|
||||
public JSONTreeReader(InputStream is) {
|
||||
try {
|
||||
parser = factory.createParser(is);
|
||||
parser.enable(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS);
|
||||
current = null;
|
||||
next = parser.nextToken();
|
||||
} catch (IOException e) {
|
||||
|
||||
@ -38,11 +38,13 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.testng.Assert;
|
||||
import org.testng.ITestResult;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.BeforeTest;
|
||||
@ -75,16 +77,16 @@ public class JsonImporterTests extends ImporterTest {
|
||||
//System Under Test
|
||||
JsonImporter SUT = null;
|
||||
|
||||
@Override
|
||||
@BeforeMethod
|
||||
public void setUp(){
|
||||
public void setUp(Method method){
|
||||
super.setUp();
|
||||
SUT = new JsonImporter();
|
||||
logger.info("About to run test method: " + method.getName());
|
||||
}
|
||||
|
||||
@Override
|
||||
@AfterMethod
|
||||
public void tearDown() {
|
||||
public void tearDown(ITestResult result) {
|
||||
// logger.info("Finished test method: " + result.getMethod().getMethodName());
|
||||
SUT = null;
|
||||
if (inputStream != null) {
|
||||
try {
|
||||
@ -100,8 +102,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
@Test
|
||||
public void canParseSample(){
|
||||
RunTest(getSample());
|
||||
|
||||
log(project);
|
||||
assertProjectCreated(project, 4, 6);
|
||||
|
||||
Row row = project.rows.get(0);
|
||||
@ -122,7 +122,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
JSONUtilities.safePut(options, "storeEmptyStrings", true);
|
||||
JSONUtilities.safePut(options, "guessCellValueTypes", false);
|
||||
|
||||
|
||||
try {
|
||||
inputStream = new ByteArrayInputStream(errJSON.getBytes( "UTF-8" ) );
|
||||
} catch (UnsupportedEncodingException e1) {
|
||||
@ -143,10 +142,10 @@ public class JsonImporterTests extends ImporterTest {
|
||||
exceptions
|
||||
);
|
||||
Assert.assertFalse(exceptions.isEmpty());
|
||||
Assert.assertEquals(exceptions.get(0).getMessage(), "Illegal unquoted " +
|
||||
"character ((CTRL-CHAR, code 10)): has to be escaped using backslash to be included in string value");
|
||||
Assert.assertEquals("Unexpected character (';' (code 59)): was expecting comma to separate Object entries",
|
||||
exceptions.get(0).getMessage());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void trimLeadingTrailingWhitespaceOnTrimStrings(){
|
||||
String ScraperwikiOutput =
|
||||
@ -159,7 +158,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
" }\n" +
|
||||
"]\n";
|
||||
RunTest(ScraperwikiOutput, true);
|
||||
log(project);
|
||||
assertProjectCreated(project, 4, 1);
|
||||
Row row = project.rows.get(0);
|
||||
Assert.assertNotNull(row);
|
||||
@ -180,7 +178,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
" }\n" +
|
||||
"]\n";
|
||||
RunTest(ScraperwikiOutput);
|
||||
log(project);
|
||||
assertProjectCreated(project, 4, 1);
|
||||
Row row = project.rows.get(0);
|
||||
Assert.assertNotNull(row);
|
||||
@ -192,8 +189,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
@Test
|
||||
public void canParseSampleWithDuplicateNestedElements(){
|
||||
RunTest(getSampleWithDuplicateNestedElements());
|
||||
|
||||
log(project);
|
||||
assertProjectCreated(project, 4, 12);
|
||||
|
||||
Row row = project.rows.get(0);
|
||||
@ -206,10 +201,7 @@ public class JsonImporterTests extends ImporterTest {
|
||||
|
||||
@Test
|
||||
public void testCanParseLineBreak(){
|
||||
|
||||
RunTest(getSampleWithLineBreak());
|
||||
|
||||
log(project);
|
||||
assertProjectCreated(project, 4, 6);
|
||||
|
||||
Row row = project.rows.get(3);
|
||||
@ -222,8 +214,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
@Test
|
||||
public void testElementsWithVaryingStructure(){
|
||||
RunTest(getSampleWithVaryingStructure());
|
||||
|
||||
log(project);
|
||||
assertProjectCreated(project, 5, 6);
|
||||
|
||||
Assert.assertEquals( project.columnModel.getColumnByCellIndex(4).getName(), JsonImporter.ANONYMOUS + " - genre");
|
||||
@ -240,7 +230,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
@Test
|
||||
public void testElementWithNestedTree(){
|
||||
RunTest(getSampleWithTreeStructure());
|
||||
log(project);
|
||||
assertProjectCreated(project, 5, 6);
|
||||
|
||||
Assert.assertEquals(project.columnModel.columnGroups.size(),1);
|
||||
@ -264,7 +253,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
JSONUtilities.safePut(options, "recordPath", path);
|
||||
|
||||
RunTest(mqlOutput, options);
|
||||
log(project);
|
||||
assertProjectCreated(project,3,16);
|
||||
}
|
||||
|
||||
@ -298,7 +286,6 @@ public class JsonImporterTests extends ImporterTest {
|
||||
" }\n" +
|
||||
"]\n";
|
||||
RunTest(ScraperwikiOutput);
|
||||
log(project);
|
||||
assertProjectCreated(project,9,2);
|
||||
}
|
||||
|
||||
@ -379,11 +366,36 @@ public class JsonImporterTests extends ImporterTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCanParseTab() throws Exception {
|
||||
// Use un-escaped tabs here.
|
||||
String sampleJson = "{\"\tfield\":\t\"\tvalue\"}";
|
||||
|
||||
JSONTreeReader parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson.getBytes("UTF-8")));
|
||||
Token token = Token.Ignorable;
|
||||
int i = 0;
|
||||
try {
|
||||
while (token != null) {
|
||||
token = parser.next();
|
||||
if (token == null) {
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
if (i == 3) {
|
||||
Assert.assertEquals(Token.Value, token);
|
||||
Assert.assertEquals("\tfield", parser.getFieldName());
|
||||
Assert.assertEquals("\tvalue", parser.getFieldValue());
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testJsonDatatypes(){
|
||||
RunTest(getSampleWithDataTypes());
|
||||
|
||||
log(project);
|
||||
assertProjectCreated(project, 2, 21,4);
|
||||
|
||||
Assert.assertEquals( project.columnModel.getColumnByCellIndex(0).getName(), JsonImporter.ANONYMOUS + " - id");
|
||||
@ -454,15 +466,13 @@ public class JsonImporterTests extends ImporterTest {
|
||||
String fileName = "grid_small.json";
|
||||
RunComplexJSONTest(getComplexJSON(fileName));
|
||||
|
||||
log(project);
|
||||
logger.info("************************ columnu number:" + project.columnModel.columns.size() +
|
||||
". \tcolumn groups number:" + project.columnModel.columnGroups.size() +
|
||||
logger.debug("************************ columnu number:" + project.columnModel.columns.size() +
|
||||
". \tcolumn groups number:" + project.columnModel.columnGroups.size() +
|
||||
".\trow number:" + project.rows.size() + ".\trecord number:" + project.recordModel.getRecordCount()) ;
|
||||
|
||||
|
||||
|
||||
assertProjectCreated(project, 63, 63, 8);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//------------helper methods---------------
|
||||
|
||||
private static String getTypicalElement(int id){
|
||||
|
||||
Loading…
Reference in New Issue
Block a user