s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use InputStream instead of Reader for JSON import - fixes #698

Browse Source
This commit is contained in:
Tom Morris 2013-03-23 18:36:05 -04:00
parent 6b3592982e
commit 6a91b5d75b
3 changed files with 893 additions and 902 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
main/src/com/google/refine/importers/JsonImporter.java
View File

@ -34,10 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.importers;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.Serializable;
import java.util.List;
@ -67,7 +65,7 @@ public class JsonImporter extends TreeImportingParserBase {
public final static String ANONYMOUS = "_";
public JsonImporter() {
super(false);
super(true);
}
static private class PreviewParsingState {
@ -84,18 +82,13 @@ public class JsonImporter extends TreeImportingParserBase {
try {
JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord);
InputStream is = new FileInputStream(file);
try {
JsonFactory factory = new JsonFactory();
JsonParser parser = factory.createJsonParser(is);
JsonFactory factory = new JsonFactory();
JsonParser parser = factory.createJsonParser(file);
PreviewParsingState state = new PreviewParsingState();
Object rootValue = parseForPreview(parser, state);
if (rootValue != null) {
JSONUtilities.safePut(options, "dom", rootValue);
}
} finally {
is.close();
PreviewParsingState state = new PreviewParsingState();
Object rootValue = parseForPreview(parser, state);
if (rootValue != null) {
JSONUtilities.safePut(options, "dom", rootValue);
}
} catch (IOException e) {
logger.error("Error generating parser UI initialization data for JSON file", e);
@ -201,11 +194,11 @@ public class JsonImporter extends TreeImportingParserBase {
@Override
public void parseOneFile(Project project, ProjectMetadata metadata,
ImportingJob job, String fileSource, Reader reader,
ImportingJob job, String fileSource, InputStream is,
ImportColumnGroup rootColumnGroup, int limit, JSONObject options, List<Exception> exceptions) {
parseOneFile(project, metadata, job, fileSource,
new JSONTreeReader(reader), rootColumnGroup, limit, options, exceptions);
new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions);
}
static public class JSONTreeReader implements TreeReader {
@ -220,9 +213,9 @@ public class JsonImporter extends TreeImportingParserBase {
private Serializable fieldValue = null;
public JSONTreeReader(Reader reader) {
public JSONTreeReader(InputStream is) {
try {
parser = factory.createJsonParser(reader);
parser = factory.createJsonParser(is);
current = null;
next = parser.nextToken();
} catch (IOException e) {

9
main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java
View File

@ -35,7 +35,6 @@ package com.google.refine.tests.importers;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.LinkedList;
@ -228,7 +227,7 @@ public class JsonImporterTests extends ImporterTest {
String sampleJson2 = "{\"field\":{}}";
String sampleJson3 = "{\"field\":[{},{}]}";
JSONTreeReader parser = new JSONTreeReader(new StringReader(sampleJson));
JSONTreeReader parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson.getBytes("UTF-8")));
Token token = Token.Ignorable;
int i = 0;
try{
@ -248,7 +247,7 @@ public class JsonImporterTests extends ImporterTest {
}
parser = new JSONTreeReader(new StringReader(sampleJson2));
parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson2.getBytes("UTF-8")));
token = Token.Ignorable;
i = 0;
try{
@ -267,7 +266,7 @@ public class JsonImporterTests extends ImporterTest {
//silent
}
parser = new JSONTreeReader(new StringReader(sampleJson3));
parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson3.getBytes("UTF-8")));
token = Token.Ignorable;
i = 0;
try{
@ -505,7 +504,7 @@ public class JsonImporterTests extends ImporterTest {
}
try {
parseOneFile(SUT, inputStream, options);
parseOneInputStream(SUT, inputStream, options);
} catch (Exception e) {
Assert.fail();
}

5
main/tests/server/src/com/google/refine/tests/importers/XmlImportUtilitiesTests.java
View File

@ -35,7 +35,6 @@ package com.google.refine.tests.importers;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
@ -197,7 +196,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
loadSampleJson();
String[] path = XmlImportUtilitiesStub.detectRecordElement(
new JSONTreeReader(new InputStreamReader(inputStream)));
new JSONTreeReader(inputStream));
Assert.assertNotNull(path);
Assert.assertEquals(path.length, 2);
Assert.assertEquals(path[0], JsonImporter.ANONYMOUS);
@ -455,7 +454,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
}
}
public TreeReader createJsonParser(){
parser = new JSONTreeReader(new InputStreamReader(inputStream));
parser = new JSONTreeReader(inputStream);
return parser;
}
}