s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use InputStream instead of Reader for JSON import - fixes #698

Browse Source
This commit is contained in:
Tom Morris 2013-03-23 18:36:05 -04:00
parent 6b3592982e
commit 6a91b5d75b
3 changed files with 893 additions and 902 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
main/src/com/google/refine/importers/JsonImporter.java
View File

@ -34,10 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.importers; package com.google.refine.importers;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.Reader;
import java.io.Serializable; import java.io.Serializable;
import java.util.List; import java.util.List;
@ -67,7 +65,7 @@ public class JsonImporter extends TreeImportingParserBase {
public final static String ANONYMOUS = "_"; public final static String ANONYMOUS = "_";
public JsonImporter() { public JsonImporter() {
super(false); super(true);
} }
static private class PreviewParsingState { static private class PreviewParsingState {
@ -84,18 +82,13 @@ public class JsonImporter extends TreeImportingParserBase {
try { try {
JSONObject firstFileRecord = fileRecords.get(0); JSONObject firstFileRecord = fileRecords.get(0);
File file = ImportingUtilities.getFile(job, firstFileRecord); File file = ImportingUtilities.getFile(job, firstFileRecord);
InputStream is = new FileInputStream(file); JsonFactory factory = new JsonFactory();
try { JsonParser parser = factory.createJsonParser(file);
JsonFactory factory = new JsonFactory();
JsonParser parser = factory.createJsonParser(is);
PreviewParsingState state = new PreviewParsingState(); PreviewParsingState state = new PreviewParsingState();
Object rootValue = parseForPreview(parser, state); Object rootValue = parseForPreview(parser, state);
if (rootValue != null) { if (rootValue != null) {
JSONUtilities.safePut(options, "dom", rootValue); JSONUtilities.safePut(options, "dom", rootValue);
}
} finally {
is.close();
} }
} catch (IOException e) { } catch (IOException e) {
logger.error("Error generating parser UI initialization data for JSON file", e); logger.error("Error generating parser UI initialization data for JSON file", e);
@ -201,11 +194,11 @@ public class JsonImporter extends TreeImportingParserBase {
@Override @Override
public void parseOneFile(Project project, ProjectMetadata metadata, public void parseOneFile(Project project, ProjectMetadata metadata,
ImportingJob job, String fileSource, Reader reader, ImportingJob job, String fileSource, InputStream is,
ImportColumnGroup rootColumnGroup, int limit, JSONObject options, List<Exception> exceptions) { ImportColumnGroup rootColumnGroup, int limit, JSONObject options, List<Exception> exceptions) {
parseOneFile(project, metadata, job, fileSource, parseOneFile(project, metadata, job, fileSource,
new JSONTreeReader(reader), rootColumnGroup, limit, options, exceptions); new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions);
} }
static public class JSONTreeReader implements TreeReader { static public class JSONTreeReader implements TreeReader {
@ -220,9 +213,9 @@ public class JsonImporter extends TreeImportingParserBase {
private Serializable fieldValue = null; private Serializable fieldValue = null;
public JSONTreeReader(Reader reader) { public JSONTreeReader(InputStream is) {
try { try {
parser = factory.createJsonParser(reader); parser = factory.createJsonParser(is);
current = null; current = null;
next = parser.nextToken(); next = parser.nextToken();
} catch (IOException e) { } catch (IOException e) {

9
main/tests/server/src/com/google/refine/tests/importers/JsonImporterTests.java
View File

@ -35,7 +35,6 @@ package com.google.refine.tests.importers;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.LinkedList; import java.util.LinkedList;
@ -228,7 +227,7 @@ public class JsonImporterTests extends ImporterTest {
String sampleJson2 = "{\"field\":{}}"; String sampleJson2 = "{\"field\":{}}";
String sampleJson3 = "{\"field\":[{},{}]}"; String sampleJson3 = "{\"field\":[{},{}]}";
JSONTreeReader parser = new JSONTreeReader(new StringReader(sampleJson)); JSONTreeReader parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson.getBytes("UTF-8")));
Token token = Token.Ignorable; Token token = Token.Ignorable;
int i = 0; int i = 0;
try{ try{
@ -248,7 +247,7 @@ public class JsonImporterTests extends ImporterTest {
} }
parser = new JSONTreeReader(new StringReader(sampleJson2)); parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson2.getBytes("UTF-8")));
token = Token.Ignorable; token = Token.Ignorable;
i = 0; i = 0;
try{ try{
@ -267,7 +266,7 @@ public class JsonImporterTests extends ImporterTest {
//silent //silent
} }
parser = new JSONTreeReader(new StringReader(sampleJson3)); parser = new JSONTreeReader(new ByteArrayInputStream(sampleJson3.getBytes("UTF-8")));
token = Token.Ignorable; token = Token.Ignorable;
i = 0; i = 0;
try{ try{
@ -505,7 +504,7 @@ public class JsonImporterTests extends ImporterTest {
} }
try { try {
parseOneFile(SUT, inputStream, options); parseOneInputStream(SUT, inputStream, options);
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }

5
main/tests/server/src/com/google/refine/tests/importers/XmlImportUtilitiesTests.java
View File

@ -35,7 +35,6 @@ package com.google.refine.tests.importers;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -197,7 +196,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
loadSampleJson(); loadSampleJson();
String[] path = XmlImportUtilitiesStub.detectRecordElement( String[] path = XmlImportUtilitiesStub.detectRecordElement(
new JSONTreeReader(new InputStreamReader(inputStream))); new JSONTreeReader(inputStream));
Assert.assertNotNull(path); Assert.assertNotNull(path);
Assert.assertEquals(path.length, 2); Assert.assertEquals(path.length, 2);
Assert.assertEquals(path[0], JsonImporter.ANONYMOUS); Assert.assertEquals(path[0], JsonImporter.ANONYMOUS);
@ -455,7 +454,7 @@ public class XmlImportUtilitiesTests extends RefineTest {
} }
} }
public TreeReader createJsonParser(){ public TreeReader createJsonParser(){
parser = new JSONTreeReader(new InputStreamReader(inputStream)); parser = new JSONTreeReader(inputStream);
return parser; return parser;
} }
} }