s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 474 - implement record limit for XML and JSON importers

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2359 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2011-11-05 16:38:19 +00:00
parent b36b229ba4
commit 85a37d23f9
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
View File

@ -189,7 +189,17 @@ abstract public class TreeImportingParserBase implements ImportingParser {
List<Exception> exceptions List<Exception> exceptions
) { ) {
String[] recordPath = JSONUtilities.getStringArray(options, "recordPath"); String[] recordPath = JSONUtilities.getStringArray(options, "recordPath");
int limit2 = JSONUtilities.getInt(options, "limit", -1);
XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit); if (limit > 0) {
if (limit2 > 0) {
limit2 = Math.min(limit, limit2);
} else {
limit2 = limit;
}
}
if (limit2 == 0) { // shouldn't really happen, but be sure since 0 is stop signal
limit2 = -1;
}
XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit2);
} }
} }

11
main/src/com/google/refine/importers/tree/XmlImportUtilities.java
View File

@ -246,14 +246,14 @@ public class XmlImportUtilities extends TreeImportUtilities {
) { ) {
logger.trace("importTreeData(TreeReader, Project, String[], ImportColumnGroup)"); logger.trace("importTreeData(TreeReader, Project, String[], ImportColumnGroup)");
try { try {
while (parser.hasNext() && (limit <= 0 || project.rows.size() < limit)) { while (parser.hasNext()) {
Token eventType = parser.next(); Token eventType = parser.next();
if (eventType == Token.StartEntity) { if (eventType == Token.StartEntity) {
findRecord(project, parser, recordPath, 0, rootColumnGroup, limit); findRecord(project, parser, recordPath, 0, rootColumnGroup, limit--);
logger.info("Project rows after findRecord = "+project.rows.size());
} }
} }
} catch (TreeReaderException e) { } catch (TreeReaderException e) {
// TODO: This error needs to be reported to the browser/user
logger.error("Exception from XML parse",e); logger.error("Exception from XML parse",e);
} }
} }
@ -290,11 +290,10 @@ public class XmlImportUtilities extends TreeImportUtilities {
String fullName = composeName(parser.getPrefix(), localName); String fullName = composeName(parser.getPrefix(), localName);
if (recordPathSegment.equals(localName) || recordPathSegment.equals(fullName)) { if (recordPathSegment.equals(localName) || recordPathSegment.equals(fullName)) {
if (pathIndex < recordPath.length - 1) { if (pathIndex < recordPath.length - 1) {
while (parser.hasNext() && (limit <= 0 || project.rows.size() < limit)) { while (parser.hasNext() && limit != 0) {
Token eventType = parser.next(); Token eventType = parser.next();
if (eventType == Token.StartEntity) { if (eventType == Token.StartEntity) {
// TODO: find instead of process?? findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit--);
findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit);
} else if (eventType == Token.EndEntity) { } else if (eventType == Token.EndEntity) {
break; break;
} else if (eventType == Token.Value) { } else if (eventType == Token.Value) {