s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 474 - implement record limit for XML and JSON importers

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2359 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2011-11-05 16:38:19 +00:00
parent b36b229ba4
commit 85a37d23f9
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
View File

@ -189,7 +189,17 @@ abstract public class TreeImportingParserBase implements ImportingParser {
List<Exception> exceptions
) {
String[] recordPath = JSONUtilities.getStringArray(options, "recordPath");
XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit);
int limit2 = JSONUtilities.getInt(options, "limit", -1);
if (limit > 0) {
if (limit2 > 0) {
limit2 = Math.min(limit, limit2);
} else {
limit2 = limit;
}
}
if (limit2 == 0) { // shouldn't really happen, but be sure since 0 is stop signal
limit2 = -1;
}
XmlImportUtilities.importTreeData(treeParser, project, recordPath, rootColumnGroup, limit2);
}
}

11
main/src/com/google/refine/importers/tree/XmlImportUtilities.java
View File

@ -246,14 +246,14 @@ public class XmlImportUtilities extends TreeImportUtilities {
) {
logger.trace("importTreeData(TreeReader, Project, String[], ImportColumnGroup)");
try {
while (parser.hasNext() && (limit <= 0 || project.rows.size() < limit)) {
while (parser.hasNext()) {
Token eventType = parser.next();
if (eventType == Token.StartEntity) {
findRecord(project, parser, recordPath, 0, rootColumnGroup, limit);
logger.info("Project rows after findRecord = "+project.rows.size());
findRecord(project, parser, recordPath, 0, rootColumnGroup, limit--);
}
}
} catch (TreeReaderException e) {
// TODO: This error needs to be reported to the browser/user
logger.error("Exception from XML parse",e);
}
}
@ -290,11 +290,10 @@ public class XmlImportUtilities extends TreeImportUtilities {
String fullName = composeName(parser.getPrefix(), localName);
if (recordPathSegment.equals(localName) || recordPathSegment.equals(fullName)) {
if (pathIndex < recordPath.length - 1) {
while (parser.hasNext() && (limit <= 0 || project.rows.size() < limit)) {
while (parser.hasNext() && limit != 0) {
Token eventType = parser.next();
if (eventType == Token.StartEntity) {
// TODO: find instead of process??
findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit);
findRecord(project, parser, recordPath, pathIndex + 1, rootColumnGroup, limit--);
} else if (eventType == Token.EndEntity) {
break;
} else if (eventType == Token.Value) {