package com.google.refine.importers; import java.io.IOException; import java.io.LineNumberReader; import java.io.Reader; import java.util.ArrayList; import java.util.List; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.refine.ProjectMetadata; import com.google.refine.importing.ImportingJob; import com.google.refine.model.Project; import com.google.refine.util.JSONUtilities; public class LineBasedImporter extends TabularImportingParserBase { static final Logger logger = LoggerFactory.getLogger(LineBasedImporter.class); public LineBasedImporter() { super(false); } @Override public JSONObject createParserUIInitializationData( ImportingJob job, List fileRecords, String format) { JSONObject options = super.createParserUIInitializationData(job, fileRecords, format); JSONUtilities.safePut(options, "linesPerRow", 1); JSONUtilities.safePut(options, "headerLines", 0); JSONUtilities.safePut(options, "guessCellValueTypes", true); return options; } @Override public void parseOneFile( Project project, ProjectMetadata metadata, ImportingJob job, String fileSource, Reader reader, int limit, JSONObject options, List exceptions ) { final int linesPerRow = JSONUtilities.getInt(options, "linesPerRow", 1); final List columnNames; if (options.has("columnNames")) { columnNames = new ArrayList(); String[] strings = JSONUtilities.getStringArray(options, "columnNames"); for (String s : strings) { columnNames.add(s); } JSONUtilities.safePut(options, "headerLines", 1); } else { columnNames = null; JSONUtilities.safePut(options, "headerLines", 0); } final LineNumberReader lnReader = new LineNumberReader(reader); try { int skip = JSONUtilities.getInt(options, "ignoreLines", -1); while (skip > 0) { lnReader.readLine(); skip--; } } catch (IOException e) { logger.error("Error reading line-based file", e); } JSONUtilities.safePut(options, "ignoreLines", -1); TableDataReader dataReader = new TableDataReader() { boolean usedColumnNames = false; @Override public List getNextRowOfCells() throws IOException { if (columnNames != null && !usedColumnNames) { usedColumnNames = true; return columnNames; } else { List cells = null; for (int i = 0; i < linesPerRow; i++) { String line = lnReader.readLine(); if (i == 0) { if (line == null) { return null; } else { cells = new ArrayList(linesPerRow); cells.add(line); } } else if (line != null) { cells.add(line); } else { break; } } return cells; } } }; TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions); } }