s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor of CreateProjectCommand.java and Importers

Browse Source 
The code for determining if an importer is suitable to import a file is now in each respective importer rather than in CreateProjectCommand.  There is an additional method, canImportData, on the Importer interface to support this.

CreateProjectCommand registers Importers from a Hashtable (this is a copy of Tom's code for registering commands in Gridworks Servlet).  Plugging in new importers should be simpler.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@861 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Iain Sproat 2010-05-26 13:18:48 +00:00
parent 017a825600
commit 1c47ff476b
7 changed files with 291 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

222
src/main/java/com/metaweb/gridworks/commands/project/CreateProjectCommand.java
View File

@ -45,12 +45,8 @@ import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.importers.ExcelImporter;
import com.metaweb.gridworks.importers.Importer;
import com.metaweb.gridworks.importers.MarcImporter;
import com.metaweb.gridworks.importers.RdfTripleImporter;
import com.metaweb.gridworks.importers.TsvCsvImporter;
import com.metaweb.gridworks.importers.XmlImporter;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.IOUtils;
import com.metaweb.gridworks.util.ParsingUtilities;
@ -58,11 +54,76 @@ import com.metaweb.gridworks.util.ParsingUtilities;
public class CreateProjectCommand extends Command {
final static Logger logger = LoggerFactory.getLogger("create-project_command");
static final private Map<String, Importer> importers = new HashMap<String, Importer>();
private static final String[][] importerNames = {
{"ExcelImporter", "com.metaweb.gridworks.importers.ExcelImporter"},
{"XmlImporter", "com.metaweb.gridworks.importers.XmlImporter"},
{"RdfTripleImporter", "com.metaweb.gridworks.importers.RdfTripleImporter"},
{"MarcImporter", "com.metaweb.gridworks.importers.MarcImporter"},
{"TsvCsvImporter", "com.metaweb.gridworks.importers.TsvCsvImporter"},
};
static {
registerImporters(importerNames);
}
static public boolean registerImporters(String[][] importers) {
boolean status = true;
for (String[] importer : importerNames) {
String importerName = importer[0];
String className = importer[1];
logger.debug("Loading command " + importerName + " class: " + className);
Importer cmd;
try {
// TODO: May need to use the servlet container's class loader here
cmd = (Importer) Class.forName(className).newInstance();
} catch (InstantiationException e) {
logger.error("Failed to load importer class " + className, e);
status = false;
continue;
} catch (IllegalAccessException e) {
logger.error("Failed to load importer class " + className, e);
status = false;
continue;
} catch (ClassNotFoundException e) {
logger.error("Failed to load importer class " + className, e);
status = false;
continue;
}
status |= registerImporter(importerName, cmd);
}
return status;
}
/**
* Register a single importer.
*
* @param name
* importer verb for importer
* @param commandObject
* object implementing the importer
* @return true if importer was loaded and registered successfully
*/
static public boolean registerImporter(String name,
Importer importerObject) {
if (importers.containsKey(name)) {
return false;
}
importers.put(name, importerObject);
return true;
}
// Currently only for test purposes
static protected boolean unregisterImporter(String verb) {
return importers.remove(verb) != null;
}
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
ProjectManager.singleton.setBusy(true);
try {
/*
@ -73,9 +134,9 @@ public class CreateProjectCommand extends Command {
* Don't call request.getParameter() before calling internalImport().
*/
Properties options = ParsingUtilities.parseUrlParameters(request);
Project project = new Project();
internalImport(request, project, options);
/*
@ -91,7 +152,7 @@ public class CreateProjectCommand extends Command {
ProjectManager.singleton.registerProject(project, pm);
project.update();
redirect(response, "/project.html?project=" + project.id);
} catch (Exception e) {
redirect(response, "/error.html?redirect=index.html&msg=" +
@ -102,7 +163,7 @@ public class CreateProjectCommand extends Command {
ProjectManager.singleton.setBusy(false);
}
}
protected void internalImport(
HttpServletRequest request,
Project project,
@ -111,7 +172,7 @@ public class CreateProjectCommand extends Command {
ServletFileUpload upload = new ServletFileUpload();
String url = null;
FileItemIterator iter = upload.getItemIterator(request);
while (iter.hasNext()) {
FileItemStream item = iter.next();
@ -138,32 +199,32 @@ public class CreateProjectCommand extends Command {
stream.close();
}
}
}
}
if (url != null && url.length() > 0) {
internalImportURL(request, project, options, url);
}
}
static class SafeInputStream extends FilterInputStream {
public SafeInputStream(InputStream stream) {
super(stream);
}
@Override
public void close() {
// some libraries attempt to close the input stream while they can't
// read anymore from it... unfortunately this behavior prevents
// some libraries attempt to close the input stream while they can't
// read anymore from it... unfortunately this behavior prevents
// the zip input stream from functioning correctly so we just have
// to ignore those close() calls and just close it ourselves
// forcefully later
}
public void reallyClose() throws IOException {
super.close();
}
}
protected void internalImportFile(
Project project,
Properties options,
@ -172,13 +233,13 @@ public class CreateProjectCommand extends Command {
) throws Exception {
logger.info("Importing '{}'", fileName);
if (fileName.endsWith(".zip") || fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz") || fileName.endsWith(".tar.bz2")) {
// first, save the file on disk, since we need two passes and we might
// first, save the file on disk, since we need two passes and we might
// not have enough memory to keep it all in there
File file = save(inputStream);
// in the first pass, gather statistics about what files are in there
// unfortunately, we have to rely on files extensions, which is horrible but
// better than nothing
@ -186,9 +247,9 @@ public class CreateProjectCommand extends Command {
FileInputStream fis = new FileInputStream(file);
InputStream is = getStream(fileName, fis);
// NOTE(SM): unfortunately, java.io does not provide any generalized class for
// archive-like input streams so while both TarInputStream and ZipInputStream
// NOTE(SM): unfortunately, java.io does not provide any generalized class for
// archive-like input streams so while both TarInputStream and ZipInputStream
// behave precisely the same, there is no polymorphic behavior so we have
// to treat each instance explicitly... one of those times you wish you had
// closures
@ -224,10 +285,10 @@ public class CreateProjectCommand extends Command {
if (values.size() == 0) {
throw new RuntimeException("The archive contains no files.");
}
// this will contain the set of extensions we'll load from the archive
HashSet<String> exts = new HashSet<String>();
// find the extension that is most frequent or those who share the highest frequency value
if (values.size() == 1) {
exts.add(values.get(0).getKey());
@ -245,7 +306,7 @@ public class CreateProjectCommand extends Command {
}
}
}
logger.info("Most frequent extensions: {}", exts.toString());
// second pass, load the data for real
@ -299,9 +360,9 @@ public class CreateProjectCommand extends Command {
return o2.getValue() - o1.getValue();
}
}
private void load(Project project, Properties options, String fileName, InputStream inputStream) throws Exception {
Importer importer = guessImporter(options, null, fileName);
Importer importer = guessImporter(null, fileName);
internalInvokeImporter(project, importer, options, inputStream, null);
}
@ -312,7 +373,7 @@ public class CreateProjectCommand extends Command {
is.close();
return temp;
}
private void mapExtension(String name, Map<String,Integer> ext_map) {
String ext = getExtension(name)[1];
if (ext_map.containsKey(ext)) {
@ -323,7 +384,7 @@ public class CreateProjectCommand extends Command {
}
private InputStream getStream(String fileName, InputStream is) throws IOException {
if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
if (fileName.endsWith(".tar.gz") || fileName.endsWith(".tgz")) {
return new TarInputStream(new GZIPInputStream(is));
} else if (fileName.endsWith(".tar.bz2")) {
return new TarInputStream(new CBZip2InputStream(is));
@ -331,7 +392,7 @@ public class CreateProjectCommand extends Command {
return new ZipInputStream(is);
}
}
private String[] getExtension(String filename) {
String[] result = new String[2];
int ext_index = filename.lastIndexOf('.');
@ -339,7 +400,7 @@ public class CreateProjectCommand extends Command {
result[1] = (ext_index == -1) ? "" : filename.substring(ext_index + 1);
return result;
}
protected void internalImportURL(
HttpServletRequest request,
Project project,
@ -348,7 +409,7 @@ public class CreateProjectCommand extends Command {
) throws Exception {
URL url = new URL(urlString);
URLConnection connection = null;
try {
connection = url.openConnection();
connection.setConnectTimeout(5000);
@ -356,27 +417,26 @@ public class CreateProjectCommand extends Command {
} catch (Exception e) {
throw new Exception("Cannot connect to " + urlString, e);
}
InputStream inputStream = null;
try {
inputStream = connection.getInputStream();
} catch (Exception e) {
throw new Exception("Cannot retrieve content from " + url, e);
}
try {
Importer importer = guessImporter(
options,
connection.getContentType(),
url.getPath()
);
internalInvokeImporter(project, importer, options, inputStream, connection.getContentEncoding());
} finally {
inputStream.close();
}
}
protected void internalInvokeImporter(
Project project,
Importer importer,
@ -387,48 +447,48 @@ public class CreateProjectCommand extends Command {
if (importer.takesReader()) {
BufferedInputStream inputStream = new BufferedInputStream(rawInputStream);
// NOTE(SM): The ICU4J char detection code requires the input stream to support mark/reset.
// Unfortunately, not all ServletInputStream implementations are marking, so we need do
// this memory-expensive wrapping to make it work. It's far from ideal but I don't have
// NOTE(SM): The ICU4J char detection code requires the input stream to support mark/reset.
// Unfortunately, not all ServletInputStream implementations are marking, so we need do
// this memory-expensive wrapping to make it work. It's far from ideal but I don't have
// a more efficient solution.
byte[] bytes = new byte[1024 * 4];
inputStream.mark(bytes.length);
inputStream.read(bytes);
inputStream.reset();
CharsetDetector detector = new CharsetDetector();
detector.setDeclaredEncoding("utf8"); // most of the content on the web is encoded in UTF-8 so start with that
Reader reader = null;
CharsetMatch[] charsetMatches = detector.setText(bytes).detectAll();
for (CharsetMatch charsetMatch : charsetMatches) {
try {
reader = new InputStreamReader(inputStream, charsetMatch.getName());
options.setProperty("encoding", charsetMatch.getName());
options.setProperty("encoding_confidence", Integer.toString(charsetMatch.getConfidence()));
logger.info("Best encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence());
break;
} catch (UnsupportedEncodingException e) {
// silent
}
}
if (reader == null) { // when all else fails
reader = encoding != null ?
new InputStreamReader(inputStream, encoding) :
new InputStreamReader(inputStream);
}
importer.read(reader, project, options);
} else {
importer.read(rawInputStream, project, options);
}
}
}
protected void internalInvokeImporter(
Project project,
Importer importer,
@ -437,58 +497,14 @@ public class CreateProjectCommand extends Command {
) throws Exception {
importer.read(reader, project, options);
}
protected Importer guessImporter(
Properties options, String contentType, String fileName) {
if (contentType != null) {
contentType = contentType.toLowerCase().trim();
if ("application/msexcel".equals(contentType) ||
"application/x-msexcel".equals(contentType) ||
"application/x-ms-excel".equals(contentType) ||
"application/vnd.ms-excel".equals(contentType) ||
"application/x-excel".equals(contentType) ||
"application/xls".equals(contentType)) {
return new ExcelImporter(false);
} else if("application/x-xls".equals(contentType)) {
return new ExcelImporter(true);
} else if("application/xml".equals(contentType) ||
"text/xml".equals(contentType) ||
"application/rss+xml".equals(contentType) ||
"application/atom+xml".equals(contentType)) {
return new XmlImporter();
} else if("application/rdf+xml".equals(contentType)) {
return new RdfTripleImporter();
} else if ("application/marc".equals(contentType)) {
return new MarcImporter();
}
} else if (fileName != null) {
fileName = fileName.toLowerCase();
if (fileName.endsWith(".xls")) {
return new ExcelImporter(false);
} else if (fileName.endsWith(".xlsx")) {
return new ExcelImporter(true);
} else if (
fileName.endsWith(".xml") ||
fileName.endsWith(".atom") ||
fileName.endsWith(".rss")
) {
return new XmlImporter();
} else if (
fileName.endsWith(".rdf")) {
return new RdfTripleImporter();
} else if (
fileName.endsWith(".mrc") ||
fileName.endsWith(".marc") ||
fileName.contains(".mrc.") ||
fileName.contains(".marc.")
) {
return new MarcImporter();
protected Importer guessImporter(String contentType, String fileName) {
for(Importer i : importers.values()){
if(i.canImportData(contentType, fileName)){
return i;
}
}
return new TsvCsvImporter();
return new TsvCsvImporter(); //default
}
}

108
src/main/java/com/metaweb/gridworks/importers/ExcelImporter.java
View File

@ -27,16 +27,12 @@ import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.Recon.Judgment;
public class ExcelImporter implements Importer {
final protected boolean _xmlBased;
public ExcelImporter(boolean xmlBased) {
_xmlBased = xmlBased;
}
protected boolean _xmlBased;
public boolean takesReader() {
return false;
}
public void read(Reader reader, Project project, Properties options) throws Exception {
throw new UnsupportedOperationException();
}
@ -45,11 +41,11 @@ public class ExcelImporter implements Importer {
int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1);
int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
int skip = ImporterUtilities.getIntegerOption("skip",options,0);
Workbook wb = null;
try {
wb = _xmlBased ?
new XSSFWorkbook(inputStream) :
wb = _xmlBased ?
new XSSFWorkbook(inputStream) :
new HSSFWorkbook(new POIFSFileSystem(inputStream));
} catch (IOException e) {
throw new Exception(
@ -58,16 +54,16 @@ public class ExcelImporter implements Importer {
e
);
}
Sheet sheet = wb.getSheetAt(0);
int firstRow = sheet.getFirstRowNum();
int lastRow = sheet.getLastRowNum();
int r = firstRow;
List<Integer> nonBlankIndices = null;
List<String> nonBlankHeaderStrings = null;
/*
* Find the header row
*/
@ -79,13 +75,13 @@ public class ExcelImporter implements Importer {
ignoreLines--;
continue;
}
short firstCell = row.getFirstCellNum();
short lastCell = row.getLastCellNum();
if (firstCell >= 0 && firstCell <= lastCell) {
nonBlankIndices = new ArrayList<Integer>(lastCell - firstCell + 1);
nonBlankHeaderStrings = new ArrayList<String>(lastCell - firstCell + 1);
for (int c = firstCell; c <= lastCell; c++) {
org.apache.poi.ss.usermodel.Cell cell = row.getCell(c);
if (cell != null) {
@ -96,18 +92,18 @@ public class ExcelImporter implements Importer {
}
}
}
if (nonBlankIndices.size() > 0) {
r++;
break;
}
}
}
if (nonBlankIndices == null || nonBlankIndices.size() == 0) {
return;
}
/*
* Create columns
*/
@ -117,59 +113,59 @@ public class ExcelImporter implements Importer {
if (nameToIndex.containsKey(cell)) {
int index = nameToIndex.get(cell);
nameToIndex.put(cell, index + 1);
cell = cell.contains(" ") ? (cell + " " + index) : (cell + index);
} else {
nameToIndex.put(cell, 2);
}
Column column = new Column(c, cell);
project.columnModel.columns.add(column);
}
/*
* Now process the data rows
*/
int rowsWithData = 0;
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (; r <= lastRow; r++) {
org.apache.poi.ss.usermodel.Row row = sheet.getRow(r);
if (row == null) {
continue;
}
short firstCell = row.getFirstCellNum();
short lastCell = row.getLastCellNum();
if (firstCell >= 0 && firstCell <= lastCell) {
Row newRow = new Row(nonBlankIndices.size());
boolean hasData = false;
for (int c = 0; c < nonBlankIndices.size(); c++) {
if (c < firstCell || c > lastCell) {
continue;
}
org.apache.poi.ss.usermodel.Cell cell = row.getCell(c);
if (cell == null) {
continue;
}
int cellType = cell.getCellType();
if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_ERROR ||
if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_ERROR ||
cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BLANK) {
continue;
}
if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_FORMULA) {
cellType = cell.getCachedFormulaResultType();
}
Serializable value = null;
if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_BOOLEAN) {
value = cell.getBooleanCellValue();
} else if (cellType == org.apache.poi.ss.usermodel.Cell.CELL_TYPE_NUMERIC) {
double d = cell.getNumericCellValue();
if (HSSFDateUtil.isCellDateFormatted(cell)) {
value = HSSFDateUtil.getJavaDate(d);
} else {
@ -181,23 +177,23 @@ public class ExcelImporter implements Importer {
value = text;
}
}
if (value != null) {
Recon recon = null;
Hyperlink hyperlink = cell.getHyperlink();
if (hyperlink != null) {
String url = hyperlink.getAddress();
if (url.startsWith("http://") ||
if (url.startsWith("http://") ||
url.startsWith("https://")) {
final String sig = "freebase.com/view";
int i = url.indexOf(sig);
if (i > 0) {
String id = url.substring(i + sig.length());
int q = id.indexOf('?');
if (q > 0) {
id = id.substring(0, q);
@ -206,7 +202,7 @@ public class ExcelImporter implements Importer {
if (h > 0) {
id = id.substring(0, h);
}
if (reconMap.containsKey(id)) {
recon = reconMap.get(id);
recon.judgmentBatchSize++;
@ -219,26 +215,26 @@ public class ExcelImporter implements Importer {
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
recon.addCandidate(recon.match);
reconMap.put(id, recon);
}
}
}
}
newRow.setCell(c, new Cell(value, recon));
hasData = true;
}
}
if (hasData) {
rowsWithData++;
if (skip <= 0 || rowsWithData > skip) {
project.rows.add(newRow);
project.columnModel.setMaxCellIndex(newRow.cells.size());
if (limit > 0 && project.rows.size() >= limit) {
break;
}
@ -247,4 +243,32 @@ public class ExcelImporter implements Importer {
}
}
}
public boolean canImportData(String contentType, String fileName) {
if (contentType != null) {
contentType = contentType.toLowerCase().trim();
if ("application/msexcel".equals(contentType) ||
"application/x-msexcel".equals(contentType) ||
"application/x-ms-excel".equals(contentType) ||
"application/vnd.ms-excel".equals(contentType) ||
"application/x-excel".equals(contentType) ||
"application/xls".equals(contentType)) {
this._xmlBased = false;
return true;
} else if("application/x-xls".equals(contentType)) {
this._xmlBased = true;
return true;
}
} else if (fileName != null) {
fileName = fileName.toLowerCase();
if (fileName.endsWith(".xls")) {
this._xmlBased = false;
return true;
} else if (fileName.endsWith(".xlsx")) {
this._xmlBased = true;
return true;
}
}
return false;
}
}

4
src/main/java/com/metaweb/gridworks/importers/Importer.java
View File

@ -8,7 +8,9 @@ import com.metaweb.gridworks.model.Project;
public interface Importer {
public boolean takesReader();
public void read(Reader reader, Project project, Properties options) throws Exception;
public void read(InputStream inputStream, Project project, Properties options) throws Exception;
public boolean canImportData(String contentType, String fileName);
}

33
src/main/java/com/metaweb/gridworks/importers/MarcImporter.java
View File

@ -20,21 +20,21 @@ public class MarcImporter implements Importer {
public boolean takesReader() {
return false;
}
public void read(Reader reader, Project project, Properties options)
throws Exception {
throw new UnsupportedOperationException();
}
public void read(
InputStream inputStream,
InputStream inputStream,
Project project,
Properties options
) throws Exception {
int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
int skip = ImporterUtilities.getIntegerOption("skip",options,0);
File tempFile = File.createTempFile("gridworks-import-", ".marc.xml");
try {
OutputStream os = new FileOutputStream(tempFile);
@ -45,7 +45,7 @@ public class MarcImporter implements Importer {
true
);
MarcWriter writer = new MarcXmlWriter(os, true);
int count = 0;
while (reader.hasNext()) {
Record record = reader.next();
@ -64,7 +64,7 @@ public class MarcImporter implements Importer {
} finally {
os.close();
}
InputStream is = new FileInputStream(tempFile);
try {
new XmlImporter().read(is, project, options);
@ -75,4 +75,25 @@ public class MarcImporter implements Importer {
tempFile.delete();
}
}
public boolean canImportData(String contentType, String fileName) {
if (contentType != null) {
contentType = contentType.toLowerCase().trim();
if ("application/marc".equals(contentType)) {
return true;
}
} else if (fileName != null) {
fileName = fileName.toLowerCase();
if (
fileName.endsWith(".mrc") ||
fileName.endsWith(".marc") ||
fileName.contains(".mrc.") ||
fileName.contains(".marc.")
) {
return true;
}
}
return false;
}
}

37
src/main/java/com/metaweb/gridworks/importers/RdfTripleImporter.java
View File

@ -43,25 +43,25 @@ public class RdfTripleImporter implements Importer{
@Override
public void read(Reader reader, Project project, Properties options) throws Exception {
String baseUrl = options.getProperty("base-url");
Graph graph = JrdfFactory.getNewGraph();
LineHandler lineHandler = nTriplesParserFactory.createParser(graph, newMapFactory);
GraphLineParser parser = new GraphLineParser(graph, lineHandler);
parser.parse(reader, baseUrl); // fills JRDF graph
Map<String, List<Row>> subjectToRows = new HashMap<String, List<Row>>();
Column subjectColumn = new Column(0, "subject");
project.columnModel.columns.add(0, subjectColumn);
project.columnModel.setKeyColumnIndex(0);
ClosableIterable<Triple> triples = graph.find(ANY_SUBJECT_NODE, ANY_PREDICATE_NODE, ANY_OBJECT_NODE);
try {
for (Triple triple : triples) {
String subject = triple.getSubject().toString();
String predicate = triple.getPredicate().toString();
String object = triple.getObject().toString();
Column column = project.columnModel.getColumnByName(predicate);
if (column == null) {
column = new Column(project.columnModel.allocateNewCellIndex(), predicate);
@ -71,7 +71,7 @@ public class RdfTripleImporter implements Importer{
// ignore
}
}
int cellIndex = column.getCellIndex();
if (subjectToRows.containsKey(subject)) {
List<Row> rows = subjectToRows.get(subject);
@ -82,20 +82,20 @@ public class RdfTripleImporter implements Importer{
break;
}
}
if (object != null) {
Row row = new Row(project.columnModel.getMaxCellIndex() + 1);
rows.add(row);
row.setCell(cellIndex, new Cell(object, null));
}
} else {
List<Row> rows = new ArrayList<Row>();
subjectToRows.put(subject, rows);
Row row = new Row(project.columnModel.getMaxCellIndex() + 1);
rows.add(row);
row.setCell(subjectColumn.getCellIndex(), new Cell(subject, null));
row.setCell(cellIndex, new Cell(object, null));
}
@ -120,4 +120,21 @@ public class RdfTripleImporter implements Importer{
return true;
}
public boolean canImportData(String contentType, String fileName) {
if (contentType != null) {
contentType = contentType.toLowerCase().trim();
if("application/rdf+xml".equals(contentType)) {
return true;
}
} else if (fileName != null) {
fileName = fileName.toLowerCase();
if (
fileName.endsWith(".rdf")) {
return true;
}
}
return false;
}
}

18
src/main/java/com/metaweb/gridworks/importers/TsvCsvImporter.java
View File

@ -2,6 +2,7 @@ package com.metaweb.gridworks.importers;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.Reader;
import java.util.ArrayList;
@ -151,10 +152,25 @@ public class TsvCsvImporter implements Importer {
}
public void read(InputStream inputStream, Project project, Properties options) throws Exception {
throw new UnsupportedOperationException();
read(new InputStreamReader(inputStream), project, options);
}
public boolean takesReader() {
return true;
}
public boolean canImportData(String contentType, String fileName) {
if (contentType != null) {
contentType = contentType.toLowerCase().trim();
return false;
} else if (fileName != null) {
fileName = fileName.toLowerCase();
if (fileName.endsWith(".tsv")) {
return true;
}else if (fileName.endsWith(".csv")){
return true;
}
}
return false;
}
}

41
src/main/java/com/metaweb/gridworks/importers/XmlImporter.java
View File

@ -12,24 +12,24 @@ import com.metaweb.gridworks.model.Project;
public class XmlImporter implements Importer {
public static final int BUFFER_SIZE = 64 * 1024;
public boolean takesReader() {
return false;
}
public void read(Reader reader, Project project, Properties options)
throws Exception {
throw new UnsupportedOperationException();
}
public void read(
InputStream inputStream,
InputStream inputStream,
Project project,
Properties options
) throws Exception {
PushbackInputStream pis = new PushbackInputStream(inputStream,BUFFER_SIZE);
String[] recordPath = null;
{
byte[] buffer = new byte[BUFFER_SIZE];
@ -40,10 +40,10 @@ public class XmlImporter implements Importer {
bytes_read +=c ;
}
pis.unread(buffer, 0, bytes_read);
if (options.containsKey("importer-record-tag")) {
recordPath = XmlImportUtilities.detectPathFromTag(
new ByteArrayInputStream(buffer, 0, bytes_read),
new ByteArrayInputStream(buffer, 0, bytes_read),
options.getProperty("importer-record-tag"));
} else {
recordPath = XmlImportUtilities.detectRecordElement(
@ -52,11 +52,34 @@ public class XmlImporter implements Importer {
}
ImportColumnGroup rootColumnGroup = new ImportColumnGroup();
XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup);
XmlImportUtilities.createColumnsFromImport(project, rootColumnGroup);
project.columnModel.update();
}
public boolean canImportData(String contentType, String fileName) {
if (contentType != null) {
contentType = contentType.toLowerCase().trim();
if("application/xml".equals(contentType) ||
"text/xml".equals(contentType) ||
"application/rss+xml".equals(contentType) ||
"application/atom+xml".equals(contentType)) {
return true;
}
} else if (fileName != null) {
fileName = fileName.toLowerCase();
if (
fileName.endsWith(".xml") ||
fileName.endsWith(".atom") ||
fileName.endsWith(".rss")
) {
return true;
}
}
return false;
}
}