s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed issue 114: "Refactor project manager api to allow importers to create project metadata" by incorporating tfmorris' patch.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1271 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-09-17 01:00:23 +00:00
parent 8d1f2d44b9
commit 2609c4049d
12 changed files with 73 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
main/src/com/google/gridworks/commands/project/CreateProjectCommand.java
View File

@ -74,15 +74,15 @@ public class CreateProjectCommand extends Command {
Properties options = ParsingUtilities.parseUrlParameters(request);
Project project = new Project();
ProjectMetadata pm = new ProjectMetadata();
internalImport(request, project, options);
internalImport(request, project, pm, options);
/*
* The import process above populates options with parameters
* in the POST body. That's why we're constructing the project
* metadata object after calling internalImport().
*/
ProjectMetadata pm = new ProjectMetadata();
pm.setName(options.getProperty("project-name"));
pm.setPassword(options.getProperty("project-password"));
pm.setEncoding(options.getProperty("encoding"));
@ -105,6 +105,7 @@ public class CreateProjectCommand extends Command {
protected void internalImport(
HttpServletRequest request,
Project project,
ProjectMetadata metadata,
Properties options
) throws Exception {
@ -121,7 +122,7 @@ public class CreateProjectCommand extends Command {
if (name.equals("raw-text")) {
Reader reader = new InputStreamReader(stream,"UTF-8");
try {
internalInvokeImporter(project, new TsvCsvImporter(), options, reader);
internalInvokeImporter(project, new TsvCsvImporter(), metadata, options, reader);
imported = true;
} finally {
reader.close();
@ -135,7 +136,7 @@ public class CreateProjectCommand extends Command {
String fileName = item.getName().toLowerCase();
if (fileName.length() > 0) {
try {
internalImportFile(project, options, fileName, stream);
internalImportFile(project, metadata, options, fileName, stream);
imported = true;
} finally {
stream.close();
@ -145,7 +146,7 @@ public class CreateProjectCommand extends Command {
}
if (!imported && url != null && url.length() > 0) {
internalImportURL(request, project, options, url);
internalImportURL(request, project, metadata, options, url);
}
}
@ -169,10 +170,11 @@ public class CreateProjectCommand extends Command {
}
protected void internalImportFile(
Project project,
Properties options,
String fileName,
InputStream inputStream
Project project,
ProjectMetadata metadata,
Properties options,
String fileName,
InputStream inputStream
) throws Exception {
logger.info("Importing '{}'", fileName);
@ -264,7 +266,7 @@ public class CreateProjectCommand extends Command {
String name = te.getName();
String ext = getExtension(name)[1];
if (exts.contains(ext)) {
internalImportFile(project, options, name, sis);
internalImportFile(project, metadata, options, name, sis);
}
}
}
@ -276,7 +278,7 @@ public class CreateProjectCommand extends Command {
String name = ze.getName();
String ext = getExtension(name)[1];
if (exts.contains(ext)) {
internalImportFile(project, options, name, sis);
internalImportFile(project, metadata, options, name, sis);
}
}
}
@ -288,11 +290,11 @@ public class CreateProjectCommand extends Command {
}
} else if (fileName.endsWith(".gz")) {
internalImportFile(project, options, getExtension(fileName)[0], new GZIPInputStream(inputStream));
internalImportFile(project, metadata, options, getExtension(fileName)[0], new GZIPInputStream(inputStream));
} else if (fileName.endsWith(".bz2")) {
internalImportFile(project, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream));
internalImportFile(project, metadata, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream));
} else {
load(project, options, fileName, inputStream);
load(project, metadata, options, fileName, inputStream);
}
}
@ -304,9 +306,9 @@ public class CreateProjectCommand extends Command {
}
}
private void load(Project project, Properties options, String fileName, InputStream inputStream) throws Exception {
private void load(Project project, ProjectMetadata metadata, Properties options, String fileName, InputStream inputStream) throws Exception {
Importer importer = ImporterRegistry.guessImporter(null, fileName);
internalInvokeImporter(project, importer, options, inputStream, null);
internalInvokeImporter(project, importer, metadata, options, inputStream, null);
}
private File save(InputStream is) throws IOException {
@ -344,16 +346,20 @@ public class CreateProjectCommand extends Command {
return result;
}
protected void internalImportURL(HttpServletRequest request,
Project project, Properties options, String urlString)
throws Exception {
protected void internalImportURL(
HttpServletRequest request,
Project project,
ProjectMetadata metadata,
Properties options,
String urlString) throws Exception {
URL url = new URL(urlString);
URLConnection connection = null;
// Try for a URL importer first
Importer importer = ImporterRegistry.guessUrlImporter(url);
if (importer instanceof UrlImporter) {
((UrlImporter) importer).read(url, project, options);
((UrlImporter) importer).read(url, project, metadata, options);
} else {
// If we couldn't find one, try opening URL and treating as a stream
try {
@ -380,7 +386,7 @@ public class CreateProjectCommand extends Command {
importer = ImporterRegistry.guessImporter(contentType, url.getPath());
internalInvokeImporter(project, importer, options, inputStream, connection.getContentEncoding());
internalInvokeImporter(project, importer, metadata, options, inputStream, connection.getContentEncoding());
} finally {
inputStream.close();
}
@ -388,11 +394,12 @@ public class CreateProjectCommand extends Command {
}
protected void internalInvokeImporter(
Project project,
Importer importer,
Properties options,
InputStream rawInputStream,
String encoding
Project project,
Importer importer,
ProjectMetadata metadata,
Properties options,
InputStream rawInputStream,
String encoding
) throws Exception {
if (importer instanceof ReaderImporter) {
@ -433,19 +440,20 @@ public class CreateProjectCommand extends Command {
new InputStreamReader(inputStream);
}
((ReaderImporter) importer).read(reader, project, options);
((ReaderImporter) importer).read(reader, project, metadata, options);
} else {
((StreamImporter) importer).read(rawInputStream, project, options);
((StreamImporter) importer).read(rawInputStream, project, metadata, options);
}
}
protected void internalInvokeImporter(
Project project,
ReaderImporter importer,
Properties options,
Reader reader
Project project,
ReaderImporter importer,
ProjectMetadata metadata,
Properties options,
Reader reader
) throws Exception {
importer.read(reader, project, options);
importer.read(reader, project, metadata, options);
}
}

3
main/src/com/google/gridworks/importers/ExcelImporter.java
View File

@ -19,6 +19,7 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Cell;
import com.google.gridworks.model.Column;
import com.google.gridworks.model.Project;
@ -31,7 +32,7 @@ public class ExcelImporter implements StreamImporter {
protected boolean _xmlBased;
@Override
public void read(InputStream inputStream, Project project, Properties options) throws ImportException {
public void read(InputStream inputStream, Project project, ProjectMetadata metadata, Properties options) throws ImportException {
int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1);
int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1);
int limit = ImporterUtilities.getIntegerOption("limit", options, -1);

5
main/src/com/google/gridworks/importers/MarcImporter.java
View File

@ -14,6 +14,7 @@ import org.marc4j.MarcWriter;
import org.marc4j.MarcXmlWriter;
import org.marc4j.marc.Record;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project;
public class MarcImporter implements StreamImporter {
@ -22,7 +23,7 @@ public class MarcImporter implements StreamImporter {
public void read(
InputStream inputStream,
Project project,
Properties options
ProjectMetadata metadata, Properties options
) throws ImportException {
int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
int skip = ImporterUtilities.getIntegerOption("skip",options,0);
@ -68,7 +69,7 @@ public class MarcImporter implements StreamImporter {
InputStream is = new FileInputStream(tempFile);
try {
new XmlImporter().read(is, project, options);
new XmlImporter().read(is, project, metadata, options);
} finally {
try {
is.close();

3
main/src/com/google/gridworks/importers/RdfTripleImporter.java
View File

@ -24,6 +24,7 @@ import static org.jrdf.graph.AnyObjectNode.ANY_OBJECT_NODE;
import static org.jrdf.graph.AnyPredicateNode.ANY_PREDICATE_NODE;
import static org.jrdf.graph.AnySubjectNode.ANY_SUBJECT_NODE;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.expr.ExpressionUtils;
import com.google.gridworks.model.Cell;
import com.google.gridworks.model.Column;
@ -43,7 +44,7 @@ public class RdfTripleImporter implements ReaderImporter{
}
@Override
public void read(Reader reader, Project project, Properties options) throws ImportException {
public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options) throws ImportException {
String baseUrl = options.getProperty("base-url");
Graph graph = _jrdfFactory.getNewGraph();

5
main/src/com/google/gridworks/importers/ReaderImporter.java
View File

@ -3,6 +3,7 @@ package com.google.gridworks.importers;
import java.io.Reader;
import java.util.Properties;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project;
/**
@ -18,10 +19,12 @@ public interface ReaderImporter extends Importer {
* the correct point and ready to go.
* @param project
* project which will contain data
* @param metadata
* metadata of new project
* @param options
* set of properties with import options
* @throws ImportException
*/
public void read(Reader reader, Project project, Properties options)
public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options)
throws ImportException;
}

4
main/src/com/google/gridworks/importers/StreamImporter.java
View File

@ -3,6 +3,7 @@ package com.google.gridworks.importers;
import java.io.InputStream;
import java.util.Properties;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project;
public interface StreamImporter extends Importer {
@ -10,10 +11,11 @@ public interface StreamImporter extends Importer {
/**
* @param inputStream stream to be imported
* @param project project to import stream into
* @param metadata metadata of new project
* @param options
* @throws ImportException
*/
public void read(InputStream inputStream, Project project,
Properties options) throws ImportException;
ProjectMetadata metadata, Properties options) throws ImportException;
}

7
main/src/com/google/gridworks/importers/TsvCsvImporter.java
View File

@ -14,6 +14,7 @@ import org.apache.commons.lang.StringUtils;
import au.com.bytecode.opencsv.CSVParser;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.expr.ExpressionUtils;
import com.google.gridworks.model.Cell;
import com.google.gridworks.model.Project;
@ -22,7 +23,7 @@ import com.google.gridworks.model.Row;
public class TsvCsvImporter implements ReaderImporter,StreamImporter {
@Override
public void read(Reader reader, Project project, Properties options) throws ImportException {
public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options) throws ImportException {
boolean splitIntoColumns = ImporterUtilities.getBooleanOption("split-into-columns", options, true);
String sep = options.getProperty("separator"); // auto-detect if not present
@ -178,8 +179,8 @@ public class TsvCsvImporter implements ReaderImporter,StreamImporter {
@Override
public void read(InputStream inputStream, Project project,
Properties options) throws ImportException {
read(new InputStreamReader(inputStream), project, options);
ProjectMetadata metadata, Properties options) throws ImportException {
read(new InputStreamReader(inputStream), project, metadata, options);
}
@Override

3
main/src/com/google/gridworks/importers/UrlImporter.java
View File

@ -3,11 +3,12 @@ package com.google.gridworks.importers;
import java.net.URL;
import java.util.Properties;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project;
public interface UrlImporter extends Importer {
public void read(URL url, Project project, Properties options) throws Exception;
public void read(URL url, Project project, ProjectMetadata metadata, Properties options) throws Exception;
public boolean canImportData(URL url);

3
main/src/com/google/gridworks/importers/XmlImporter.java
View File

@ -9,6 +9,7 @@ import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.XmlImportUtilities.ImportColumnGroup;
import com.google.gridworks.model.Project;
@ -22,7 +23,7 @@ public class XmlImporter implements StreamImporter {
public void read(
InputStream inputStream,
Project project,
Properties options
ProjectMetadata metadata, Properties options
) throws ImportException {
logger.trace("XmlImporter.read");
PushbackInputStream pis = new PushbackInputStream(inputStream,BUFFER_SIZE);

9
main/tests/server/src/com/google/gridworks/tests/importers/RdfTripleImporterTests.java
View File

@ -9,6 +9,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.RdfTripleImporter;
import com.google.gridworks.model.Project;
import com.google.gridworks.tests.GridworksTest;
@ -41,7 +42,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf);
try {
SUT.read(reader, project, options);
SUT.read(reader, project, new ProjectMetadata(), options);
project.update();
} catch (Exception e) {
Assert.fail();
@ -64,7 +65,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf);
try {
SUT.read(reader, project, options);
SUT.read(reader, project, new ProjectMetadata(), options);
project.update();
} catch (Exception e) {
Assert.fail();
@ -106,7 +107,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf);
try {
SUT.read(reader, project, options);
SUT.read(reader, project, new ProjectMetadata(), options);
project.update();
} catch (Exception e) {
Assert.fail();
@ -141,7 +142,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf);
try {
SUT.read(reader, project, options);
SUT.read(reader, project, new ProjectMetadata(), options);
project.update();
} catch (Exception e) {
Assert.fail();

5
main/tests/server/src/com/google/gridworks/tests/importers/TsvCsvImporterTests.java
View File

@ -18,6 +18,7 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.TsvCsvImporter;
import com.google.gridworks.model.Project;
import com.google.gridworks.tests.GridworksTest;
@ -464,7 +465,7 @@ public class TsvCsvImporterTests extends GridworksTest {
whenGetIntegerOption("ignore-quotes",properties,0);
try {
SUT.read(reader, project, properties);
SUT.read(reader, project, new ProjectMetadata(), properties);
} catch (Exception e) {
Assert.fail();
}
@ -497,7 +498,7 @@ public class TsvCsvImporterTests extends GridworksTest {
whenGetBooleanOption("ignore-quotes",properties,true);
try {
SUT.read(reader, project, properties);
SUT.read(reader, project, new ProjectMetadata(), properties);
} catch (Exception e) {
Assert.fail();
}

3
main/tests/server/src/com/google/gridworks/tests/importers/XmlImporterTests.java
View File

@ -14,6 +14,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.XmlImporter;
import com.google.gridworks.model.Project;
import com.google.gridworks.model.Row;
@ -223,7 +224,7 @@ public class XmlImporterTests extends GridworksTest {
}
try {
SUT.read(inputStream, project, options);
SUT.read(inputStream, project, new ProjectMetadata(), options);
} catch (Exception e) {
Assert.fail();
}