s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed issue 114: "Refactor project manager api to allow importers to create project metadata" by incorporating tfmorris' patch.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1271 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-09-17 01:00:23 +00:00
parent 8d1f2d44b9
commit 2609c4049d
12 changed files with 73 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
main/src/com/google/gridworks/commands/project/CreateProjectCommand.java
View File

@ -74,15 +74,15 @@ public class CreateProjectCommand extends Command {
Properties options = ParsingUtilities.parseUrlParameters(request); Properties options = ParsingUtilities.parseUrlParameters(request);
Project project = new Project(); Project project = new Project();
ProjectMetadata pm = new ProjectMetadata();
internalImport(request, project, options); internalImport(request, project, pm, options);
/* /*
* The import process above populates options with parameters * The import process above populates options with parameters
* in the POST body. That's why we're constructing the project * in the POST body. That's why we're constructing the project
* metadata object after calling internalImport(). * metadata object after calling internalImport().
*/ */
ProjectMetadata pm = new ProjectMetadata();
pm.setName(options.getProperty("project-name")); pm.setName(options.getProperty("project-name"));
pm.setPassword(options.getProperty("project-password")); pm.setPassword(options.getProperty("project-password"));
pm.setEncoding(options.getProperty("encoding")); pm.setEncoding(options.getProperty("encoding"));
@ -105,6 +105,7 @@ public class CreateProjectCommand extends Command {
protected void internalImport( protected void internalImport(
HttpServletRequest request, HttpServletRequest request,
Project project, Project project,
ProjectMetadata metadata,
Properties options Properties options
) throws Exception { ) throws Exception {
@ -121,7 +122,7 @@ public class CreateProjectCommand extends Command {
if (name.equals("raw-text")) { if (name.equals("raw-text")) {
Reader reader = new InputStreamReader(stream,"UTF-8"); Reader reader = new InputStreamReader(stream,"UTF-8");
try { try {
internalInvokeImporter(project, new TsvCsvImporter(), options, reader); internalInvokeImporter(project, new TsvCsvImporter(), metadata, options, reader);
imported = true; imported = true;
} finally { } finally {
reader.close(); reader.close();
@ -135,7 +136,7 @@ public class CreateProjectCommand extends Command {
String fileName = item.getName().toLowerCase(); String fileName = item.getName().toLowerCase();
if (fileName.length() > 0) { if (fileName.length() > 0) {
try { try {
internalImportFile(project, options, fileName, stream); internalImportFile(project, metadata, options, fileName, stream);
imported = true; imported = true;
} finally { } finally {
stream.close(); stream.close();
@ -145,7 +146,7 @@ public class CreateProjectCommand extends Command {
} }
if (!imported && url != null && url.length() > 0) { if (!imported && url != null && url.length() > 0) {
internalImportURL(request, project, options, url); internalImportURL(request, project, metadata, options, url);
} }
} }
@ -169,10 +170,11 @@ public class CreateProjectCommand extends Command {
} }
protected void internalImportFile( protected void internalImportFile(
Project project, Project project,
Properties options, ProjectMetadata metadata,
String fileName, Properties options,
InputStream inputStream String fileName,
InputStream inputStream
) throws Exception { ) throws Exception {
logger.info("Importing '{}'", fileName); logger.info("Importing '{}'", fileName);
@ -264,7 +266,7 @@ public class CreateProjectCommand extends Command {
String name = te.getName(); String name = te.getName();
String ext = getExtension(name)[1]; String ext = getExtension(name)[1];
if (exts.contains(ext)) { if (exts.contains(ext)) {
internalImportFile(project, options, name, sis); internalImportFile(project, metadata, options, name, sis);
} }
} }
} }
@ -276,7 +278,7 @@ public class CreateProjectCommand extends Command {
String name = ze.getName(); String name = ze.getName();
String ext = getExtension(name)[1]; String ext = getExtension(name)[1];
if (exts.contains(ext)) { if (exts.contains(ext)) {
internalImportFile(project, options, name, sis); internalImportFile(project, metadata, options, name, sis);
} }
} }
} }
@ -288,11 +290,11 @@ public class CreateProjectCommand extends Command {
} }
} else if (fileName.endsWith(".gz")) { } else if (fileName.endsWith(".gz")) {
internalImportFile(project, options, getExtension(fileName)[0], new GZIPInputStream(inputStream)); internalImportFile(project, metadata, options, getExtension(fileName)[0], new GZIPInputStream(inputStream));
} else if (fileName.endsWith(".bz2")) { } else if (fileName.endsWith(".bz2")) {
internalImportFile(project, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream)); internalImportFile(project, metadata, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream));
} else { } else {
load(project, options, fileName, inputStream); load(project, metadata, options, fileName, inputStream);
} }
} }
@ -304,9 +306,9 @@ public class CreateProjectCommand extends Command {
} }
} }
private void load(Project project, Properties options, String fileName, InputStream inputStream) throws Exception { private void load(Project project, ProjectMetadata metadata, Properties options, String fileName, InputStream inputStream) throws Exception {
Importer importer = ImporterRegistry.guessImporter(null, fileName); Importer importer = ImporterRegistry.guessImporter(null, fileName);
internalInvokeImporter(project, importer, options, inputStream, null); internalInvokeImporter(project, importer, metadata, options, inputStream, null);
} }
private File save(InputStream is) throws IOException { private File save(InputStream is) throws IOException {
@ -344,16 +346,20 @@ public class CreateProjectCommand extends Command {
return result; return result;
} }
protected void internalImportURL(HttpServletRequest request, protected void internalImportURL(
Project project, Properties options, String urlString) HttpServletRequest request,
throws Exception { Project project,
ProjectMetadata metadata,
Properties options,
String urlString) throws Exception {
URL url = new URL(urlString); URL url = new URL(urlString);
URLConnection connection = null; URLConnection connection = null;
// Try for a URL importer first // Try for a URL importer first
Importer importer = ImporterRegistry.guessUrlImporter(url); Importer importer = ImporterRegistry.guessUrlImporter(url);
if (importer instanceof UrlImporter) { if (importer instanceof UrlImporter) {
((UrlImporter) importer).read(url, project, options); ((UrlImporter) importer).read(url, project, metadata, options);
} else { } else {
// If we couldn't find one, try opening URL and treating as a stream // If we couldn't find one, try opening URL and treating as a stream
try { try {
@ -380,7 +386,7 @@ public class CreateProjectCommand extends Command {
importer = ImporterRegistry.guessImporter(contentType, url.getPath()); importer = ImporterRegistry.guessImporter(contentType, url.getPath());
internalInvokeImporter(project, importer, options, inputStream, connection.getContentEncoding()); internalInvokeImporter(project, importer, metadata, options, inputStream, connection.getContentEncoding());
} finally { } finally {
inputStream.close(); inputStream.close();
} }
@ -388,11 +394,12 @@ public class CreateProjectCommand extends Command {
} }
protected void internalInvokeImporter( protected void internalInvokeImporter(
Project project, Project project,
Importer importer, Importer importer,
Properties options, ProjectMetadata metadata,
InputStream rawInputStream, Properties options,
String encoding InputStream rawInputStream,
String encoding
) throws Exception { ) throws Exception {
if (importer instanceof ReaderImporter) { if (importer instanceof ReaderImporter) {
@ -433,19 +440,20 @@ public class CreateProjectCommand extends Command {
new InputStreamReader(inputStream); new InputStreamReader(inputStream);
} }
((ReaderImporter) importer).read(reader, project, options); ((ReaderImporter) importer).read(reader, project, metadata, options);
} else { } else {
((StreamImporter) importer).read(rawInputStream, project, options); ((StreamImporter) importer).read(rawInputStream, project, metadata, options);
} }
} }
protected void internalInvokeImporter( protected void internalInvokeImporter(
Project project, Project project,
ReaderImporter importer, ReaderImporter importer,
Properties options, ProjectMetadata metadata,
Reader reader Properties options,
Reader reader
) throws Exception { ) throws Exception {
importer.read(reader, project, options); importer.read(reader, project, metadata, options);
} }
} }

3
main/src/com/google/gridworks/importers/ExcelImporter.java
View File

@ -19,6 +19,7 @@ import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Cell; import com.google.gridworks.model.Cell;
import com.google.gridworks.model.Column; import com.google.gridworks.model.Column;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
@ -31,7 +32,7 @@ public class ExcelImporter implements StreamImporter {
protected boolean _xmlBased; protected boolean _xmlBased;
@Override @Override
public void read(InputStream inputStream, Project project, Properties options) throws ImportException { public void read(InputStream inputStream, Project project, ProjectMetadata metadata, Properties options) throws ImportException {
int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1); int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1);
int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1); int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1);
int limit = ImporterUtilities.getIntegerOption("limit", options, -1); int limit = ImporterUtilities.getIntegerOption("limit", options, -1);

5
main/src/com/google/gridworks/importers/MarcImporter.java
View File

@ -14,6 +14,7 @@ import org.marc4j.MarcWriter;
import org.marc4j.MarcXmlWriter; import org.marc4j.MarcXmlWriter;
import org.marc4j.marc.Record; import org.marc4j.marc.Record;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
public class MarcImporter implements StreamImporter { public class MarcImporter implements StreamImporter {
@ -22,7 +23,7 @@ public class MarcImporter implements StreamImporter {
public void read( public void read(
InputStream inputStream, InputStream inputStream,
Project project, Project project,
Properties options ProjectMetadata metadata, Properties options
) throws ImportException { ) throws ImportException {
int limit = ImporterUtilities.getIntegerOption("limit",options,-1); int limit = ImporterUtilities.getIntegerOption("limit",options,-1);
int skip = ImporterUtilities.getIntegerOption("skip",options,0); int skip = ImporterUtilities.getIntegerOption("skip",options,0);
@ -68,7 +69,7 @@ public class MarcImporter implements StreamImporter {
InputStream is = new FileInputStream(tempFile); InputStream is = new FileInputStream(tempFile);
try { try {
new XmlImporter().read(is, project, options); new XmlImporter().read(is, project, metadata, options);
} finally { } finally {
try { try {
is.close(); is.close();

3
main/src/com/google/gridworks/importers/RdfTripleImporter.java
View File

@ -24,6 +24,7 @@ import static org.jrdf.graph.AnyObjectNode.ANY_OBJECT_NODE;
import static org.jrdf.graph.AnyPredicateNode.ANY_PREDICATE_NODE; import static org.jrdf.graph.AnyPredicateNode.ANY_PREDICATE_NODE;
import static org.jrdf.graph.AnySubjectNode.ANY_SUBJECT_NODE; import static org.jrdf.graph.AnySubjectNode.ANY_SUBJECT_NODE;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.expr.ExpressionUtils; import com.google.gridworks.expr.ExpressionUtils;
import com.google.gridworks.model.Cell; import com.google.gridworks.model.Cell;
import com.google.gridworks.model.Column; import com.google.gridworks.model.Column;
@ -43,7 +44,7 @@ public class RdfTripleImporter implements ReaderImporter{
} }
@Override @Override
public void read(Reader reader, Project project, Properties options) throws ImportException { public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options) throws ImportException {
String baseUrl = options.getProperty("base-url"); String baseUrl = options.getProperty("base-url");
Graph graph = _jrdfFactory.getNewGraph(); Graph graph = _jrdfFactory.getNewGraph();

5
main/src/com/google/gridworks/importers/ReaderImporter.java
View File

@ -3,6 +3,7 @@ package com.google.gridworks.importers;
import java.io.Reader; import java.io.Reader;
import java.util.Properties; import java.util.Properties;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
/** /**
@ -18,10 +19,12 @@ public interface ReaderImporter extends Importer {
* the correct point and ready to go. * the correct point and ready to go.
* @param project * @param project
* project which will contain data * project which will contain data
* @param metadata
* metadata of new project
* @param options * @param options
* set of properties with import options * set of properties with import options
* @throws ImportException * @throws ImportException
*/ */
public void read(Reader reader, Project project, Properties options) public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options)
throws ImportException; throws ImportException;
} }

4
main/src/com/google/gridworks/importers/StreamImporter.java
View File

@ -3,6 +3,7 @@ package com.google.gridworks.importers;
import java.io.InputStream; import java.io.InputStream;
import java.util.Properties; import java.util.Properties;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
public interface StreamImporter extends Importer { public interface StreamImporter extends Importer {
@ -10,10 +11,11 @@ public interface StreamImporter extends Importer {
/** /**
* @param inputStream stream to be imported * @param inputStream stream to be imported
* @param project project to import stream into * @param project project to import stream into
* @param metadata metadata of new project
* @param options * @param options
* @throws ImportException * @throws ImportException
*/ */
public void read(InputStream inputStream, Project project, public void read(InputStream inputStream, Project project,
Properties options) throws ImportException; ProjectMetadata metadata, Properties options) throws ImportException;
} }

7
main/src/com/google/gridworks/importers/TsvCsvImporter.java
View File

@ -14,6 +14,7 @@ import org.apache.commons.lang.StringUtils;
import au.com.bytecode.opencsv.CSVParser; import au.com.bytecode.opencsv.CSVParser;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.expr.ExpressionUtils; import com.google.gridworks.expr.ExpressionUtils;
import com.google.gridworks.model.Cell; import com.google.gridworks.model.Cell;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
@ -22,7 +23,7 @@ import com.google.gridworks.model.Row;
public class TsvCsvImporter implements ReaderImporter,StreamImporter { public class TsvCsvImporter implements ReaderImporter,StreamImporter {
@Override @Override
public void read(Reader reader, Project project, Properties options) throws ImportException { public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options) throws ImportException {
boolean splitIntoColumns = ImporterUtilities.getBooleanOption("split-into-columns", options, true); boolean splitIntoColumns = ImporterUtilities.getBooleanOption("split-into-columns", options, true);
String sep = options.getProperty("separator"); // auto-detect if not present String sep = options.getProperty("separator"); // auto-detect if not present
@ -178,8 +179,8 @@ public class TsvCsvImporter implements ReaderImporter,StreamImporter {
@Override @Override
public void read(InputStream inputStream, Project project, public void read(InputStream inputStream, Project project,
Properties options) throws ImportException { ProjectMetadata metadata, Properties options) throws ImportException {
read(new InputStreamReader(inputStream), project, options); read(new InputStreamReader(inputStream), project, metadata, options);
} }
@Override @Override

3
main/src/com/google/gridworks/importers/UrlImporter.java
View File

@ -3,11 +3,12 @@ package com.google.gridworks.importers;
import java.net.URL; import java.net.URL;
import java.util.Properties; import java.util.Properties;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
public interface UrlImporter extends Importer { public interface UrlImporter extends Importer {
public void read(URL url, Project project, Properties options) throws Exception; public void read(URL url, Project project, ProjectMetadata metadata, Properties options) throws Exception;
public boolean canImportData(URL url); public boolean canImportData(URL url);

3
main/src/com/google/gridworks/importers/XmlImporter.java
View File

@ -9,6 +9,7 @@ import java.util.Properties;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.XmlImportUtilities.ImportColumnGroup; import com.google.gridworks.importers.XmlImportUtilities.ImportColumnGroup;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
@ -22,7 +23,7 @@ public class XmlImporter implements StreamImporter {
public void read( public void read(
InputStream inputStream, InputStream inputStream,
Project project, Project project,
Properties options ProjectMetadata metadata, Properties options
) throws ImportException { ) throws ImportException {
logger.trace("XmlImporter.read"); logger.trace("XmlImporter.read");
PushbackInputStream pis = new PushbackInputStream(inputStream,BUFFER_SIZE); PushbackInputStream pis = new PushbackInputStream(inputStream,BUFFER_SIZE);

9
main/tests/server/src/com/google/gridworks/tests/importers/RdfTripleImporterTests.java
View File

@ -9,6 +9,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.RdfTripleImporter; import com.google.gridworks.importers.RdfTripleImporter;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
import com.google.gridworks.tests.GridworksTest; import com.google.gridworks.tests.GridworksTest;
@ -41,7 +42,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, new ProjectMetadata(), options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
@ -64,7 +65,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, new ProjectMetadata(), options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
@ -106,7 +107,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, new ProjectMetadata(), options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
@ -141,7 +142,7 @@ public class RdfTripleImporterTests extends GridworksTest {
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, new ProjectMetadata(), options);
project.update(); project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();

5
main/tests/server/src/com/google/gridworks/tests/importers/TsvCsvImporterTests.java
View File

@ -18,6 +18,7 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.DataProvider; import org.testng.annotations.DataProvider;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.TsvCsvImporter; import com.google.gridworks.importers.TsvCsvImporter;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
import com.google.gridworks.tests.GridworksTest; import com.google.gridworks.tests.GridworksTest;
@ -464,7 +465,7 @@ public class TsvCsvImporterTests extends GridworksTest {
whenGetIntegerOption("ignore-quotes",properties,0); whenGetIntegerOption("ignore-quotes",properties,0);
try { try {
SUT.read(reader, project, properties); SUT.read(reader, project, new ProjectMetadata(), properties);
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -497,7 +498,7 @@ public class TsvCsvImporterTests extends GridworksTest {
whenGetBooleanOption("ignore-quotes",properties,true); whenGetBooleanOption("ignore-quotes",properties,true);
try { try {
SUT.read(reader, project, properties); SUT.read(reader, project, new ProjectMetadata(), properties);
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }

3
main/tests/server/src/com/google/gridworks/tests/importers/XmlImporterTests.java
View File

@ -14,6 +14,7 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest; import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test; import org.testng.annotations.Test;
import com.google.gridworks.ProjectMetadata;
import com.google.gridworks.importers.XmlImporter; import com.google.gridworks.importers.XmlImporter;
import com.google.gridworks.model.Project; import com.google.gridworks.model.Project;
import com.google.gridworks.model.Row; import com.google.gridworks.model.Row;
@ -223,7 +224,7 @@ public class XmlImporterTests extends GridworksTest {
} }
try { try {
SUT.read(inputStream, project, options); SUT.read(inputStream, project, new ProjectMetadata(), options);
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }