From 2609c4049db889d9c61db81ab602eb341acd6a94 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Fri, 17 Sep 2010 01:00:23 +0000 Subject: [PATCH] Fixed issue 114: "Refactor project manager api to allow importers to create project metadata" by incorporating tfmorris' patch. git-svn-id: http://google-refine.googlecode.com/svn/trunk@1271 7d457c2a-affb-35e4-300a-418c747d4874 --- .../project/CreateProjectCommand.java | 74 ++++++++++--------- .../gridworks/importers/ExcelImporter.java | 3 +- .../gridworks/importers/MarcImporter.java | 5 +- .../importers/RdfTripleImporter.java | 3 +- .../gridworks/importers/ReaderImporter.java | 5 +- .../gridworks/importers/StreamImporter.java | 4 +- .../gridworks/importers/TsvCsvImporter.java | 7 +- .../gridworks/importers/UrlImporter.java | 3 +- .../gridworks/importers/XmlImporter.java | 3 +- .../importers/RdfTripleImporterTests.java | 9 ++- .../tests/importers/TsvCsvImporterTests.java | 5 +- .../tests/importers/XmlImporterTests.java | 3 +- 12 files changed, 73 insertions(+), 51 deletions(-) diff --git a/main/src/com/google/gridworks/commands/project/CreateProjectCommand.java b/main/src/com/google/gridworks/commands/project/CreateProjectCommand.java index 07cf88545..4ce7b9a75 100644 --- a/main/src/com/google/gridworks/commands/project/CreateProjectCommand.java +++ b/main/src/com/google/gridworks/commands/project/CreateProjectCommand.java @@ -74,15 +74,15 @@ public class CreateProjectCommand extends Command { Properties options = ParsingUtilities.parseUrlParameters(request); Project project = new Project(); + ProjectMetadata pm = new ProjectMetadata(); - internalImport(request, project, options); + internalImport(request, project, pm, options); /* * The import process above populates options with parameters * in the POST body. That's why we're constructing the project * metadata object after calling internalImport(). */ - ProjectMetadata pm = new ProjectMetadata(); pm.setName(options.getProperty("project-name")); pm.setPassword(options.getProperty("project-password")); pm.setEncoding(options.getProperty("encoding")); @@ -105,6 +105,7 @@ public class CreateProjectCommand extends Command { protected void internalImport( HttpServletRequest request, Project project, + ProjectMetadata metadata, Properties options ) throws Exception { @@ -121,7 +122,7 @@ public class CreateProjectCommand extends Command { if (name.equals("raw-text")) { Reader reader = new InputStreamReader(stream,"UTF-8"); try { - internalInvokeImporter(project, new TsvCsvImporter(), options, reader); + internalInvokeImporter(project, new TsvCsvImporter(), metadata, options, reader); imported = true; } finally { reader.close(); @@ -135,7 +136,7 @@ public class CreateProjectCommand extends Command { String fileName = item.getName().toLowerCase(); if (fileName.length() > 0) { try { - internalImportFile(project, options, fileName, stream); + internalImportFile(project, metadata, options, fileName, stream); imported = true; } finally { stream.close(); @@ -145,7 +146,7 @@ public class CreateProjectCommand extends Command { } if (!imported && url != null && url.length() > 0) { - internalImportURL(request, project, options, url); + internalImportURL(request, project, metadata, options, url); } } @@ -169,10 +170,11 @@ public class CreateProjectCommand extends Command { } protected void internalImportFile( - Project project, - Properties options, - String fileName, - InputStream inputStream + Project project, + ProjectMetadata metadata, + Properties options, + String fileName, + InputStream inputStream ) throws Exception { logger.info("Importing '{}'", fileName); @@ -264,7 +266,7 @@ public class CreateProjectCommand extends Command { String name = te.getName(); String ext = getExtension(name)[1]; if (exts.contains(ext)) { - internalImportFile(project, options, name, sis); + internalImportFile(project, metadata, options, name, sis); } } } @@ -276,7 +278,7 @@ public class CreateProjectCommand extends Command { String name = ze.getName(); String ext = getExtension(name)[1]; if (exts.contains(ext)) { - internalImportFile(project, options, name, sis); + internalImportFile(project, metadata, options, name, sis); } } } @@ -288,11 +290,11 @@ public class CreateProjectCommand extends Command { } } else if (fileName.endsWith(".gz")) { - internalImportFile(project, options, getExtension(fileName)[0], new GZIPInputStream(inputStream)); + internalImportFile(project, metadata, options, getExtension(fileName)[0], new GZIPInputStream(inputStream)); } else if (fileName.endsWith(".bz2")) { - internalImportFile(project, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream)); + internalImportFile(project, metadata, options, getExtension(fileName)[0], new CBZip2InputStream(inputStream)); } else { - load(project, options, fileName, inputStream); + load(project, metadata, options, fileName, inputStream); } } @@ -304,9 +306,9 @@ public class CreateProjectCommand extends Command { } } - private void load(Project project, Properties options, String fileName, InputStream inputStream) throws Exception { + private void load(Project project, ProjectMetadata metadata, Properties options, String fileName, InputStream inputStream) throws Exception { Importer importer = ImporterRegistry.guessImporter(null, fileName); - internalInvokeImporter(project, importer, options, inputStream, null); + internalInvokeImporter(project, importer, metadata, options, inputStream, null); } private File save(InputStream is) throws IOException { @@ -344,16 +346,20 @@ public class CreateProjectCommand extends Command { return result; } - protected void internalImportURL(HttpServletRequest request, - Project project, Properties options, String urlString) - throws Exception { + protected void internalImportURL( + HttpServletRequest request, + Project project, + ProjectMetadata metadata, + Properties options, + String urlString) throws Exception { + URL url = new URL(urlString); URLConnection connection = null; // Try for a URL importer first Importer importer = ImporterRegistry.guessUrlImporter(url); if (importer instanceof UrlImporter) { - ((UrlImporter) importer).read(url, project, options); + ((UrlImporter) importer).read(url, project, metadata, options); } else { // If we couldn't find one, try opening URL and treating as a stream try { @@ -380,7 +386,7 @@ public class CreateProjectCommand extends Command { importer = ImporterRegistry.guessImporter(contentType, url.getPath()); - internalInvokeImporter(project, importer, options, inputStream, connection.getContentEncoding()); + internalInvokeImporter(project, importer, metadata, options, inputStream, connection.getContentEncoding()); } finally { inputStream.close(); } @@ -388,11 +394,12 @@ public class CreateProjectCommand extends Command { } protected void internalInvokeImporter( - Project project, - Importer importer, - Properties options, - InputStream rawInputStream, - String encoding + Project project, + Importer importer, + ProjectMetadata metadata, + Properties options, + InputStream rawInputStream, + String encoding ) throws Exception { if (importer instanceof ReaderImporter) { @@ -433,19 +440,20 @@ public class CreateProjectCommand extends Command { new InputStreamReader(inputStream); } - ((ReaderImporter) importer).read(reader, project, options); + ((ReaderImporter) importer).read(reader, project, metadata, options); } else { - ((StreamImporter) importer).read(rawInputStream, project, options); + ((StreamImporter) importer).read(rawInputStream, project, metadata, options); } } protected void internalInvokeImporter( - Project project, - ReaderImporter importer, - Properties options, - Reader reader + Project project, + ReaderImporter importer, + ProjectMetadata metadata, + Properties options, + Reader reader ) throws Exception { - importer.read(reader, project, options); + importer.read(reader, project, metadata, options); } } diff --git a/main/src/com/google/gridworks/importers/ExcelImporter.java b/main/src/com/google/gridworks/importers/ExcelImporter.java index b4b1605ee..ecd07c526 100644 --- a/main/src/com/google/gridworks/importers/ExcelImporter.java +++ b/main/src/com/google/gridworks/importers/ExcelImporter.java @@ -19,6 +19,7 @@ import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.model.Cell; import com.google.gridworks.model.Column; import com.google.gridworks.model.Project; @@ -31,7 +32,7 @@ public class ExcelImporter implements StreamImporter { protected boolean _xmlBased; @Override - public void read(InputStream inputStream, Project project, Properties options) throws ImportException { + public void read(InputStream inputStream, Project project, ProjectMetadata metadata, Properties options) throws ImportException { int ignoreLines = ImporterUtilities.getIntegerOption("ignore", options, -1); int headerLines = ImporterUtilities.getIntegerOption("header-lines", options, 1); int limit = ImporterUtilities.getIntegerOption("limit", options, -1); diff --git a/main/src/com/google/gridworks/importers/MarcImporter.java b/main/src/com/google/gridworks/importers/MarcImporter.java index a3cc86898..4abe94c5c 100644 --- a/main/src/com/google/gridworks/importers/MarcImporter.java +++ b/main/src/com/google/gridworks/importers/MarcImporter.java @@ -14,6 +14,7 @@ import org.marc4j.MarcWriter; import org.marc4j.MarcXmlWriter; import org.marc4j.marc.Record; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.model.Project; public class MarcImporter implements StreamImporter { @@ -22,7 +23,7 @@ public class MarcImporter implements StreamImporter { public void read( InputStream inputStream, Project project, - Properties options + ProjectMetadata metadata, Properties options ) throws ImportException { int limit = ImporterUtilities.getIntegerOption("limit",options,-1); int skip = ImporterUtilities.getIntegerOption("skip",options,0); @@ -68,7 +69,7 @@ public class MarcImporter implements StreamImporter { InputStream is = new FileInputStream(tempFile); try { - new XmlImporter().read(is, project, options); + new XmlImporter().read(is, project, metadata, options); } finally { try { is.close(); diff --git a/main/src/com/google/gridworks/importers/RdfTripleImporter.java b/main/src/com/google/gridworks/importers/RdfTripleImporter.java index d2b08481f..a0f9219a4 100644 --- a/main/src/com/google/gridworks/importers/RdfTripleImporter.java +++ b/main/src/com/google/gridworks/importers/RdfTripleImporter.java @@ -24,6 +24,7 @@ import static org.jrdf.graph.AnyObjectNode.ANY_OBJECT_NODE; import static org.jrdf.graph.AnyPredicateNode.ANY_PREDICATE_NODE; import static org.jrdf.graph.AnySubjectNode.ANY_SUBJECT_NODE; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.expr.ExpressionUtils; import com.google.gridworks.model.Cell; import com.google.gridworks.model.Column; @@ -43,7 +44,7 @@ public class RdfTripleImporter implements ReaderImporter{ } @Override - public void read(Reader reader, Project project, Properties options) throws ImportException { + public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options) throws ImportException { String baseUrl = options.getProperty("base-url"); Graph graph = _jrdfFactory.getNewGraph(); diff --git a/main/src/com/google/gridworks/importers/ReaderImporter.java b/main/src/com/google/gridworks/importers/ReaderImporter.java index 66fe2bd20..48982c405 100644 --- a/main/src/com/google/gridworks/importers/ReaderImporter.java +++ b/main/src/com/google/gridworks/importers/ReaderImporter.java @@ -3,6 +3,7 @@ package com.google.gridworks.importers; import java.io.Reader; import java.util.Properties; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.model.Project; /** @@ -18,10 +19,12 @@ public interface ReaderImporter extends Importer { * the correct point and ready to go. * @param project * project which will contain data + * @param metadata + * metadata of new project * @param options * set of properties with import options * @throws ImportException */ - public void read(Reader reader, Project project, Properties options) + public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options) throws ImportException; } diff --git a/main/src/com/google/gridworks/importers/StreamImporter.java b/main/src/com/google/gridworks/importers/StreamImporter.java index 860faedbf..1ab0b8bbe 100644 --- a/main/src/com/google/gridworks/importers/StreamImporter.java +++ b/main/src/com/google/gridworks/importers/StreamImporter.java @@ -3,6 +3,7 @@ package com.google.gridworks.importers; import java.io.InputStream; import java.util.Properties; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.model.Project; public interface StreamImporter extends Importer { @@ -10,10 +11,11 @@ public interface StreamImporter extends Importer { /** * @param inputStream stream to be imported * @param project project to import stream into + * @param metadata metadata of new project * @param options * @throws ImportException */ public void read(InputStream inputStream, Project project, - Properties options) throws ImportException; + ProjectMetadata metadata, Properties options) throws ImportException; } diff --git a/main/src/com/google/gridworks/importers/TsvCsvImporter.java b/main/src/com/google/gridworks/importers/TsvCsvImporter.java index ad678cbd1..cc75a1b57 100644 --- a/main/src/com/google/gridworks/importers/TsvCsvImporter.java +++ b/main/src/com/google/gridworks/importers/TsvCsvImporter.java @@ -14,6 +14,7 @@ import org.apache.commons.lang.StringUtils; import au.com.bytecode.opencsv.CSVParser; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.expr.ExpressionUtils; import com.google.gridworks.model.Cell; import com.google.gridworks.model.Project; @@ -22,7 +23,7 @@ import com.google.gridworks.model.Row; public class TsvCsvImporter implements ReaderImporter,StreamImporter { @Override - public void read(Reader reader, Project project, Properties options) throws ImportException { + public void read(Reader reader, Project project, ProjectMetadata metadata, Properties options) throws ImportException { boolean splitIntoColumns = ImporterUtilities.getBooleanOption("split-into-columns", options, true); String sep = options.getProperty("separator"); // auto-detect if not present @@ -178,8 +179,8 @@ public class TsvCsvImporter implements ReaderImporter,StreamImporter { @Override public void read(InputStream inputStream, Project project, - Properties options) throws ImportException { - read(new InputStreamReader(inputStream), project, options); + ProjectMetadata metadata, Properties options) throws ImportException { + read(new InputStreamReader(inputStream), project, metadata, options); } @Override diff --git a/main/src/com/google/gridworks/importers/UrlImporter.java b/main/src/com/google/gridworks/importers/UrlImporter.java index 873be5f81..5013b2ba6 100644 --- a/main/src/com/google/gridworks/importers/UrlImporter.java +++ b/main/src/com/google/gridworks/importers/UrlImporter.java @@ -3,11 +3,12 @@ package com.google.gridworks.importers; import java.net.URL; import java.util.Properties; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.model.Project; public interface UrlImporter extends Importer { - public void read(URL url, Project project, Properties options) throws Exception; + public void read(URL url, Project project, ProjectMetadata metadata, Properties options) throws Exception; public boolean canImportData(URL url); diff --git a/main/src/com/google/gridworks/importers/XmlImporter.java b/main/src/com/google/gridworks/importers/XmlImporter.java index bb1f17bd5..3aa7c4893 100644 --- a/main/src/com/google/gridworks/importers/XmlImporter.java +++ b/main/src/com/google/gridworks/importers/XmlImporter.java @@ -9,6 +9,7 @@ import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.importers.XmlImportUtilities.ImportColumnGroup; import com.google.gridworks.model.Project; @@ -22,7 +23,7 @@ public class XmlImporter implements StreamImporter { public void read( InputStream inputStream, Project project, - Properties options + ProjectMetadata metadata, Properties options ) throws ImportException { logger.trace("XmlImporter.read"); PushbackInputStream pis = new PushbackInputStream(inputStream,BUFFER_SIZE); diff --git a/main/tests/server/src/com/google/gridworks/tests/importers/RdfTripleImporterTests.java b/main/tests/server/src/com/google/gridworks/tests/importers/RdfTripleImporterTests.java index 726370c7e..70306f64f 100644 --- a/main/tests/server/src/com/google/gridworks/tests/importers/RdfTripleImporterTests.java +++ b/main/tests/server/src/com/google/gridworks/tests/importers/RdfTripleImporterTests.java @@ -9,6 +9,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.importers.RdfTripleImporter; import com.google.gridworks.model.Project; import com.google.gridworks.tests.GridworksTest; @@ -41,7 +42,7 @@ public class RdfTripleImporterTests extends GridworksTest { StringReader reader = new StringReader(sampleRdf); try { - SUT.read(reader, project, options); + SUT.read(reader, project, new ProjectMetadata(), options); project.update(); } catch (Exception e) { Assert.fail(); @@ -64,7 +65,7 @@ public class RdfTripleImporterTests extends GridworksTest { StringReader reader = new StringReader(sampleRdf); try { - SUT.read(reader, project, options); + SUT.read(reader, project, new ProjectMetadata(), options); project.update(); } catch (Exception e) { Assert.fail(); @@ -106,7 +107,7 @@ public class RdfTripleImporterTests extends GridworksTest { StringReader reader = new StringReader(sampleRdf); try { - SUT.read(reader, project, options); + SUT.read(reader, project, new ProjectMetadata(), options); project.update(); } catch (Exception e) { Assert.fail(); @@ -141,7 +142,7 @@ public class RdfTripleImporterTests extends GridworksTest { StringReader reader = new StringReader(sampleRdf); try { - SUT.read(reader, project, options); + SUT.read(reader, project, new ProjectMetadata(), options); project.update(); } catch (Exception e) { Assert.fail(); diff --git a/main/tests/server/src/com/google/gridworks/tests/importers/TsvCsvImporterTests.java b/main/tests/server/src/com/google/gridworks/tests/importers/TsvCsvImporterTests.java index b903d86d1..7df7b5afa 100644 --- a/main/tests/server/src/com/google/gridworks/tests/importers/TsvCsvImporterTests.java +++ b/main/tests/server/src/com/google/gridworks/tests/importers/TsvCsvImporterTests.java @@ -18,6 +18,7 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.importers.TsvCsvImporter; import com.google.gridworks.model.Project; import com.google.gridworks.tests.GridworksTest; @@ -464,7 +465,7 @@ public class TsvCsvImporterTests extends GridworksTest { whenGetIntegerOption("ignore-quotes",properties,0); try { - SUT.read(reader, project, properties); + SUT.read(reader, project, new ProjectMetadata(), properties); } catch (Exception e) { Assert.fail(); } @@ -497,7 +498,7 @@ public class TsvCsvImporterTests extends GridworksTest { whenGetBooleanOption("ignore-quotes",properties,true); try { - SUT.read(reader, project, properties); + SUT.read(reader, project, new ProjectMetadata(), properties); } catch (Exception e) { Assert.fail(); } diff --git a/main/tests/server/src/com/google/gridworks/tests/importers/XmlImporterTests.java b/main/tests/server/src/com/google/gridworks/tests/importers/XmlImporterTests.java index 21d1b5597..d3e6a4f0d 100644 --- a/main/tests/server/src/com/google/gridworks/tests/importers/XmlImporterTests.java +++ b/main/tests/server/src/com/google/gridworks/tests/importers/XmlImporterTests.java @@ -14,6 +14,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.google.gridworks.ProjectMetadata; import com.google.gridworks.importers.XmlImporter; import com.google.gridworks.model.Project; import com.google.gridworks.model.Row; @@ -223,7 +224,7 @@ public class XmlImporterTests extends GridworksTest { } try { - SUT.read(inputStream, project, options); + SUT.read(inputStream, project, new ProjectMetadata(), options); } catch (Exception e) { Assert.fail(); }