diff --git a/.classpath b/.classpath index a47daf4bc..8448958a5 100644 --- a/.classpath +++ b/.classpath @@ -17,7 +17,6 @@ - @@ -77,7 +76,6 @@ - @@ -85,6 +83,9 @@ - - + + + + + diff --git a/.gitignore b/.gitignore deleted file mode 100644 index d2340d41e..000000000 --- a/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -*~ -\#*# -.*.swp -*.DS_Store -*.class -.com.apple.timemachine.supported -.import-temp/ -build/ -dist/ -server/classes/ -main/webapp/WEB-INF/classes/ -main/tests/server/classes/ -main/test-output/ -appengine/classes/ -tools/ -extensions/sample-extension/module/MOD-INF/classes/ -extensions/jython/module/MOD-INF/classes/ -extensions/jython/module/MOD-INF/lib/cachedir/ -extensions/rdf-exporter/module/MOD-INF/classes/ -broker/appengine/module/MOD-INF/classes/ -broker/core/module/MOD-INF/classes/ -broker/core/WEB-INF/lib/ -broker/core/data/ -broker/core/test-output/ -tmp/ -/test-output -/bin -open-refine.log diff --git a/extensions/pc-axis/src/com/google/refine/pcaxis/PCAxisImporter.java b/extensions/pc-axis/src/com/google/refine/pcaxis/PCAxisImporter.java index 984569e1b..64e78a67d 100644 --- a/extensions/pc-axis/src/com/google/refine/pcaxis/PCAxisImporter.java +++ b/extensions/pc-axis/src/com/google/refine/pcaxis/PCAxisImporter.java @@ -88,5 +88,7 @@ public class PCAxisImporter extends TabularImportingParserBase { TabularImportingParserBase.readTable( project, metadata, job, dataReader, fileSource, limit, options, exceptions); + + super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions); } } diff --git a/main/src/com/google/refine/ProjectMetadata.java b/main/src/com/google/refine/ProjectMetadata.java index cd0f3f476..5aa574d5b 100644 --- a/main/src/com/google/refine/ProjectMetadata.java +++ b/main/src/com/google/refine/ProjectMetadata.java @@ -40,6 +40,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Properties; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONWriter; @@ -61,6 +62,10 @@ public class ProjectMetadata implements Jsonizable { private String _encoding; private int _encodingConfidence; + private String _description; + // import options is an array for 1-n data sources + private JSONArray _importOptionMetaData = new JSONArray(); + private Map _customMetadata = new HashMap(); private PreferenceStore _preferenceStore = new PreferenceStore(); @@ -99,6 +104,12 @@ public class ProjectMetadata implements Jsonizable { } writer.endObject(); + // write JSONArray to file directly + if (_importOptionMetaData.length() > 0 ) { + writer.key("importOptionMetaData"); + writer.value(_importOptionMetaData); + } + if ("save".equals(options.getProperty("mode"))) { writer.key("password"); writer.value(_password); @@ -182,6 +193,16 @@ public class ProjectMetadata implements Jsonizable { // ignore } } + + if (obj.has("importOptionMetaData") && !obj.isNull("importOptionMetaData")) { + try { + JSONArray jsonArray = obj.getJSONArray("importOptionMetaData"); + pm._importOptionMetaData = jsonArray; + } catch (JSONException e) { + // ignore + } + } + pm.written = new Date(); // Mark it as not needing writing until modified @@ -263,4 +284,19 @@ public class ProjectMetadata implements Jsonizable { } updateModified(); } + + public JSONArray getImportOptionMetaData() { + return _importOptionMetaData; + } + + public void setImportOptionMetaData(JSONArray jsonArray) { + _importOptionMetaData = jsonArray; + updateModified(); + } + + public void appendImportOptionMetaData(JSONObject obj) { + _importOptionMetaData.put(obj); + updateModified(); + } + } diff --git a/main/src/com/google/refine/importers/ExcelImporter.java b/main/src/com/google/refine/importers/ExcelImporter.java index 4de1b3369..5838d7a87 100644 --- a/main/src/com/google/refine/importers/ExcelImporter.java +++ b/main/src/com/google/refine/importers/ExcelImporter.java @@ -225,6 +225,8 @@ public class ExcelImporter extends TabularImportingParserBase { exceptions ); } + + super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions); } static protected Serializable extractCell(org.apache.poi.ss.usermodel.Cell cell) { diff --git a/main/src/com/google/refine/importers/FixedWidthImporter.java b/main/src/com/google/refine/importers/FixedWidthImporter.java index fa3cd6ce2..89ca6ad0a 100644 --- a/main/src/com/google/refine/importers/FixedWidthImporter.java +++ b/main/src/com/google/refine/importers/FixedWidthImporter.java @@ -107,6 +107,8 @@ public class FixedWidthImporter extends TabularImportingParserBase { }; TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions); + + super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions); } /** diff --git a/main/src/com/google/refine/importers/ImportingParserBase.java b/main/src/com/google/refine/importers/ImportingParserBase.java index b36c452d2..b28bab03c 100644 --- a/main/src/com/google/refine/importers/ImportingParserBase.java +++ b/main/src/com/google/refine/importers/ImportingParserBase.java @@ -40,6 +40,7 @@ import java.io.Reader; import java.util.List; import org.apache.commons.lang.NotImplementedException; +import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +50,7 @@ import com.google.refine.importers.ImporterUtilities.MultiFileReadingProgress; import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingParser; import com.google.refine.importing.ImportingUtilities; +import com.google.refine.io.ProjectMetadataUtilities; import com.google.refine.model.Column; import com.google.refine.model.ModelException; import com.google.refine.model.Project; @@ -145,7 +147,18 @@ abstract public class ImportingParserBase implements ImportingParser { JSONObject options, List exceptions ) { - throw new NotImplementedException(); +// throw new NotImplementedException(); + pushImportingOptions(metadata, fileSource, options); + } + + private void pushImportingOptions(ProjectMetadata metadata, String fileSource, JSONObject options) { + try { + options.put("fileSource", fileSource); + } catch (JSONException e) { + // ignore + } + // set the import options to metadata: + metadata.appendImportOptionMetaData(options); } public void parseOneFile( @@ -158,7 +171,8 @@ abstract public class ImportingParserBase implements ImportingParser { JSONObject options, List exceptions ) { - throw new NotImplementedException(); +// throw new NotImplementedException(); + pushImportingOptions(metadata, fileSource, options); } diff --git a/main/src/com/google/refine/importers/JsonImporter.java b/main/src/com/google/refine/importers/JsonImporter.java index 35aae5152..35092e83d 100644 --- a/main/src/com/google/refine/importers/JsonImporter.java +++ b/main/src/com/google/refine/importers/JsonImporter.java @@ -199,6 +199,8 @@ public class JsonImporter extends TreeImportingParserBase { parseOneFile(project, metadata, job, fileSource, new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions); + + super.parseOneFile(project, metadata, job, fileSource, is, rootColumnGroup, limit, options, exceptions); } static public class JSONTreeReader implements TreeReader { diff --git a/main/src/com/google/refine/importers/LineBasedImporter.java b/main/src/com/google/refine/importers/LineBasedImporter.java index b3d0ad320..c0e561a77 100644 --- a/main/src/com/google/refine/importers/LineBasedImporter.java +++ b/main/src/com/google/refine/importers/LineBasedImporter.java @@ -104,5 +104,7 @@ public class LineBasedImporter extends TabularImportingParserBase { }; TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions); + + super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions); } } diff --git a/main/src/com/google/refine/importers/OdsImporter.java b/main/src/com/google/refine/importers/OdsImporter.java index 1fc927ffe..47f69f864 100644 --- a/main/src/com/google/refine/importers/OdsImporter.java +++ b/main/src/com/google/refine/importers/OdsImporter.java @@ -183,6 +183,8 @@ public class OdsImporter extends TabularImportingParserBase { exceptions ); } + + super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions); } static protected Serializable extractCell(OdfTableCell cell) { diff --git a/main/src/com/google/refine/importers/RdfTripleImporter.java b/main/src/com/google/refine/importers/RdfTripleImporter.java index 739476c47..bd73402bb 100644 --- a/main/src/com/google/refine/importers/RdfTripleImporter.java +++ b/main/src/com/google/refine/importers/RdfTripleImporter.java @@ -160,5 +160,7 @@ public class RdfTripleImporter extends ImportingParserBase { } finally { triples.iterator().close(); } + + super.parseOneFile(project, metadata, job, fileSource, input, limit, options, exceptions); } } diff --git a/main/src/com/google/refine/importers/SeparatorBasedImporter.java b/main/src/com/google/refine/importers/SeparatorBasedImporter.java index 4980b0a87..ec662486e 100644 --- a/main/src/com/google/refine/importers/SeparatorBasedImporter.java +++ b/main/src/com/google/refine/importers/SeparatorBasedImporter.java @@ -121,6 +121,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase { }; TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions); + super.parseOneFile(project, metadata, job, fileSource, lnReader, limit, options, exceptions); } static protected ArrayList getCells(String line, CSVParser parser, LineNumberReader lnReader) diff --git a/main/src/com/google/refine/importers/TabularImportingParserBase.java b/main/src/com/google/refine/importers/TabularImportingParserBase.java index c00ab91cb..c7cdff217 100644 --- a/main/src/com/google/refine/importers/TabularImportingParserBase.java +++ b/main/src/com/google/refine/importers/TabularImportingParserBase.java @@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package com.google.refine.importers; import java.io.IOException; +import java.io.Reader; import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -200,4 +201,9 @@ abstract public class TabularImportingParserBase extends ImportingParserBase { exceptions.add(e); } } + + public void parseOneFile(Project project, ProjectMetadata metadata, ImportingJob job, String fileSource, + Reader dataReader, int limit, JSONObject options, List exceptions) { + super.parseOneFile(project, metadata, job, fileSource, dataReader, limit, options, exceptions); + } } diff --git a/main/src/com/google/refine/importers/WikitextImporter.java b/main/src/com/google/refine/importers/WikitextImporter.java index 1106256b3..2a5eb2706 100644 --- a/main/src/com/google/refine/importers/WikitextImporter.java +++ b/main/src/com/google/refine/importers/WikitextImporter.java @@ -731,6 +731,8 @@ public class WikitextImporter extends TabularImportingParserBase { exceptions.add(e1); e1.printStackTrace(); } + + super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions); } private StandardReconConfig getReconConfig(String url) { diff --git a/main/src/com/google/refine/importers/XmlImporter.java b/main/src/com/google/refine/importers/XmlImporter.java index bc3cb9f86..92cb866d5 100644 --- a/main/src/com/google/refine/importers/XmlImporter.java +++ b/main/src/com/google/refine/importers/XmlImporter.java @@ -201,6 +201,8 @@ public class XmlImporter extends TreeImportingParserBase { try { parseOneFile(project, metadata, job, fileSource, new XmlParser(inputStream), rootColumnGroup, limit, options, exceptions); + + super.parseOneFile(project, metadata, job, fileSource, inputStream, rootColumnGroup, limit, options, exceptions); } catch (XMLStreamException e) { exceptions.add(e); } catch (IOException e) { diff --git a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java index 1f2cfb403..9fceab5b1 100644 --- a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java +++ b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java @@ -174,7 +174,8 @@ abstract public class TreeImportingParserBase extends ImportingParserBase { JSONObject options, List exceptions ) { - throw new NotImplementedException(); + // throw new NotImplementedException(); + super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions); } /** diff --git a/main/src/com/google/refine/io/ProjectMetadataUtilities.java b/main/src/com/google/refine/io/ProjectMetadataUtilities.java index bf220e3ae..75640597d 100644 --- a/main/src/com/google/refine/io/ProjectMetadataUtilities.java +++ b/main/src/com/google/refine/io/ProjectMetadataUtilities.java @@ -154,4 +154,9 @@ public class ProjectMetadataUtilities { reader.close(); } } + + public static void merge(ProjectMetadata metadata, JSONObject options) { + // TODO Auto-generated method stub + + } }