add the import options to metadata

.classpath

@@ -17,7 +17,6 @@
 	<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-io-1.4.jar"/>
 	<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/>
-	<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/>
 	<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/>
 	<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="main/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
@@ -77,7 +76,6 @@
 	<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar"/>
 	<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar"/>
 	<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar"/>
-	<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jackson-core-2.1.3.jar"/>
 	<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/transaction-api-1.1.jar"/>
 	<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-3.13-20150929.jar"/>
 	<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.13-20150929.jar"/>
@@ -86,5 +84,8 @@
 	<classpathentry kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.7.1.jar"/>
 	<classpathentry kind="lib" path="main/tests/data"/>
 	<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/swc-parser-lazy-3.1.5-jar-with-dependencies.jar" sourcepath="main/webapp/WEB-INF/lib-src/swc-parser-lazy-3.1.5-sources.jar"/>
-	<classpathentry kind="output" path="build"/>
+	<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-annotations-2.9.1.jar"/>
+	<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-2.9.1.jar"/>
+	<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-databind-2.9.1.jar"/>
+	<classpathentry kind="output" path="main/webapp/WEB-INF/classes"/>
 </classpath>

.gitignore

@@ -1,28 +0,0 @@
-*~
-\#*#
-.*.swp
-*.DS_Store
-*.class
-.com.apple.timemachine.supported
-.import-temp/
-build/
-dist/
-server/classes/
-main/webapp/WEB-INF/classes/
-main/tests/server/classes/
-main/test-output/
-appengine/classes/
-tools/
-extensions/sample-extension/module/MOD-INF/classes/
-extensions/jython/module/MOD-INF/classes/
-extensions/jython/module/MOD-INF/lib/cachedir/
-extensions/rdf-exporter/module/MOD-INF/classes/
-broker/appengine/module/MOD-INF/classes/
-broker/core/module/MOD-INF/classes/
-broker/core/WEB-INF/lib/
-broker/core/data/
-broker/core/test-output/
-tmp/
-/test-output
-/bin
-open-refine.log

extensions/pc-axis/src/com/google/refine/pcaxis/PCAxisImporter.java

@@ -88,5 +88,7 @@ public class PCAxisImporter extends TabularImportingParserBase {
         TabularImportingParserBase.readTable(
             project, metadata, job, dataReader,
             fileSource, limit, options, exceptions);
+        
+        super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
     }
 }

main/src/com/google/refine/ProjectMetadata.java

@@ -40,6 +40,7 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.json.JSONWriter;
@@ -61,6 +62,10 @@ public class ProjectMetadata implements Jsonizable {
     private String         _encoding;
     private int            _encodingConfidence;
     
+    private String _description;
+    // import options is an array for 1-n data sources
+    private JSONArray _importOptionMetaData = new JSONArray();
+    
     private Map<String, Serializable>   _customMetadata = new HashMap<String, Serializable>();
     private PreferenceStore             _preferenceStore = new PreferenceStore();
 
@@ -99,6 +104,12 @@ public class ProjectMetadata implements Jsonizable {
         }
         writer.endObject();
         
+        // write JSONArray to file directly 
+        if (_importOptionMetaData.length() > 0 ) {
+            writer.key("importOptionMetaData"); 
+            writer.value(_importOptionMetaData);
+        }
+        
         if ("save".equals(options.getProperty("mode"))) {
             writer.key("password"); writer.value(_password);
 
@@ -183,6 +194,16 @@ public class ProjectMetadata implements Jsonizable {
             }
         }
         
+        if (obj.has("importOptionMetaData") && !obj.isNull("importOptionMetaData")) {
+            try {
+                JSONArray jsonArray = obj.getJSONArray("importOptionMetaData");
+                pm._importOptionMetaData = jsonArray;
+            } catch (JSONException e) {
+                // ignore
+            }
+        }
+        
+
         pm.written = new Date(); // Mark it as not needing writing until modified
         
         return pm;
@@ -263,4 +284,19 @@ public class ProjectMetadata implements Jsonizable {
         }
         updateModified();
     }
+    
+    public JSONArray getImportOptionMetaData() {
+        return _importOptionMetaData;
+    }
+    
+    public void setImportOptionMetaData(JSONArray jsonArray) {
+        _importOptionMetaData = jsonArray;
+        updateModified();
+    }
+    
+    public void appendImportOptionMetaData(JSONObject obj) {
+        _importOptionMetaData.put(obj);
+        updateModified();
+    }
+    
 }

main/src/com/google/refine/importers/ExcelImporter.java

@@ -225,6 +225,8 @@ public class ExcelImporter extends TabularImportingParserBase {
                 exceptions
             );
         }
+        
+        super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
     }
     
     static protected Serializable extractCell(org.apache.poi.ss.usermodel.Cell cell) {

main/src/com/google/refine/importers/FixedWidthImporter.java

@@ -107,6 +107,8 @@ public class FixedWidthImporter extends TabularImportingParserBase {
         };
         
         TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
+        
+        super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
     }
     
     /**

main/src/com/google/refine/importers/ImportingParserBase.java

@@ -40,6 +40,7 @@ import java.io.Reader;
 import java.util.List;
 
 import org.apache.commons.lang.NotImplementedException;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,6 +50,7 @@ import com.google.refine.importers.ImporterUtilities.MultiFileReadingProgress;
 import com.google.refine.importing.ImportingJob;
 import com.google.refine.importing.ImportingParser;
 import com.google.refine.importing.ImportingUtilities;
+import com.google.refine.io.ProjectMetadataUtilities;
 import com.google.refine.model.Column;
 import com.google.refine.model.ModelException;
 import com.google.refine.model.Project;
@@ -145,7 +147,18 @@ abstract public class ImportingParserBase implements ImportingParser {
         JSONObject options,
         List<Exception> exceptions
     ) {
-        throw new NotImplementedException();
+//        throw new NotImplementedException();
+        pushImportingOptions(metadata, fileSource, options);
+    }
+
+    private void pushImportingOptions(ProjectMetadata metadata, String fileSource, JSONObject options) {
+        try {
+            options.put("fileSource", fileSource);
+        } catch (JSONException e) {
+            // ignore
+        }
+     // set the import options to metadata:
+        metadata.appendImportOptionMetaData(options);
     }
     
     public void parseOneFile(
@@ -158,7 +171,8 @@ abstract public class ImportingParserBase implements ImportingParser {
         JSONObject options,
         List<Exception> exceptions
     ) {
-        throw new NotImplementedException();
+//        throw new NotImplementedException();
+        pushImportingOptions(metadata, fileSource, options);
     }
     
     

main/src/com/google/refine/importers/JsonImporter.java

@@ -199,6 +199,8 @@ public class JsonImporter extends TreeImportingParserBase {
         
         parseOneFile(project, metadata, job, fileSource,
             new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions);
+        
+        super.parseOneFile(project, metadata, job, fileSource, is, rootColumnGroup, limit, options, exceptions);
     }
     
     static public class JSONTreeReader implements TreeReader {

main/src/com/google/refine/importers/LineBasedImporter.java

@@ -104,5 +104,7 @@ public class LineBasedImporter extends TabularImportingParserBase {
         };
         
         TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
+        
+        super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
     }
 }

main/src/com/google/refine/importers/OdsImporter.java

@@ -183,6 +183,8 @@ public class OdsImporter extends TabularImportingParserBase {
                     exceptions
             );
         }
+        
+        super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
     }
 
     static protected Serializable extractCell(OdfTableCell cell) {

main/src/com/google/refine/importers/RdfTripleImporter.java

@@ -160,5 +160,7 @@ public class RdfTripleImporter extends ImportingParserBase {
         } finally {
             triples.iterator().close();
         }
+        
+        super.parseOneFile(project, metadata, job, fileSource, input, limit, options, exceptions);
     }
 }

main/src/com/google/refine/importers/SeparatorBasedImporter.java

@@ -121,6 +121,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
         };
         
         TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
+        super.parseOneFile(project, metadata, job, fileSource, lnReader, limit, options, exceptions);
     }
     
     static protected ArrayList<Object> getCells(String line, CSVParser parser, LineNumberReader lnReader)

main/src/com/google/refine/importers/TabularImportingParserBase.java

@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 package com.google.refine.importers;
 
 import java.io.IOException;
+import java.io.Reader;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
@@ -200,4 +201,9 @@ abstract public class TabularImportingParserBase extends ImportingParserBase {
             exceptions.add(e);
         }
     }
+
+    public void parseOneFile(Project project, ProjectMetadata metadata, ImportingJob job, String fileSource,
+            Reader dataReader, int limit, JSONObject options, List<Exception> exceptions) {
+        super.parseOneFile(project, metadata, job, fileSource, dataReader, limit, options, exceptions);
+    }
 }

main/src/com/google/refine/importers/WikitextImporter.java

@@ -731,6 +731,8 @@ public class WikitextImporter extends TabularImportingParserBase {
             exceptions.add(e1);
             e1.printStackTrace();
         }
+        
+        super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
     }
     
     private StandardReconConfig getReconConfig(String url) {

main/src/com/google/refine/importers/XmlImporter.java

@@ -201,6 +201,8 @@ public class XmlImporter extends TreeImportingParserBase {
         try {
             parseOneFile(project, metadata, job, fileSource,
                 new XmlParser(inputStream), rootColumnGroup, limit, options, exceptions);
+            
+            super.parseOneFile(project, metadata, job, fileSource, inputStream, rootColumnGroup, limit, options, exceptions);
         } catch (XMLStreamException e) {
             exceptions.add(e);
         } catch (IOException e) {

main/src/com/google/refine/importers/tree/TreeImportingParserBase.java

@@ -174,7 +174,8 @@ abstract public class TreeImportingParserBase extends ImportingParserBase {
         JSONObject options,
         List<Exception> exceptions
     ) {
-        throw new NotImplementedException();
+        // throw new NotImplementedException();
+        super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
     }
     
     /**

main/src/com/google/refine/io/ProjectMetadataUtilities.java

@@ -154,4 +154,9 @@ public class ProjectMetadataUtilities {
             reader.close();
         }
     }
+
+    public static void merge(ProjectMetadata metadata, JSONObject options) {
+        // TODO Auto-generated method stub
+        
+    }
 }