s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add the import options to metadata

Browse Source
This commit is contained in:
Jacky 2017-10-21 23:33:19 -04:00
parent 473b1b135d
commit 818e139b43
17 changed files with 89 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.classpath
View File

@ -17,7 +17,6 @@
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-io-1.4.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/commons-lang-2.5.jar" sourcepath="main/webapp/WEB-INF/lib-src/commons-lang-2.5-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/dom4j-1.6.1.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-asl-1.9.12.jar" sourcepath="main/webapp/WEB-INF/lib-src/jackson-src-1.9.9.zip"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jcl-over-slf4j-1.5.6.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/jrdf-0.5.6.jar" sourcepath="main/webapp/WEB-INF/lib-src/jrdf-0.5.6-sources.jar"/>
<classpathentry exported="true" kind="lib" path="main/webapp/WEB-INF/lib/json-20100208.jar" sourcepath="main/webapp/WEB-INF/lib-src/json-20100208-sources.jar"/>
@ -77,7 +76,6 @@
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-http-client-jackson2-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/google-oauth-client-servlet-1.20.0.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/jackson-core-2.1.3.jar"/>
<classpathentry kind="lib" path="extensions/gdata/module/MOD-INF/lib/transaction-api-1.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-3.13-20150929.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/poi-ooxml-3.13-20150929.jar"/>
@ -85,6 +83,9 @@
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="extensions/jython/module/MOD-INF/lib/jython-standalone-2.7.1.jar"/>
<classpathentry kind="lib" path="main/tests/data"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/swc-parser-lazy-3.1.5-jar-with-dependencies.jar" sourcepath="main/webapp/WEB-INF/lib-src/swc-parser-lazy-3.1.5-sources.jar" />
<classpathentry kind="output" path="build"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/swc-parser-lazy-3.1.5-jar-with-dependencies.jar" sourcepath="main/webapp/WEB-INF/lib-src/swc-parser-lazy-3.1.5-sources.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-annotations-2.9.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-core-2.9.1.jar"/>
<classpathentry kind="lib" path="main/webapp/WEB-INF/lib/jackson-databind-2.9.1.jar"/>
<classpathentry kind="output" path="main/webapp/WEB-INF/classes"/>
</classpath>

28
.gitignore vendored
View File

@ -1,28 +0,0 @@
*~
\#*#
.*.swp
*.DS_Store
*.class
.com.apple.timemachine.supported
.import-temp/
build/
dist/
server/classes/
main/webapp/WEB-INF/classes/
main/tests/server/classes/
main/test-output/
appengine/classes/
tools/
extensions/sample-extension/module/MOD-INF/classes/
extensions/jython/module/MOD-INF/classes/
extensions/jython/module/MOD-INF/lib/cachedir/
extensions/rdf-exporter/module/MOD-INF/classes/
broker/appengine/module/MOD-INF/classes/
broker/core/module/MOD-INF/classes/
broker/core/WEB-INF/lib/
broker/core/data/
broker/core/test-output/
tmp/
/test-output
/bin
open-refine.log

2
extensions/pc-axis/src/com/google/refine/pcaxis/PCAxisImporter.java
View File

@ -88,5 +88,7 @@ public class PCAxisImporter extends TabularImportingParserBase {
TabularImportingParserBase.readTable(
project, metadata, job, dataReader,
fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
}

36
main/src/com/google/refine/ProjectMetadata.java
View File

@ -40,6 +40,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
@ -61,6 +62,10 @@ public class ProjectMetadata implements Jsonizable {
private String _encoding;
private int _encodingConfidence;
private String _description;
// import options is an array for 1-n data sources
private JSONArray _importOptionMetaData = new JSONArray();
private Map<String, Serializable> _customMetadata = new HashMap<String, Serializable>();
private PreferenceStore _preferenceStore = new PreferenceStore();
@ -99,6 +104,12 @@ public class ProjectMetadata implements Jsonizable {
}
writer.endObject();
// write JSONArray to file directly
if (_importOptionMetaData.length() > 0 ) {
writer.key("importOptionMetaData");
writer.value(_importOptionMetaData);
}
if ("save".equals(options.getProperty("mode"))) {
writer.key("password"); writer.value(_password);
@ -183,6 +194,16 @@ public class ProjectMetadata implements Jsonizable {
}
}
if (obj.has("importOptionMetaData") && !obj.isNull("importOptionMetaData")) {
try {
JSONArray jsonArray = obj.getJSONArray("importOptionMetaData");
pm._importOptionMetaData = jsonArray;
} catch (JSONException e) {
// ignore
}
}
pm.written = new Date(); // Mark it as not needing writing until modified
return pm;
@ -263,4 +284,19 @@ public class ProjectMetadata implements Jsonizable {
}
updateModified();
}
public JSONArray getImportOptionMetaData() {
return _importOptionMetaData;
}
public void setImportOptionMetaData(JSONArray jsonArray) {
_importOptionMetaData = jsonArray;
updateModified();
}
public void appendImportOptionMetaData(JSONObject obj) {
_importOptionMetaData.put(obj);
updateModified();
}
}

2
main/src/com/google/refine/importers/ExcelImporter.java
View File

@ -225,6 +225,8 @@ public class ExcelImporter extends TabularImportingParserBase {
exceptions
);
}
super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
}
static protected Serializable extractCell(org.apache.poi.ss.usermodel.Cell cell) {

2
main/src/com/google/refine/importers/FixedWidthImporter.java
View File

@ -107,6 +107,8 @@ public class FixedWidthImporter extends TabularImportingParserBase {
};
TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
/**

18
main/src/com/google/refine/importers/ImportingParserBase.java
View File

@ -40,6 +40,7 @@ import java.io.Reader;
import java.util.List;
import org.apache.commons.lang.NotImplementedException;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -49,6 +50,7 @@ import com.google.refine.importers.ImporterUtilities.MultiFileReadingProgress;
import com.google.refine.importing.ImportingJob;
import com.google.refine.importing.ImportingParser;
import com.google.refine.importing.ImportingUtilities;
import com.google.refine.io.ProjectMetadataUtilities;
import com.google.refine.model.Column;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
@ -145,7 +147,18 @@ abstract public class ImportingParserBase implements ImportingParser {
JSONObject options,
List<Exception> exceptions
) {
throw new NotImplementedException();
// throw new NotImplementedException();
pushImportingOptions(metadata, fileSource, options);
}
private void pushImportingOptions(ProjectMetadata metadata, String fileSource, JSONObject options) {
try {
options.put("fileSource", fileSource);
} catch (JSONException e) {
// ignore
}
// set the import options to metadata:
metadata.appendImportOptionMetaData(options);
}
public void parseOneFile(
@ -158,7 +171,8 @@ abstract public class ImportingParserBase implements ImportingParser {
JSONObject options,
List<Exception> exceptions
) {
throw new NotImplementedException();
// throw new NotImplementedException();
pushImportingOptions(metadata, fileSource, options);
}

2
main/src/com/google/refine/importers/JsonImporter.java
View File

@ -199,6 +199,8 @@ public class JsonImporter extends TreeImportingParserBase {
parseOneFile(project, metadata, job, fileSource,
new JSONTreeReader(is), rootColumnGroup, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, is, rootColumnGroup, limit, options, exceptions);
}
static public class JSONTreeReader implements TreeReader {

2
main/src/com/google/refine/importers/LineBasedImporter.java
View File

@ -104,5 +104,7 @@ public class LineBasedImporter extends TabularImportingParserBase {
};
TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
}

2
main/src/com/google/refine/importers/OdsImporter.java
View File

@ -183,6 +183,8 @@ public class OdsImporter extends TabularImportingParserBase {
exceptions
);
}
super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
}
static protected Serializable extractCell(OdfTableCell cell) {

2
main/src/com/google/refine/importers/RdfTripleImporter.java
View File

@ -160,5 +160,7 @@ public class RdfTripleImporter extends ImportingParserBase {
} finally {
triples.iterator().close();
}
super.parseOneFile(project, metadata, job, fileSource, input, limit, options, exceptions);
}
}

1
main/src/com/google/refine/importers/SeparatorBasedImporter.java
View File

@ -121,6 +121,7 @@ public class SeparatorBasedImporter extends TabularImportingParserBase {
};
TabularImportingParserBase.readTable(project, metadata, job, dataReader, fileSource, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, lnReader, limit, options, exceptions);
}
static protected ArrayList<Object> getCells(String line, CSVParser parser, LineNumberReader lnReader)

6
main/src/com/google/refine/importers/TabularImportingParserBase.java
View File

@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.importers;
import java.io.IOException;
import java.io.Reader;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@ -200,4 +201,9 @@ abstract public class TabularImportingParserBase extends ImportingParserBase {
exceptions.add(e);
}
}
public void parseOneFile(Project project, ProjectMetadata metadata, ImportingJob job, String fileSource,
Reader dataReader, int limit, JSONObject options, List<Exception> exceptions) {
super.parseOneFile(project, metadata, job, fileSource, dataReader, limit, options, exceptions);
}
}

2
main/src/com/google/refine/importers/WikitextImporter.java
View File

@ -731,6 +731,8 @@ public class WikitextImporter extends TabularImportingParserBase {
exceptions.add(e1);
e1.printStackTrace();
}
super.parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
}
private StandardReconConfig getReconConfig(String url) {

2
main/src/com/google/refine/importers/XmlImporter.java
View File

@ -201,6 +201,8 @@ public class XmlImporter extends TreeImportingParserBase {
try {
parseOneFile(project, metadata, job, fileSource,
new XmlParser(inputStream), rootColumnGroup, limit, options, exceptions);
super.parseOneFile(project, metadata, job, fileSource, inputStream, rootColumnGroup, limit, options, exceptions);
} catch (XMLStreamException e) {
exceptions.add(e);
} catch (IOException e) {

3
main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
View File

@ -174,7 +174,8 @@ abstract public class TreeImportingParserBase extends ImportingParserBase {
JSONObject options,
List<Exception> exceptions
) {
throw new NotImplementedException();
// throw new NotImplementedException();
super.parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
}
/**

5
main/src/com/google/refine/io/ProjectMetadataUtilities.java
View File

@ -154,4 +154,9 @@ public class ProjectMetadataUtilities {
reader.close();
}
}
public static void merge(ProjectMetadata metadata, JSONObject options) {
// TODO Auto-generated method stub
}
}