From f023b922e1aec65d0a632435f155ac9f1e3f6249 Mon Sep 17 00:00:00 2001 From: David Huynh Date: Mon, 22 Aug 2011 17:55:06 +0000 Subject: [PATCH] Implemented encoding selectors in a few importing parser UIs. git-svn-id: http://google-refine.googlecode.com/svn/trunk@2214 7d457c2a-affb-35e4-300a-418c747d4874 --- .../refine/importers/ImportingParserBase.java | 9 ++- .../tree/TreeImportingParserBase.java | 7 ++- .../refine/importing/ImportingUtilities.java | 13 +++-- .../webapp/modules/core/MOD-INF/controller.js | 23 +++++++- main/webapp/modules/core/index.vt | 1 + .../core/scripts/index/create-project-ui.js | 58 +++++++++++++++++++ .../fixed-width-parser-ui.html | 2 +- .../fixed-width-parser-ui.js | 13 ++++- .../line-based-parser-ui.html | 2 +- .../parser-interfaces/line-based-parser-ui.js | 11 +++- .../separator-based-parser-ui.html | 2 +- .../separator-based-parser-ui.js | 14 ++++- .../scripts/index/select-encoding-dialog.html | 22 +++++++ .../core/styles/index/create-project-ui.less | 12 ++++ 14 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 main/webapp/modules/core/scripts/index/select-encoding-dialog.html diff --git a/main/src/com/google/refine/importers/ImportingParserBase.java b/main/src/com/google/refine/importers/ImportingParserBase.java index ccd6e74da..6ec385faa 100644 --- a/main/src/com/google/refine/importers/ImportingParserBase.java +++ b/main/src/com/google/refine/importers/ImportingParserBase.java @@ -48,6 +48,7 @@ import com.google.refine.importing.ImportingJob; import com.google.refine.importing.ImportingParser; import com.google.refine.importing.ImportingUtilities; import com.google.refine.model.Project; +import com.google.refine.util.JSONUtilities; abstract public class ImportingParserBase implements ImportingParser { final protected boolean useInputStream; @@ -98,7 +99,13 @@ abstract public class ImportingParserBase implements ImportingParser { if (useInputStream) { parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions); } else { - Reader reader = ImportingUtilities.getReaderFromStream(inputStream, fileRecord); + String commonEncoding = JSONUtilities.getString(options, "encoding", null); + if (commonEncoding != null && commonEncoding.isEmpty()) { + commonEncoding = null; + } + + Reader reader = ImportingUtilities.getReaderFromStream( + inputStream, fileRecord, commonEncoding); parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions); } diff --git a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java index 3bbdb0f1b..2a8067a7c 100644 --- a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java +++ b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java @@ -111,7 +111,12 @@ abstract public class TreeImportingParserBase implements ImportingParser { parseOneFile(project, metadata, job, fileSource, inputStream, rootColumnGroup, limit, options, exceptions); } else { - Reader reader = ImportingUtilities.getFileReader(file, fileRecord); + String commonEncoding = JSONUtilities.getString(options, "encoding", null); + if (commonEncoding != null && commonEncoding.isEmpty()) { + commonEncoding = null; + } + + Reader reader = ImportingUtilities.getFileReader(file, fileRecord, commonEncoding); parseOneFile(project, metadata, job, fileSource, reader, rootColumnGroup, limit, options, exceptions); } diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java index 3435882c0..21e0c01e4 100644 --- a/main/src/com/google/refine/importing/ImportingUtilities.java +++ b/main/src/com/google/refine/importing/ImportingUtilities.java @@ -373,18 +373,21 @@ public class ImportingUtilities { return file; } - static public Reader getFileReader(ImportingJob job, JSONObject fileRecord) + static public Reader getFileReader(ImportingJob job, JSONObject fileRecord, String commonEncoding) throws FileNotFoundException { - return getFileReader(getFile(job, JSONUtilities.getString(fileRecord, "location", "")), fileRecord); + return getFileReader(getFile(job, JSONUtilities.getString(fileRecord, "location", "")), fileRecord, commonEncoding); } - static public Reader getFileReader(File file, JSONObject fileRecord) throws FileNotFoundException { - return getReaderFromStream(new FileInputStream(file), fileRecord); + static public Reader getFileReader(File file, JSONObject fileRecord, String commonEncoding) throws FileNotFoundException { + return getReaderFromStream(new FileInputStream(file), fileRecord, commonEncoding); } - static public Reader getReaderFromStream(InputStream inputStream, JSONObject fileRecord) { + static public Reader getReaderFromStream(InputStream inputStream, JSONObject fileRecord, String commonEncoding) { String encoding = getEncoding(fileRecord); + if (encoding == null) { + encoding = commonEncoding; + } if (encoding != null) { try { return new InputStreamReader(inputStream, encoding); diff --git a/main/webapp/modules/core/MOD-INF/controller.js b/main/webapp/modules/core/MOD-INF/controller.js index 551ba4cb8..a2fa1beff 100644 --- a/main/webapp/modules/core/MOD-INF/controller.js +++ b/main/webapp/modules/core/MOD-INF/controller.js @@ -557,7 +557,28 @@ function process(path, request, response) { } context.scriptInjection = scriptInjection.join("\n"); } - + + if (path == "/index") { + var encodings = []; + + var sortedCharsetMap = Packages.java.nio.charset.Charset.availableCharsets(); + for each (var code in sortedCharsetMap.keySet().toArray()) { + var charset = sortedCharsetMap.get(code); + var aliases = []; + for each (var alias in charset.aliases().toArray()) { + aliases.push(alias); + } + + encodings.push({ + code: code, + name: charset.displayName(), + aliases: aliases + }); + } + + context.encodingJson = butterfly.toJSONString(encodings); + } + send(request, response, path + ".vt", context); } } diff --git a/main/webapp/modules/core/index.vt b/main/webapp/modules/core/index.vt index 5ad498c85..6427766b9 100644 --- a/main/webapp/modules/core/index.vt +++ b/main/webapp/modules/core/index.vt @@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. $scriptInjection $styleInjection +