Implemented encoding selectors in a few importing parser UIs.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2214 7d457c2a-affb-35e4-300a-418c747d4874
2011-08-22 17:55:06 +00:00 · 2011-08-22 17:55:06 +00:00 · f023b922e1
commit f023b922e1
parent bde63ff417
14 changed files with 173 additions and 16 deletions
--- a/main/src/com/google/refine/importers/ImportingParserBase.java
+++ b/main/src/com/google/refine/importers/ImportingParserBase.java
@ -48,6 +48,7 @@ import com.google.refine.importing.ImportingJob;
 import com.google.refine.importing.ImportingParser;
 import com.google.refine.importing.ImportingUtilities;
 import com.google.refine.model.Project;
+import com.google.refine.util.JSONUtilities;

 abstract public class ImportingParserBase implements ImportingParser {
    final protected boolean useInputStream;
@ -98,7 +99,13 @@ abstract public class ImportingParserBase implements ImportingParser {
                if (useInputStream) {
                    parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
                } else {
-                    Reader reader = ImportingUtilities.getReaderFromStream(inputStream, fileRecord);
+                    String commonEncoding = JSONUtilities.getString(options, "encoding", null);
+                    if (commonEncoding != null && commonEncoding.isEmpty()) {
+                        commonEncoding = null;
+                    }
+                    
+                    Reader reader = ImportingUtilities.getReaderFromStream(
+                        inputStream, fileRecord, commonEncoding);
                    
                    parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
                }
--- a/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
+++ b/main/src/com/google/refine/importers/tree/TreeImportingParserBase.java
@ -111,7 +111,12 @@ abstract public class TreeImportingParserBase implements ImportingParser {
                    parseOneFile(project, metadata, job, fileSource, inputStream,
                            rootColumnGroup, limit, options, exceptions);
                } else {
-                    Reader reader = ImportingUtilities.getFileReader(file, fileRecord);
+                    String commonEncoding = JSONUtilities.getString(options, "encoding", null);
+                    if (commonEncoding != null && commonEncoding.isEmpty()) {
+                        commonEncoding = null;
+                    }
+                    
+                    Reader reader = ImportingUtilities.getFileReader(file, fileRecord, commonEncoding);
                    parseOneFile(project, metadata, job, fileSource, reader,
                            rootColumnGroup, limit, options, exceptions);
                }
--- a/main/src/com/google/refine/importing/ImportingUtilities.java
+++ b/main/src/com/google/refine/importing/ImportingUtilities.java
@ -373,18 +373,21 @@ public class ImportingUtilities {
        return file;
    }
    
-    static public Reader getFileReader(ImportingJob job, JSONObject fileRecord)
+    static public Reader getFileReader(ImportingJob job, JSONObject fileRecord, String commonEncoding)
        throws FileNotFoundException {
        
-        return getFileReader(getFile(job, JSONUtilities.getString(fileRecord, "location", "")), fileRecord);
+        return getFileReader(getFile(job, JSONUtilities.getString(fileRecord, "location", "")), fileRecord, commonEncoding);
    }
    
-    static public Reader getFileReader(File file, JSONObject fileRecord) throws FileNotFoundException {
-        return getReaderFromStream(new FileInputStream(file), fileRecord);
+    static public Reader getFileReader(File file, JSONObject fileRecord, String commonEncoding) throws FileNotFoundException {
+        return getReaderFromStream(new FileInputStream(file), fileRecord, commonEncoding);
    }
    
-    static public Reader getReaderFromStream(InputStream inputStream, JSONObject fileRecord) {
+    static public Reader getReaderFromStream(InputStream inputStream, JSONObject fileRecord, String commonEncoding) {
        String encoding = getEncoding(fileRecord);
+        if (encoding == null) {
+            encoding = commonEncoding;
+        }
        if (encoding != null) {
            try {
                return new InputStreamReader(inputStream, encoding);
--- a/main/webapp/modules/core/MOD-INF/controller.js
+++ b/main/webapp/modules/core/MOD-INF/controller.js
@ -557,7 +557,28 @@ function process(path, request, response) {
          }
          context.scriptInjection = scriptInjection.join("\n");
        }
-
+        
+        if (path == "/index") {
+          var encodings = [];
+          
+          var sortedCharsetMap = Packages.java.nio.charset.Charset.availableCharsets();
+          for each (var code in sortedCharsetMap.keySet().toArray()) {
+            var charset = sortedCharsetMap.get(code);
+            var aliases = [];
+            for each (var alias in charset.aliases().toArray()) {
+              aliases.push(alias);
+            }
+            
+            encodings.push({
+              code: code,
+              name: charset.displayName(),
+              aliases: aliases
+            });
+          }
+          
+          context.encodingJson = butterfly.toJSONString(encodings);
+        }
+        
        send(request, response, path + ".vt", context);
      }
    }
--- a/main/webapp/modules/core/index.vt
+++ b/main/webapp/modules/core/index.vt
@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  <script type="text/javascript" src="wirings.js"></script>
  $scriptInjection
  $styleInjection
+  <script>Refine.encodings = $encodingJson;</script>
 </head>
 <body>
  <div id="header">
--- a/main/webapp/modules/core/scripts/index/create-project-ui.js
+++ b/main/webapp/modules/core/scripts/index/create-project-ui.js
@ -246,3 +246,61 @@ Refine.CreateProjectUI.prototype.showImportJobError = function(message, stack) {
  });
 };

+Refine.CreateProjectUI.populateEncodings = function(select) {
+  $('<option>')
+    .text('Unspecified')
+    .attr('value', '')
+    .appendTo(select);
+    
+  $.each(Refine.encodings, function() {
+    $('<option>')
+      .text(this.name + ' (' + this.aliases.join(', ') + ')')
+      .attr('value', this.code)
+      .appendTo(select);
+  });
+};
+
+Refine.CreateProjectUI.selectEncoding = function(input, onDone) {
+  var self = this;
+  var frame = $(DOM.loadHTML("core", "scripts/index/select-encoding-dialog.html"));
+  var elmts = DOM.bind(frame);
+  var level = DialogSystem.showDialog(frame);
+  
+  $("#select-encodings-tabs").tabs({ selected: 0 });
+  $("#select-encodings-tabs-all").css("display", "");
+  
+  var pickEncoding = function(encoding) {
+    input[0].value = encoding.code;
+    DialogSystem.dismissUntil(level - 1);
+    
+    if (onDone) {
+      onDone();
+    }
+  };
+  var renderEncoding = function(table, encoding) {
+    var tr = table.insertRow(table.rows.length);
+    $('<a>')
+      .text(encoding.name)
+      .attr('href', 'javascript:{}')
+      .click(function() {
+        return pickEncoding(encoding);
+      })
+      .appendTo(tr.insertCell(0));
+    $(tr.insertCell(1)).text(encoding.aliases.join(', '));
+  };
+  var generateEncodingList = function(container, filter) {
+    var table = $('<table>').html('<tr><th>Encoding</th><th>Aliases</th></tr>').appendTo(container)[0];
+    $.each(Refine.encodings, function() {
+      if (filter == null || this.code in filter) {
+        renderEncoding(table, this);
+      }
+    });
+  };
+  generateEncodingList(elmts.commonList,
+    { 'US-ASCII':1, 'ISO-8859-1':1, 'UTF-8':1, 'UTF-16BE':1, 'UTF-16LE':1, 'UTF-16':1 });
+  generateEncodingList(elmts.allList, null);
+  
+  elmts.cancelButton.click(function() {
+    DialogSystem.dismissUntil(level - 1);
+  });
+};
--- a/main/webapp/modules/core/scripts/index/parser-interfaces/fixed-width-parser-ui.html
+++ b/main/webapp/modules/core/scripts/index/parser-interfaces/fixed-width-parser-ui.html
@ -3,7 +3,7 @@
    <td><div class="grid-layout layout-tighter"><table>
      <tr>
        <td width="1%">Character&nbsp;encoding</td>
-        <td><select bind="encodingSelect"></select></td>
+        <td><input bind="encodingInput"></input></td>
      </tr>
    </table></div></td>
    <td colspan="2"><div class="grid-layout layout-tighter layout-full"><table>
--- a/main/webapp/modules/core/scripts/index/parser-interfaces/fixed-width-parser-ui.js
+++ b/main/webapp/modules/core/scripts/index/parser-interfaces/fixed-width-parser-ui.js
@ -75,7 +75,8 @@ Refine.FixedWidthParserUI.prototype.confirmReadyToCreateProject = function() {

 Refine.FixedWidthParserUI.prototype.getOptions = function() {
  var options = {
-      columnWidths: this._getColumnWidths()
+    encoding: $.trim(this._optionContainerElmts.encodingInput[0].value),
+    columnWidths: this._getColumnWidths()
  };

  var columnNames = $.trim(this._optionContainerElmts.columnNamesInput[0].value).replace(/,\s+/g, ',').split(',');
@ -139,7 +140,15 @@ Refine.FixedWidthParserUI.prototype._initialize = function() {
  this._optionContainer.unbind().empty().html(
      DOM.loadHTML("core", "scripts/index/parser-interfaces/fixed-width-parser-ui.html"));
  this._optionContainerElmts = DOM.bind(this._optionContainer);
-  this._optionContainerElmts.previewButton.click(function() { self._updatePreview(); });
+  this._optionContainerElmts.previewButton.click(function() { self.updatePreview(); });
+
+  this._optionContainerElmts.encodingInput
+    .attr('value', this._config.encoding || '')
+    .click(function() {
+      Refine.CreateProjectUI.selectEncoding($(this), function() {
+        self.updatePreview();
+      });
+    });

  this._optionContainerElmts.columnWidthsInput[0].value = this._config.columnWidths.join(',');
  if ('columnNames' in this._config) {
--- a/main/webapp/modules/core/scripts/index/parser-interfaces/line-based-parser-ui.html
+++ b/main/webapp/modules/core/scripts/index/parser-interfaces/line-based-parser-ui.html
@ -3,7 +3,7 @@
    <td><div class="grid-layout layout-tighter"><table>
      <tr>
        <td width="1%">Character&nbsp;encoding</td>
-        <td><select bind="encodingSelect"></select></td>
+        <td><input bind="encodingInput"></input></td>
      </tr>
    </table></div></td>
    <td colspan="2"><div class="grid-layout layout-tighter layout-full"><table>
--- a/main/webapp/modules/core/scripts/index/parser-interfaces/line-based-parser-ui.js
+++ b/main/webapp/modules/core/scripts/index/parser-interfaces/line-based-parser-ui.js
@ -63,7 +63,8 @@ Refine.LineBasedParserUI.prototype.dispose = function() {

 Refine.LineBasedParserUI.prototype.getOptions = function() {
  var options = {
-      recordPath: this._config.recordPath
+    encoding: $.trim(this._optionContainerElmts.encodingInput[0].value),
+    recordPath: this._config.recordPath
  };

  var parseIntDefault = function(s, def) {
@ -118,6 +119,14 @@ Refine.LineBasedParserUI.prototype._initialize = function() {
  this._optionContainerElmts = DOM.bind(this._optionContainer);
  this._optionContainerElmts.previewButton.click(function() { self._updatePreview(); });

+  this._optionContainerElmts.encodingInput
+    .attr('value', this._config.encoding || '')
+    .click(function() {
+      Refine.CreateProjectUI.selectEncoding($(this), function() {
+        self._updatePreview();
+      });
+    });
+
  this._optionContainerElmts.linesPerRowInput[0].value =
    this._config.linesPerRow.toString();

--- a/main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.html
+++ b/main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.html
@ -3,7 +3,7 @@
    <td><div class="grid-layout layout-tighter"><table>
      <tr>
        <td width="1%">Character&nbsp;encoding</td>
-        <td><select bind="encodingSelect"></select></td>
+        <td><input bind="encodingInput"></input></td>
      </tr>
    </table></div></td>
    <td colspan="2"><div class="grid-layout layout-tighter layout-full"><table>
--- a/main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.js
+++ b/main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.js
@ -73,8 +73,10 @@ Refine.SeparatorBasedParserUI.prototype.confirmReadyToCreateProject = function()
 };

 Refine.SeparatorBasedParserUI.prototype.getOptions = function() {
-  var options = {};
-
+  var options = {
+    encoding: $.trim(this._optionContainerElmts.encodingInput[0].value)
+  };
+  
  switch (this._optionContainer.find("input[name='row-separator']:checked")[0].value) {
  case 'new-line':
    options.lineSeparator = "\n";
@ -145,6 +147,14 @@ Refine.SeparatorBasedParserUI.prototype._initialize = function() {
  this._optionContainerElmts = DOM.bind(this._optionContainer);
  this._optionContainerElmts.previewButton.click(function() { self._updatePreview(); });

+  this._optionContainerElmts.encodingInput
+    .attr('value', this._config.encoding || '')
+    .click(function() {
+      Refine.CreateProjectUI.selectEncoding($(this), function() {
+        self._updatePreview();
+      });
+    });
+  
  var rowSeparatorValue = (this._config.lineSeparator == "\n") ? 'new-line' : 'custom';
  this._optionContainer.find(
      "input[name='row-separator'][value='" + rowSeparatorValue + "']").attr("checked", "checked");
--- a/main/webapp/modules/core/scripts/index/select-encoding-dialog.html
+++ b/main/webapp/modules/core/scripts/index/select-encoding-dialog.html
@ -0,0 +1,22 @@
+<div class="dialog-frame" style="width: 600px;">
+  <div class="dialog-border">
+    <div class="dialog-header" bind="dialogHeader">Select Encoding</div>
+    <div class="dialog-body" bind="dialogBody">
+      <div id="select-encodings-tabs" class="refine-tabs">
+        <ul>
+          <li><a href="#select-encodings-tabs-common">Common Encodings</a></li>
+          <li><a href="#select-encodings-tabs-all">All Encodings</a></li>
+        </ul>
+        <div id="select-encodings-tabs-common">
+          <div class="select-encoding-dialog-encoding-list" bind="commonList"></div>
+        </div>
+        <div id="select-encodings-tabs-all" style="display: none;">
+          <div class="select-encoding-dialog-encoding-list" bind="allList"></div>
+        </div>
+      </div>
+    </div>
+    <div class="dialog-footer" bind="dialogFooter">
+      <button class="button" bind="cancelButton">Cancel</button>
+    </div>
+  </div>
+</div>
--- a/main/webapp/modules/core/styles/index/create-project-ui.less
+++ b/main/webapp/modules/core/styles/index/create-project-ui.less
@ -143,3 +143,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  padding: @padding_normal;
  border: 1px solid @chrome_primary;
  }
+
+.select-encoding-dialog-encoding-list {
+  height: 20em;
+  overflow: auto;
+  }
+.select-encoding-dialog-encoding-list > table > tbody > tr > td,
+.select-encoding-dialog-encoding-list > table > tbody > tr > th {
+  padding: @padding_tight;
+  }
+.select-encoding-dialog-encoding-list a {
+  white-space: pre;
+  }