Implemented encoding selectors in a few importing parser UIs.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@2214 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/google/refine/importers/ImportingParserBase.java

@@ -48,6 +48,7 @@ import com.google.refine.importing.ImportingJob;
 import com.google.refine.importing.ImportingParser;
 import com.google.refine.importing.ImportingUtilities;
 import com.google.refine.model.Project;
+import com.google.refine.util.JSONUtilities;
 
 abstract public class ImportingParserBase implements ImportingParser {
     final protected boolean useInputStream;
@@ -98,7 +99,13 @@ abstract public class ImportingParserBase implements ImportingParser {
                 if (useInputStream) {
                     parseOneFile(project, metadata, job, fileSource, inputStream, limit, options, exceptions);
                 } else {
-                    Reader reader = ImportingUtilities.getReaderFromStream(inputStream, fileRecord);
+                    String commonEncoding = JSONUtilities.getString(options, "encoding", null);
+                    if (commonEncoding != null && commonEncoding.isEmpty()) {
+                        commonEncoding = null;
+                    }
+                    
+                    Reader reader = ImportingUtilities.getReaderFromStream(
+                        inputStream, fileRecord, commonEncoding);
                     
                     parseOneFile(project, metadata, job, fileSource, reader, limit, options, exceptions);
                 }

main/src/com/google/refine/importers/tree/TreeImportingParserBase.java

@@ -111,7 +111,12 @@ abstract public class TreeImportingParserBase implements ImportingParser {
                     parseOneFile(project, metadata, job, fileSource, inputStream,
                             rootColumnGroup, limit, options, exceptions);
                 } else {
-                    Reader reader = ImportingUtilities.getFileReader(file, fileRecord);
+                    String commonEncoding = JSONUtilities.getString(options, "encoding", null);
+                    if (commonEncoding != null && commonEncoding.isEmpty()) {
+                        commonEncoding = null;
+                    }
+                    
+                    Reader reader = ImportingUtilities.getFileReader(file, fileRecord, commonEncoding);
                     parseOneFile(project, metadata, job, fileSource, reader,
                             rootColumnGroup, limit, options, exceptions);
                 }

main/src/com/google/refine/importing/ImportingUtilities.java

@@ -373,18 +373,21 @@ public class ImportingUtilities {
         return file;
     }
     
-    static public Reader getFileReader(ImportingJob job, JSONObject fileRecord)
+    static public Reader getFileReader(ImportingJob job, JSONObject fileRecord, String commonEncoding)
         throws FileNotFoundException {
         
-        return getFileReader(getFile(job, JSONUtilities.getString(fileRecord, "location", "")), fileRecord);
+        return getFileReader(getFile(job, JSONUtilities.getString(fileRecord, "location", "")), fileRecord, commonEncoding);
     }
     
-    static public Reader getFileReader(File file, JSONObject fileRecord) throws FileNotFoundException {
+    static public Reader getFileReader(File file, JSONObject fileRecord, String commonEncoding) throws FileNotFoundException {
-        return getReaderFromStream(new FileInputStream(file), fileRecord);
+        return getReaderFromStream(new FileInputStream(file), fileRecord, commonEncoding);
     }
     
-    static public Reader getReaderFromStream(InputStream inputStream, JSONObject fileRecord) {
+    static public Reader getReaderFromStream(InputStream inputStream, JSONObject fileRecord, String commonEncoding) {
         String encoding = getEncoding(fileRecord);
+        if (encoding == null) {
+            encoding = commonEncoding;
+        }
         if (encoding != null) {
             try {
                 return new InputStreamReader(inputStream, encoding);

main/webapp/modules/core/MOD-INF/controller.js

@@ -557,7 +557,28 @@ function process(path, request, response) {
           }
           context.scriptInjection = scriptInjection.join("\n");
         }
-
+        
+        if (path == "/index") {
+          var encodings = [];
+          
+          var sortedCharsetMap = Packages.java.nio.charset.Charset.availableCharsets();
+          for each (var code in sortedCharsetMap.keySet().toArray()) {
+            var charset = sortedCharsetMap.get(code);
+            var aliases = [];
+            for each (var alias in charset.aliases().toArray()) {
+              aliases.push(alias);
+            }
+            
+            encodings.push({
+              code: code,
+              name: charset.displayName(),
+              aliases: aliases
+            });
+          }
+          
+          context.encodingJson = butterfly.toJSONString(encodings);
+        }
+        
         send(request, response, path + ".vt", context);
       }
     }

main/webapp/modules/core/index.vt

@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   <script type="text/javascript" src="wirings.js"></script>
   $scriptInjection
   $styleInjection
+  <script>Refine.encodings = $encodingJson;</script>
 </head>
 <body>
   <div id="header">

main/webapp/modules/core/scripts/index/create-project-ui.js

@@ -246,3 +246,61 @@ Refine.CreateProjectUI.prototype.showImportJobError = function(message, stack) {
   });
 };
 
+Refine.CreateProjectUI.populateEncodings = function(select) {
+  $('<option>')
+    .text('Unspecified')
+    .attr('value', '')
+    .appendTo(select);
+    
+  $.each(Refine.encodings, function() {
+    $('<option>')
+      .text(this.name + ' (' + this.aliases.join(', ') + ')')
+      .attr('value', this.code)
+      .appendTo(select);
+  });
+};
+
+Refine.CreateProjectUI.selectEncoding = function(input, onDone) {
+  var self = this;
+  var frame = $(DOM.loadHTML("core", "scripts/index/select-encoding-dialog.html"));
+  var elmts = DOM.bind(frame);
+  var level = DialogSystem.showDialog(frame);
+  
+  $("#select-encodings-tabs").tabs({ selected: 0 });
+  $("#select-encodings-tabs-all").css("display", "");
+  
+  var pickEncoding = function(encoding) {
+    input[0].value = encoding.code;
+    DialogSystem.dismissUntil(level - 1);
+    
+    if (onDone) {
+      onDone();
+    }
+  };
+  var renderEncoding = function(table, encoding) {
+    var tr = table.insertRow(table.rows.length);
+    $('<a>')
+      .text(encoding.name)
+      .attr('href', 'javascript:{}')
+      .click(function() {
+        return pickEncoding(encoding);
+      })
+      .appendTo(tr.insertCell(0));
+    $(tr.insertCell(1)).text(encoding.aliases.join(', '));
+  };
+  var generateEncodingList = function(container, filter) {
+    var table = $('<table>').html('<tr><th>Encoding</th><th>Aliases</th></tr>').appendTo(container)[0];
+    $.each(Refine.encodings, function() {
+      if (filter == null || this.code in filter) {
+        renderEncoding(table, this);
+      }
+    });
+  };
+  generateEncodingList(elmts.commonList,
+    { 'US-ASCII':1, 'ISO-8859-1':1, 'UTF-8':1, 'UTF-16BE':1, 'UTF-16LE':1, 'UTF-16':1 });
+  generateEncodingList(elmts.allList, null);
+  
+  elmts.cancelButton.click(function() {
+    DialogSystem.dismissUntil(level - 1);
+  });
+};

main/webapp/modules/core/scripts/index/parser-interfaces/fixed-width-parser-ui.html

@@ -3,7 +3,7 @@
     <td><div class="grid-layout layout-tighter"><table>
       <tr>
         <td width="1%">Character&nbsp;encoding</td>
-        <td><select bind="encodingSelect"></select></td>
+        <td><input bind="encodingInput"></input></td>
       </tr>
     </table></div></td>
     <td colspan="2"><div class="grid-layout layout-tighter layout-full"><table>

main/webapp/modules/core/scripts/index/parser-interfaces/fixed-width-parser-ui.js

@@ -75,7 +75,8 @@ Refine.FixedWidthParserUI.prototype.confirmReadyToCreateProject = function() {
 
 Refine.FixedWidthParserUI.prototype.getOptions = function() {
   var options = {
-      columnWidths: this._getColumnWidths()
+    encoding: $.trim(this._optionContainerElmts.encodingInput[0].value),
+    columnWidths: this._getColumnWidths()
   };
 
   var columnNames = $.trim(this._optionContainerElmts.columnNamesInput[0].value).replace(/,\s+/g, ',').split(',');
@@ -139,7 +140,15 @@ Refine.FixedWidthParserUI.prototype._initialize = function() {
   this._optionContainer.unbind().empty().html(
       DOM.loadHTML("core", "scripts/index/parser-interfaces/fixed-width-parser-ui.html"));
   this._optionContainerElmts = DOM.bind(this._optionContainer);
-  this._optionContainerElmts.previewButton.click(function() { self._updatePreview(); });
+  this._optionContainerElmts.previewButton.click(function() { self.updatePreview(); });
+
+  this._optionContainerElmts.encodingInput
+    .attr('value', this._config.encoding || '')
+    .click(function() {
+      Refine.CreateProjectUI.selectEncoding($(this), function() {
+        self.updatePreview();
+      });
+    });
 
   this._optionContainerElmts.columnWidthsInput[0].value = this._config.columnWidths.join(',');
   if ('columnNames' in this._config) {

main/webapp/modules/core/scripts/index/parser-interfaces/line-based-parser-ui.html

@@ -3,7 +3,7 @@
     <td><div class="grid-layout layout-tighter"><table>
       <tr>
         <td width="1%">Character&nbsp;encoding</td>
-        <td><select bind="encodingSelect"></select></td>
+        <td><input bind="encodingInput"></input></td>
       </tr>
     </table></div></td>
     <td colspan="2"><div class="grid-layout layout-tighter layout-full"><table>

main/webapp/modules/core/scripts/index/parser-interfaces/line-based-parser-ui.js

@@ -63,7 +63,8 @@ Refine.LineBasedParserUI.prototype.dispose = function() {
 
 Refine.LineBasedParserUI.prototype.getOptions = function() {
   var options = {
-      recordPath: this._config.recordPath
+    encoding: $.trim(this._optionContainerElmts.encodingInput[0].value),
+    recordPath: this._config.recordPath
   };
 
   var parseIntDefault = function(s, def) {
@@ -118,6 +119,14 @@ Refine.LineBasedParserUI.prototype._initialize = function() {
   this._optionContainerElmts = DOM.bind(this._optionContainer);
   this._optionContainerElmts.previewButton.click(function() { self._updatePreview(); });
 
+  this._optionContainerElmts.encodingInput
+    .attr('value', this._config.encoding || '')
+    .click(function() {
+      Refine.CreateProjectUI.selectEncoding($(this), function() {
+        self._updatePreview();
+      });
+    });
+
   this._optionContainerElmts.linesPerRowInput[0].value =
     this._config.linesPerRow.toString();
 

main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.html

@@ -3,7 +3,7 @@
     <td><div class="grid-layout layout-tighter"><table>
       <tr>
         <td width="1%">Character&nbsp;encoding</td>
-        <td><select bind="encodingSelect"></select></td>
+        <td><input bind="encodingInput"></input></td>
       </tr>
     </table></div></td>
     <td colspan="2"><div class="grid-layout layout-tighter layout-full"><table>

main/webapp/modules/core/scripts/index/parser-interfaces/separator-based-parser-ui.js

@@ -73,8 +73,10 @@ Refine.SeparatorBasedParserUI.prototype.confirmReadyToCreateProject = function()
 };
 
 Refine.SeparatorBasedParserUI.prototype.getOptions = function() {
-  var options = {};
+  var options = {
-
+    encoding: $.trim(this._optionContainerElmts.encodingInput[0].value)
+  };
+  
   switch (this._optionContainer.find("input[name='row-separator']:checked")[0].value) {
   case 'new-line':
     options.lineSeparator = "\n";
@@ -145,6 +147,14 @@ Refine.SeparatorBasedParserUI.prototype._initialize = function() {
   this._optionContainerElmts = DOM.bind(this._optionContainer);
   this._optionContainerElmts.previewButton.click(function() { self._updatePreview(); });
 
+  this._optionContainerElmts.encodingInput
+    .attr('value', this._config.encoding || '')
+    .click(function() {
+      Refine.CreateProjectUI.selectEncoding($(this), function() {
+        self._updatePreview();
+      });
+    });
+  
   var rowSeparatorValue = (this._config.lineSeparator == "\n") ? 'new-line' : 'custom';
   this._optionContainer.find(
       "input[name='row-separator'][value='" + rowSeparatorValue + "']").attr("checked", "checked");

main/webapp/modules/core/scripts/index/select-encoding-dialog.html

@@ -0,0 +1,22 @@
+<div class="dialog-frame" style="width: 600px;">
+  <div class="dialog-border">
+    <div class="dialog-header" bind="dialogHeader">Select Encoding</div>
+    <div class="dialog-body" bind="dialogBody">
+      <div id="select-encodings-tabs" class="refine-tabs">
+        <ul>
+          <li><a href="#select-encodings-tabs-common">Common Encodings</a></li>
+          <li><a href="#select-encodings-tabs-all">All Encodings</a></li>
+        </ul>
+        <div id="select-encodings-tabs-common">
+          <div class="select-encoding-dialog-encoding-list" bind="commonList"></div>
+        </div>
+        <div id="select-encodings-tabs-all" style="display: none;">
+          <div class="select-encoding-dialog-encoding-list" bind="allList"></div>
+        </div>
+      </div>
+    </div>
+    <div class="dialog-footer" bind="dialogFooter">
+      <button class="button" bind="cancelButton">Cancel</button>
+    </div>
+  </div>
+</div>

main/webapp/modules/core/styles/index/create-project-ui.less

@@ -143,3 +143,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   padding: @padding_normal;
   border: 1px solid @chrome_primary;
   }
+
+.select-encoding-dialog-encoding-list {
+  height: 20em;
+  overflow: auto;
+  }
+.select-encoding-dialog-encoding-list > table > tbody > tr > td,
+.select-encoding-dialog-encoding-list > table > tbody > tr > th {
+  padding: @padding_tight;
+  }
+.select-encoding-dialog-encoding-list a {
+  white-space: pre;
+  }