Merge pull request #2110 from msaby/joincolumns

Add menu for joining columns

main/webapp/modules/core/MOD-INF/controller.js

@@ -516,6 +516,7 @@ function init() {
       "styles/widgets/slider-widget.less",
 
       "styles/views/data-table-view.less",
+      "styles/views/column-join.less",
 
       "styles/dialogs/expression-preview-dialog.less",
       "styles/dialogs/clustering-dialog.less",

main/webapp/modules/core/langs/translation-en.json

@@ -550,6 +550,7 @@
     "core-views/url-fetch": "Formulate the URLs to fetch:",
     "core-views/http-headers": "HTTP headers to be used when fetching URLs:",
     "core-views/enter-col-name": "Enter new column name",
+    "core-views/join-col":"Join columns",
     "core-views/split-col": "Split column",
     "core-views/several-col": "into several columns",
     "core-views/how-split": "How to Split Column",
@@ -680,6 +681,18 @@
     "core-views/use-values-as-identifiers2": "Mark cells as reconciled with their values as identifiers",
     "core-views/choose-reconciliation-service": "Choose a reconciliation service",
     "core-views/choose-reconciliation-service-alert": "Please choose a reconciliation service first.",
+    "core-views/column-join": "Join columns",
+    "core-views/column-join-before-column-picker": "Select and order columns to join",
+    "core-views/column-join-before-options": "Select options",
+    "core-views/column-join-field-separator": "Separator between the content of each column :",
+    "core-views/column-join-field-separator-advice": "Enter one or more characters, or keep blank to join the columns without separator.",
+    "core-views/column-join-replace-nulls": "Replace nulls with...",
+    "core-views/column-join-skip-nulls": "Skip nulls.",
+    "core-views/column-join-replace-nulls-advice": "Enter one or more characters, or keep blank to replace nulls with blank strings.",
+    "core-views/column-join-write-selected-column": "Write result in selected column.",
+    "core-views/column-join-copy-to-new-column": "Write result in new column named...",
+    "core-views/column-join-delete-joined-columns": "Delete joined columns.",
+    "core-views/column-join-dont-escape": "In separator and nulls substitutes, use \\n for new lines, \\t for tabulation, \\\\n for \\n, \\\\t for \\t.",
     "core-buttons/cancel": "Cancel",
     "core-buttons/ok": "  OK  ",
     "core-buttons/import-proj": "Import Project",

main/webapp/modules/core/scripts/views/data-table/column-join.html

@@ -0,0 +1,64 @@
+<div class="dialog-frame" style="width: 800px;">
+  <div class="dialog-header" bind="dialogHeader"></div>
+  <div class="dialog-body" bind="dialogBody">
+      <div class="grid-layout grid-layout-for-ui layout-full" >
+        <div class="column-join-left-pane">
+          <p bind = "or_views_column_join_before_column_picker"></p>
+          <div class="column-join-inner-left-pane">
+            <div bind="column_join_columnPicker" class="column-join-inner-left-pane-list"></div>
+            <div class="column-join-global-selection">
+              <button class="button" bind="selectAllButton"></button>
+              <button class="button" bind="deselectAllButton"></button>
+            </div>
+          </div>
+        </div>
+        <div class="column-join-right-pane grid-layout ">
+          <p bind="or_views_column_join_before_options"></p>
+          <div class="column-join-inner-right-pane">
+          <div class="option">
+            <label for="$field-separator" bind="or_views_column_join_field_separator"></label>
+            <input size="10" value="" bind="field_separatorInput" id="$field-separator"/>
+            <p class = "tip" bind= "or_views_column_join_field_separator_advice"></p>
+          </div>
+          <div class="option">
+            <div>
+            <input type="radio" checked="true" bind="replace_nullsInput" name="manage-nulls" value="replace-nulls" id="$replace-nulls" >
+            <label for="$replace-nulls" bind="or_views_column_join_replace_nulls"></label>
+            <input size="10" value="" bind="null_substituteInput" id="$null-substitute" />
+            <p class = "tip" bind= "or_views_column_join_replace_nulls_advice"></p>
+            
+          </div>
+          <div>
+            <input type="radio" bind="skip_nullsInput" name="manage-nulls" value="skip-nulls" id="$skip-nulls" />
+            <label for="$skip-nulls" bind="or_views_column_join_skip_nulls"></label>
+            <p class = "tip" bind= "or_views_column_join_skip_nulls_advice"></p>
+          </div>
+          </div>
+          <div class="option">
+            <input type="checkbox" bind ="dont_escapeInput" name="dont-escape" id="$dont-escapeInput" />
+            <label for="$dont-escape" bind="or_views_column_join_dont_escape"></label>
+          </div>
+          <div class="option">
+          <div>
+            <input type="radio" bind ="write_selected_columnInput" name="write-or-copy" value="write-selected-column" id="$write-selected-column" checked />
+            <label for="$write-selected-column" bind="or_views_column_join_write_selected_column"></label>
+          </div>
+          <div>
+            <input type="radio" bind ="copy_to_new_columnInput" name="write-or-copy" value="copy-to-new-column" id="$copy-to-new-column" />
+            <label for="$copy-to-new-column" bind="or_views_column_join_copy_to_new_column"></label>
+            <input size="20" value="" bind ="new_column_nameInput" id="$new-column-name" />
+          </div>
+          </div>
+          <div class="option">
+            <input type="checkbox" bind ="delete_joined_columnsInput" name="delete-joined-columns" id="$delete-joined-columns" />
+            <label for="$delete-joined-columns" bind="or_views_column_join_delete_joined_columns"></label>
+          </div>
+         </div>
+       </div>
+     </div>
+  </div>
+  <div class="dialog-footer" bind="dialogFooter">
+    <button class="button" bind="okButton"></button>
+    <button class="button" bind="cancelButton"></button>
+  </div>
+</div>

main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js

@@ -33,6 +33,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
   var columnIndex = Refine.columnNameToColumnIndex(column.name);
+  var doTextTransform = function(columnName, expression, onError, repeat, repeatCount, callbacks) {
+      callbacks = callbacks || {};
+	    Refine.postCoreProcess(
+	      "text-transform",
+	      {
+	        columnName: columnName,
+	        onError: onError,
+	        repeat: repeat,
+	        repeatCount: repeatCount
+	      },
+	      { expression: expression },
+	      { cellsChanged: true },
+	      callbacks
+	    );
+	  };
+	  
   var doAddColumn = function() {
     var frame = $(
         DOM.loadHTML("core", "scripts/views/data-table/add-column-dialog.html")
@@ -320,14 +336,224 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
       );
       dismiss();
     });
+  }; 
+  
+  var doJoinColumns = function() {
+    var self = this;
+    var dialog = $(DOM.loadHTML("core","scripts/views/data-table/column-join.html"));
+    var elmts = DOM.bind(dialog);
+    var level = DialogSystem.showDialog(dialog);
+    // Escape strings
+    function escapeString(s,dontEscape) {
+      var dontEscape = dontEscape || false;
+      var temp = s;
+      if (dontEscape) {
+        // replace "\n" with newline and "\t" with tab
+        temp = temp.replace(/\\n/g, '\n').replace(/\\t/g, '\t');
+        // replace "\" with "\\"
+        temp = temp.replace(/\\/g, '\\\\');
+        // replace "\newline" with "\n" and "\tab" with "\t"
+        temp = temp.replace(/\\\n/g, '\\n').replace(/\\\t/g, '\\t');
+        // replace ' with \'
+        temp = temp.replace(/'/g, "\\'");
+      } 
+      else {
+     // escape \ and '
+        temp = s.replace(/\\/g, '\\\\').replace(/'/g, "\\'") ; 
+        // useless : .replace(/"/g, '\\"')
+      }
+      return temp;
+    };
+    // Close the dialog window
+    var dismiss = function() {
+       DialogSystem.dismissUntil(level - 1);
+     };
+     // Join the columns according to user input
+    var transform = function() {
+      // function called in a callback
+      var deleteColumns = function() {
+        if (deleteJoinedColumns) {
+          console.log (theProject);
+          var columnsToKeep = theProject.columnModel.columns
+          .map (function (col) {return col.name;})
+          .filter (function(colName) {
+            // keep the selected column if it contains the result
+            return (
+                (columnsToJoin.indexOf (colName) == -1) ||
+                ((writeOrCopy !="copy-to-new-column") && (colName == column.name)));
+            }); 
+          Refine.postCoreProcess(
+              "reorder-columns",
+              null,
+              { "columnNames" : JSON.stringify(columnsToKeep) }, 
+              { modelsChanged: true },
+              { includeEngine: false }
+          );
+        }
+      };
+      // get options
+      var onError = "keep-original" ;
+      var repeat = false ;
+      var repeatCount = "";
+      var deleteJoinedColumns = elmts.delete_joined_columnsInput[0].checked;
+      var writeOrCopy = $("input[name='write-or-copy']:checked")[0].value;
+      var newColumnName = $.trim(elmts.new_column_nameInput[0].value);
+      var manageNulls = $("input[name='manage-nulls']:checked")[0].value;
+      var nullSubstitute = elmts.null_substituteInput[0].value;
+      var fieldSeparator = elmts.field_separatorInput[0].value;
+      var dontEscape = elmts.dont_escapeInput[0].checked;
+      // fix options if they are not consistent
+      if (newColumnName != "") {
+        writeOrCopy ="copy-to-new-column";
+        } else
+        {
+          writeOrCopy ="write-selected-column";
+        }
+      if (nullSubstitute != "") {
+          manageNulls ="replace-nulls";
+      }   
+      // build GREL expression
+      var columnsToJoin = [];
+      elmts.column_join_columnPicker
+        .find('.column-join-column input[type="checkbox"]:checked')
+        .each(function() {
+            columnsToJoin.push (this.closest ('.column-join-column').getAttribute('column'));
+         });
+      expression = columnsToJoin.map (function (colName) {
+        if (manageNulls == "skip-nulls") {
+          return "cells['"+escapeString(colName) +"'].value";
+        }
+      else {
+          return "coalesce(cells['"+escapeString(colName)+"'].value,'"+ escapeString(nullSubstitute,dontEscape) + "')";
+        }
+      }).join (',');
+      expression = 'join ([' + expression + '],\'' + escapeString(fieldSeparator,dontEscape) + "')";
+      // apply expression to selected column or new column
+      if (writeOrCopy =="copy-to-new-column") {
+        Refine.postCoreProcess(
+          "add-column", 
+          {
+          baseColumnName: column.name,  
+          newColumnName: newColumnName, 
+          columnInsertIndex: columnIndex + 1,
+          onError: onError
+          },
+          { expression: expression },
+          { modelsChanged: true },
+          { onFinallyDone: deleteColumns}
+        );
+      } 
+      else {
+        doTextTransform(
+            column.name,
+            expression,
+            onError,
+            repeat,
+            repeatCount,
+            { onFinallyDone: deleteColumns});
+      }
+    };
+    // core of doJoinColumn
+    elmts.dialogHeader.text($.i18n('core-views/column-join'));
+    elmts.or_views_column_join_before_column_picker.text($.i18n('core-views/column-join-before-column-picker'));
+    elmts.or_views_column_join_before_options.text($.i18n('core-views/column-join-before-options'));
+    elmts.or_views_column_join_replace_nulls.text($.i18n('core-views/column-join-replace-nulls'));
+    elmts.or_views_column_join_replace_nulls_advice.text($.i18n('core-views/column-join-replace-nulls-advice'));
+    elmts.or_views_column_join_skip_nulls.text($.i18n('core-views/column-join-skip-nulls'));
+    elmts.or_views_column_join_write_selected_column.text($.i18n('core-views/column-join-write-selected-column'));
+    elmts.or_views_column_join_copy_to_new_column.text($.i18n('core-views/column-join-copy-to-new-column'));
+    elmts.or_views_column_join_delete_joined_columns.text($.i18n('core-views/column-join-delete-joined-columns'));
+    elmts.or_views_column_join_field_separator.text($.i18n('core-views/column-join-field-separator'));
+    elmts.or_views_column_join_field_separator_advice.text($.i18n('core-views/column-join-field-separator-advice'));
+    elmts.or_views_column_join_dont_escape.text($.i18n('core-views/column-join-dont-escape'));
+    elmts.selectAllButton.html($.i18n('core-buttons/select-all'));
+    elmts.deselectAllButton.html($.i18n('core-buttons/deselect-all'));
+    elmts.okButton.html($.i18n('core-buttons/ok'));
+    elmts.cancelButton.html($.i18n('core-buttons/cancel'));
+    /*
+     * Populate column list.
+     */
+    for (var i = 0; i < theProject.columnModel.columns.length; i++) {
+      var col = theProject.columnModel.columns[i];
+      var colName = col.name;
+      var div = $('<div>').
+        addClass("column-join-column")
+        .attr("column", colName)
+        .appendTo(elmts.column_join_columnPicker);
+      $('<input>').
+        attr('type', 'checkbox')
+        .prop('checked',(i == columnIndex) ? true : false)
+        .appendTo(div);
+      $('<span>')
+        .text(colName)
+        .appendTo(div);
+    }
+    // Move the selected column on the top of the list
+    if (columnIndex > 0) {
+      selectedColumn = elmts.column_join_columnPicker
+        .find('.column-join-column')
+        .eq(columnIndex);
+      selectedColumn.parent().prepend(selectedColumn);
+     }
+    // Make the list sortable
+    elmts.column_join_columnPicker.sortable({});
+   /*
+    * Hook up event handlers.
+    */
+    elmts.column_join_columnPicker
+      .find('.column-join-column')
+      .click(function() {
+        elmts.column_join_columnPicker
+        .find('.column-join-column')
+        .removeClass('selected');
+        $(this).addClass('selected');
+      });
+    elmts.selectAllButton
+      .click(function() {
+        elmts.column_join_columnPicker
+        .find('input[type="checkbox"]')
+        .prop('checked',true);
+       });
+    elmts.deselectAllButton
+      .click(function() {
+        elmts.column_join_columnPicker
+        .find('input[type="checkbox"]')
+        .prop('checked',false);
+      });
+    elmts.okButton.click(function() {
+      transform();
+      dismiss();
+    });
+    elmts.cancelButton.click(function() {
+      dismiss();
+    });
+    elmts.new_column_nameInput.change(function() {
+      if (elmts.new_column_nameInput[0].value != "") {
+        elmts.copy_to_new_columnInput.prop('checked',true);
+      } else
+        {
+        elmts.write_selected_columnInput.prop('checked',true);
+        }
+    });
+    elmts.null_substituteInput.change(function() {
+        elmts.replace_nullsInput.prop('checked',true);
+    });
   };
 
+/*
+ * Create global menu
+ */
   MenuSystem.appendTo(menu, [ "core/edit-column" ], [
       {
         id: "core/split-column",
         label: $.i18n('core-views/split-into-col')+"...",
         click: doSplitColumn
       },
+      {
+        id: "core/join-column",
+        label: $.i18n('core-views/join-col')+"...",
+          click : doJoinColumns
+        },
       {},
       {
         id: "core/add-column",
@@ -369,7 +595,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
       {
         id: "core/move-column-to-left",
         label: $.i18n('core-views/move-to-left'),
-        click: function() { doMoveColumnBy(-1); }
+        click: function() { doMoveColumnBy(-1);}
       },
       {
         id: "core/move-column-to-right",

main/webapp/modules/core/styles/views/column-join.less

@@ -0,0 +1,85 @@
+/*
+
+Copyright 2019, Mathieu Saby.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+/*
+Same properties as dialogs/custom-tabular-exporter-dialog.less, applied to the class and ids of colum-join.html
+*/
+
+@import-less url("../theme.less");
+
+.dialog-header,.dialog_body,.dialog-footer,.global_selection {
+  clear:both;
+}
+
+.column-join-left-pane {
+  float:left;
+  width : 40%;
+  margin-right: 10px;
+}
+
+.column-join-inner-left-pane,.column-join-inner-right-pane {
+  border: 1px solid @chrome_primary;
+  height: 24em;
+  padding: @padding_loose;
+  }
+  
+.column-join-inner-left-pane-list {
+  overflow: auto;
+  height:22em;
+  }
+  
+
+.column-join-global-selection {
+  margin-top:0.5em;
+}
+
+.column-join-column {
+  border: 1px solid @chrome_primary;
+  background: @fill_secondary;
+  padding: @padding_tighter;
+  margin: @padding_tight;
+  cursor: move;
+  .rounded_corners();
+  }
+.column-join-column.selected {
+  background: @chrome_primary;
+  font-weight: bold;
+  }
+
+.column-join-inner-right-pane .option {
+  margin-top:1em;
+}
+
+.column-join-inner-right-pane .tip,.column-join-inner-left-pane .tip{
+  font-size:0.9em;
+}