Merge pull request #1278 from ostephens/cell-split-regex

Cell split regex

main/src/com/google/refine/commands/cell/SplitMultiValueCellsCommand.java

@@ -33,17 +33,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.commands.cell;
 
-  import java.io.IOException;
+import java.io.IOException;
 import java.util.Properties;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.json.JSONArray;
+
 import com.google.refine.commands.Command;
 import com.google.refine.model.AbstractOperation;
 import com.google.refine.model.Project;
 import com.google.refine.operations.cell.MultiValuedCellSplitOperation;
+import com.google.refine.util.ParsingUtilities;
 import com.google.refine.process.Process;
 
 public class SplitMultiValueCellsCommand extends Command {
@@ -58,11 +61,33 @@ public class SplitMultiValueCellsCommand extends Command {
             String keyColumnName = request.getParameter("keyColumnName");
             String separator = request.getParameter("separator");
             String mode = request.getParameter("mode");
-            
+            Boolean regex = Boolean.parseBoolean(request.getParameter("regex"));
-            AbstractOperation op = new MultiValuedCellSplitOperation(columnName, keyColumnName, separator, mode);
+
-            Process process = op.createProcess(project, new Properties());
+            if ("separator".equals(mode)) {
-            
+                AbstractOperation op = new MultiValuedCellSplitOperation(columnName, 
-            performProcessAndRespond(request, response, project, process);
+                                                                         keyColumnName,
+                                                                         separator, 
+                                                                         regex);
+                Process process = op.createProcess(project, new Properties());
+                
+                performProcessAndRespond(request, response, project, process);
+            } else {
+                String s = request.getParameter("fieldLengths");
+                
+                JSONArray a = ParsingUtilities.evaluateJsonStringToArray(s);
+                int[] fieldLengths = new int[a.length()];
+                
+                for (int i = 0; i < fieldLengths.length; i++) {
+                    fieldLengths[i] = a.getInt(i);
+                }
+                
+                AbstractOperation op = new MultiValuedCellSplitOperation(columnName,
+                                                                         keyColumnName,
+                                                                         fieldLengths);
+                Process process = op.createProcess(project, new Properties());
+                
+                performProcessAndRespond(request, response, project, process);
+            }
         } catch (Exception e) {
             respondException(response, e);
         }

main/src/com/google/refine/operations/cell/MultiValuedCellSplitOperation.java

@@ -33,9 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 package com.google.refine.operations.cell;
 
- import java.util.ArrayList;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang.StringUtils;
 import org.json.JSONException;
@@ -50,32 +51,64 @@ import com.google.refine.model.Project;
 import com.google.refine.model.Row;
 import com.google.refine.model.changes.MassRowChange;
 import com.google.refine.operations.OperationRegistry;
+import com.google.refine.util.JSONUtilities;
 
 public class MultiValuedCellSplitOperation extends AbstractOperation {
     final protected String  _columnName;
     final protected String  _keyColumnName;
-    final protected String  _separator;
     final protected String  _mode;
+    final protected String  _separator;
+    final protected boolean _regex;
+    
+    final protected int[]      _fieldLengths;
 
     static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
-        return new MultiValuedCellSplitOperation(
+        String mode = obj.getString("mode");
-            obj.getString("columnName"),
+
-            obj.getString("keyColumnName"),
+        if ("separator".equals(mode)) {
-            obj.getString("separator"),
+            return new MultiValuedCellSplitOperation(
-            obj.getString("mode")
+                obj.getString("columnName"),
-        );
+                obj.getString("keyColumnName"),
+                obj.getString("separator"),
+                obj.getBoolean("regex")
+            );
+        } else {
+            return new MultiValuedCellSplitOperation(
+                obj.getString("columnName"),
+                obj.getString("keyColumnName"),
+                JSONUtilities.getIntArray(obj, "fieldLengths")
+            );
+        }
     }
     
     public MultiValuedCellSplitOperation(
         String      columnName,
         String      keyColumnName,
-        String    separator,
+        String      separator,
-        String    mode
+        boolean     regex
     ) {
         _columnName = columnName;
         _keyColumnName = keyColumnName;
         _separator = separator;
-        _mode = mode;
+        _mode = "separator";
+        _regex = regex;
+        
+        _fieldLengths = null;
+    }
+
+    public MultiValuedCellSplitOperation(
+        String      columnName,
+        String      keyColumnName,
+        int[]       fieldLengths
+    ) {
+        _columnName = columnName;
+        _keyColumnName = keyColumnName;
+
+        _mode = "lengths";
+        _separator = null;
+        _regex = false;
+
+        _fieldLengths = fieldLengths;
     }
 
     @Override
@@ -87,8 +120,17 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
         writer.key("description"); writer.value("Split multi-valued cells in column " + _columnName);
         writer.key("columnName"); writer.value(_columnName);
         writer.key("keyColumnName"); writer.value(_keyColumnName);
-        writer.key("separator"); writer.value(_separator);
         writer.key("mode"); writer.value(_mode);
+        if ("separator".equals(_mode)) {
+            writer.key("separator"); writer.value(_separator);
+            writer.key("regex"); writer.value(_regex);
+        } else {
+            writer.key("fieldLengths"); writer.array();
+            for (int l : _fieldLengths) {
+                writer.value(l);
+            }
+            writer.endArray();
+        }
         writer.endObject();
     }
 
@@ -110,7 +152,7 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
             throw new Exception("No key column named " + _keyColumnName);
         }
         int keyCellIndex = keyColumn.getCellIndex();
-        
+
         List<Row> newRows = new ArrayList<Row>();
         
         int oldRowCount = project.rows.size();
@@ -124,8 +166,28 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
             Object value = oldRow.getCellValue(cellIndex);
             String s = value instanceof String ? ((String) value) : value.toString();
             String[] values = null;
-            if (_mode.equals("regex")) {
+            if("lengths".equals(_mode)) {
-                values = s.split(_separator);
+                if (_fieldLengths.length >= 0 && _fieldLengths[0] > 0) {
+                    values = new String[_fieldLengths.length];
+                    
+                    int lastIndex = 0;
+                    
+                    for (int i = 0; i < _fieldLengths.length; i++) {
+                        int thisIndex = lastIndex;
+                        
+                        Object o = _fieldLengths[i];
+                        if (o instanceof Number) {
+                            thisIndex = Math.min(s.length(), lastIndex + Math.max(0, ((Number) o).intValue()));
+                        }
+                        
+                        values[i] = s.substring(lastIndex, thisIndex);
+                        lastIndex = thisIndex;
+                    }
+                }
+            }
+            else if (_regex) {
+                Pattern pattern = Pattern.compile(_separator);
+                values = pattern.split(s);
             } else {
                 values = StringUtils.splitByWholeSeparatorPreserveAllTokens(s, _separator);
             }
@@ -138,14 +200,14 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
             // First value goes into the same row
             {
                 Row firstNewRow = oldRow.dup();
-                firstNewRow.setCell(cellIndex, new Cell(values[0].trim(), null));
+                firstNewRow.setCell(cellIndex, new Cell(values[0], null));
                 
                 newRows.add(firstNewRow);
             }
             
             int r2 = r + 1;
             for (int v = 1; v < values.length; v++) {
-                Cell newCell = new Cell(values[v].trim(), null);
+                Cell newCell = new Cell(values[v], null);
                 
                 if (r2 < project.rows.size()) {
                     Row oldRow2 = project.rows.get(r2);

main/tests/server/src/com/google/refine/tests/operations/cell/SplitMultiValuedCellsTests.java

@@ -0,0 +1,223 @@
+/*
+
+Copyright 2010, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+package com.google.refine.tests.model;
+
+import static org.mockito.Mockito.mock;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Properties;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.google.refine.ProjectManager;
+import com.google.refine.ProjectMetadata;
+import com.google.refine.RefineServlet;
+import com.google.refine.importers.SeparatorBasedImporter;
+import com.google.refine.importing.ImportingJob;
+import com.google.refine.importing.ImportingManager;
+import com.google.refine.io.FileProjectManager;
+import com.google.refine.model.AbstractOperation;
+import com.google.refine.model.ModelException;
+import com.google.refine.model.Project;
+import com.google.refine.process.Process;
+import com.google.refine.operations.cell.MultiValuedCellSplitOperation;
+import com.google.refine.tests.RefineServletStub;
+import com.google.refine.tests.RefineTest;
+import com.google.refine.tests.util.TestUtils;
+
+
+public class SplitMultiValuedCellsTests extends RefineTest {
+    // dependencies
+    private RefineServlet servlet;
+    private Project project;
+    private ProjectMetadata pm;
+    private JSONObject options;
+    private ImportingJob job;
+    private SeparatorBasedImporter importer;
+
+
+    @Override
+    @BeforeTest
+    public void init() {
+        logger = LoggerFactory.getLogger(this.getClass());
+    }
+
+    @BeforeMethod
+    public void SetUp() throws JSONException, IOException, ModelException {
+        servlet = new RefineServletStub();
+            File dir = TestUtils.createTempDirectory("openrefine-test-workspace-dir");
+            FileProjectManager.initialize(dir);
+            project = new Project();
+            pm = new ProjectMetadata();
+            pm.setName("SplitMultiValuedCells test");
+            ProjectManager.singleton.registerProject(project, pm);
+            options = mock(JSONObject.class);
+
+        ImportingManager.initialize(servlet);
+            job = ImportingManager.createJob();
+        importer = new SeparatorBasedImporter(); 
+    }
+
+    @AfterMethod
+    public void TearDown() {
+        ImportingManager.disposeJob(job.id);
+        ProjectManager.singleton.deleteProject(project.id);
+        job = null;
+            project = null;
+        pm = null;
+            options = null;
+    }
+
+    /**
+     * Test to demonstrate the intended behaviour of the function, for issue #1268
+     * https://github.com/OpenRefine/OpenRefine/issues/1268
+     */
+
+    @Test
+    public void testSplitMultiValuedCellsTextSeparator() throws Exception {
+        String csv = "Key,Value\n"
+            + "Record_1,one:two;three four\n";
+        prepareOptions(",", 10, 0, 0, 1, false, false);
+            List<Exception> exceptions = new ArrayList<Exception>();
+            importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
+            project.update();
+            ProjectManager.singleton.registerProject(project, pm);
+
+        AbstractOperation op = new MultiValuedCellSplitOperation(
+            "Value",
+            "Key",
+            ":",
+            false);
+            Process process = op.createProcess(project, new Properties());
+            process.performImmediate();
+
+        int keyCol = project.columnModel.getColumnByName("Key").getCellIndex();
+        int valueCol = project.columnModel.getColumnByName("Value").getCellIndex();
+        
+        Assert.assertEquals(project.rows.get(0).getCellValue(keyCol), "Record_1");
+        Assert.assertEquals(project.rows.get(0).getCellValue(valueCol), "one");
+        Assert.assertEquals(project.rows.get(1).getCellValue(keyCol), null);
+        Assert.assertEquals(project.rows.get(1).getCellValue(valueCol), "two;three four");
+    }
+
+    @Test
+    public void testSplitMultiValuedCellsRegExSeparator() throws Exception {
+        String csv = "Key,Value\n"
+            + "Record_1,one:two;three four\n";
+        prepareOptions(",", 10, 0, 0, 1, false, false);
+            List<Exception> exceptions = new ArrayList<Exception>();
+            importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
+            project.update();
+            ProjectManager.singleton.registerProject(project, pm);
+
+        AbstractOperation op = new MultiValuedCellSplitOperation(
+            "Value",
+            "Key",
+            "\\W",
+            true);
+            Process process = op.createProcess(project, new Properties());
+            process.performImmediate();
+
+        int keyCol = project.columnModel.getColumnByName("Key").getCellIndex();
+        int valueCol = project.columnModel.getColumnByName("Value").getCellIndex();
+        
+        Assert.assertEquals(project.rows.get(0).getCellValue(keyCol), "Record_1");
+        Assert.assertEquals(project.rows.get(0).getCellValue(valueCol), "one");
+        Assert.assertEquals(project.rows.get(1).getCellValue(keyCol), null);
+        Assert.assertEquals(project.rows.get(1).getCellValue(valueCol), "two");
+        Assert.assertEquals(project.rows.get(2).getCellValue(keyCol), null);
+        Assert.assertEquals(project.rows.get(2).getCellValue(valueCol), "three");
+        Assert.assertEquals(project.rows.get(3).getCellValue(keyCol), null);
+        Assert.assertEquals(project.rows.get(3).getCellValue(valueCol), "four");
+    }
+
+    @Test
+    public void testSplitMultiValuedCellsLengths() throws Exception {
+        String csv = "Key,Value\n"
+            + "Record_1,one:two;three four\n";
+        prepareOptions(",", 10, 0, 0, 1, false, false);
+        List<Exception> exceptions = new ArrayList<Exception>();
+        importer.parseOneFile(project, pm, job, "filesource", new StringReader(csv), -1, options, exceptions);
+        project.update();
+        ProjectManager.singleton.registerProject(project, pm);
+        int[] lengths = {4,4,6,4};
+
+        AbstractOperation op = new MultiValuedCellSplitOperation(
+            "Value",
+            "Key",
+            lengths);
+        Process process = op.createProcess(project, new Properties());
+        process.performImmediate();
+
+        int keyCol = project.columnModel.getColumnByName("Key").getCellIndex();
+        int valueCol = project.columnModel.getColumnByName("Value").getCellIndex();
+        
+        Assert.assertEquals(project.rows.get(0).getCellValue(keyCol), "Record_1");
+        Assert.assertEquals(project.rows.get(0).getCellValue(valueCol), "one:");
+        Assert.assertEquals(project.rows.get(1).getCellValue(keyCol), null);
+        Assert.assertEquals(project.rows.get(1).getCellValue(valueCol), "two;");
+        Assert.assertEquals(project.rows.get(2).getCellValue(keyCol), null);
+        Assert.assertEquals(project.rows.get(2).getCellValue(valueCol), "three ");
+        Assert.assertEquals(project.rows.get(3).getCellValue(keyCol), null);
+        Assert.assertEquals(project.rows.get(3).getCellValue(valueCol), "four");
+    }
+
+    private void prepareOptions(
+            String sep, int limit, int skip, int ignoreLines,
+            int headerLines, boolean guessValueType, boolean ignoreQuotes) {
+            
+            whenGetStringOption("separator", options, sep);
+            whenGetIntegerOption("limit", options, limit);
+            whenGetIntegerOption("skipDataLines", options, skip);
+            whenGetIntegerOption("ignoreLines", options, ignoreLines);
+            whenGetIntegerOption("headerLines", options, headerLines);
+            whenGetBooleanOption("guessCellValueTypes", options, guessValueType);
+            whenGetBooleanOption("processQuotes", options, !ignoreQuotes);
+            whenGetBooleanOption("storeBlankCellsAsNulls", options, true);
+        }
+
+
+}
+

main/webapp/modules/core/langs/translation-en.json

@@ -525,6 +525,7 @@
         "split-col": "Split column",
         "several-col": "into several columns",
         "how-split": "How to Split Column",
+        "how-split-cells": "How to split multi-valued cells",
         "by-sep": "by separator",
         "separator": "Separator",
         "reg-exp": "regular expression",

main/webapp/modules/core/scripts/views/data-table/menu-edit-cells.js

@@ -125,7 +125,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
         {
           columnName: column.name,
           keyColumnName: theProject.columnModel.keyColumnName,
-          separator: separator
+          separator
         },
         null,
         { rowsChanged: true }
@@ -134,20 +134,76 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
   };
 
   var doSplitMultiValueCells = function() {
-    var separator = window.prompt($.i18n._('core-views')["what-separator"], ",");
+
-    if (separator !== null) {
+    var frame = $(DOM.loadHTML("core", "scripts/views/data-table/split-multi-valued-cells-dialog.html"));
+    var elmts = DOM.bind(frame);
+    elmts.dialogHeader.text($.i18n._('core-views')["split-cells"]);
+    
+    elmts.or_views_howSplit.text($.i18n._('core-views')["how-split-cells"]);
+    elmts.or_views_bySep.text($.i18n._('core-views')["by-sep"]);
+    elmts.or_views_separator.text($.i18n._('core-views')["separator"]);
+    elmts.or_views_regExp.text($.i18n._('core-views')["reg-exp"]);
+
+    elmts.or_views_fieldLen.text($.i18n._('core-views')["field-len"]);
+    elmts.or_views_listInt.text($.i18n._('core-views')["list-int"]);
+
+    elmts.okButton.html($.i18n._('core-buttons')["ok"]);
+    elmts.cancelButton.text($.i18n._('core-buttons')["cancel"]);
+
+    var level = DialogSystem.showDialog(frame);
+    var dismiss = function() { DialogSystem.dismissUntil(level - 1); };
+
+    elmts.cancelButton.click(dismiss);
+    elmts.okButton.click(function() {
+      var mode = $("input[name='split-by-mode']:checked")[0].value;
+      var config = {
+        columnName: column.name,
+        keyColumnName: theProject.columnModel.keyColumnName,
+        mode
+      };
+      if (mode === "separator") {
+        config.separator = elmts.separatorInput[0].value;
+        if (!(config.separator)) {
+          alert($.i18n._('core-views')["specify-sep"]);
+          return;
+        }
+
+        config.regex = elmts.regexInput[0].checked;
+
+      } else {
+        var s = "[" + elmts.lengthsTextarea[0].value + "]";
+        try {
+          var a = JSON.parse(s);
+
+          var lengths = [];
+          $.each(a, function(i,n) { 
+            if (typeof n == "number") {
+              lengths.push(n); 
+            }
+          });
+
+          if (lengths.length === 0) {
+            alert($.i18n._('core-views')["warning-no-length"]);
+            return;
+          }
+
+          config.fieldLengths = JSON.stringify(lengths);
+          
+        } catch (e) {
+          alert($.i18n._('core-views')["warning-format"]);
+          return;
+        }
+      }
+
       Refine.postCoreProcess(
         "split-multi-value-cells", 
-        {
+        config,
-          columnName: column.name,
-          keyColumnName: theProject.columnModel.keyColumnName,
-          separator: separator,
-          mode: "plain"
-        },
         null,
         { rowsChanged: true }
       );
-    }
+
+      dismiss();
+    });
   };
 
   MenuSystem.appendTo(menu, [ "core/edit-cells" ], [

main/webapp/modules/core/scripts/views/data-table/split-multi-valued-cells-dialog.html

@@ -0,0 +1,40 @@
+<div class="dialog-frame" style="width: 600px;">
+  <div class="dialog-border">
+    <div class="dialog-header" bind="dialogHeader"></div>
+    <div class="dialog-body" bind="dialogBody">
+      <div class="grid-layout layout-looser layout-full"><table><tr>
+        <td>
+          <div class="grid-layout layout-tighter"><table>
+            <tr>
+              <td colspan="3"><h3><span bind="or_views_howSplit"></span></h3></td>
+            </tr>
+            <tr>
+              <td width="1%"><input type="radio" checked="true" name="split-by-mode" value="separator" id="$split-multi-valued-cells-by-separator" /></td>
+              <td colspan="2"><label for="$split-multi-valued-cells-by-separator" bind="or_views_bySep"></label></td>
+            </tr>
+            <tr><td></td>
+              <td bind="or_views_separator"></td>
+              <td><input size="10" value="," bind="separatorInput" />
+                <input type="checkbox" bind="regexInput" id="$split-column-regex" />
+                <label for="$split-multi-valued-cells-regex" bind="or_views_regExp"></label></td>
+            </tr>
+            <tr>
+              <td width="1%"><input type="radio" name="split-by-mode" value="lengths" id="$split-multi-valued-cells-by-lengths" /></td>
+              <td colspan="2"><label for="$split-multi-valued-cells-by-lengths" bind="or_views_fieldLen"></label></td>
+            </tr>
+            <tr><td></td>
+              <td colspan="2"><textarea style="width: 100%;" bind="lengthsTextarea"></textarea></td>
+            </tr>
+            <tr><td></td>
+              <td colspan="2" bind="or_views_listInt"></td>
+            </tr>
+          </table></div>
+        </td>
+      </table></div>
+    </div>
+    <div class="dialog-footer" bind="dialogFooter">
+      <button class="button" bind="okButton"></button>
+      <button class="button" bind="cancelButton"></button>
+    </div>
+  </div>
+</div>