Fixed xml importer: subgroups should now line up properly by rows.

Added command to reorder columns using drag and drop.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1227 7d457c2a-affb-35e4-300a-418c747d4874

main/src/com/google/gridworks/commands/column/ReorderColumnsCommand.java

@@ -0,0 +1,25 @@
+package com.google.gridworks.commands.column;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.json.JSONObject;
+
+import com.google.gridworks.commands.EngineDependentCommand;
+import com.google.gridworks.model.AbstractOperation;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.operations.column.ColumnReorderOperation;
+import com.google.gridworks.util.JSONUtilities;
+import com.google.gridworks.util.ParsingUtilities;
+
+public class ReorderColumnsCommand extends EngineDependentCommand {
+    
+    @Override
+    protected AbstractOperation createOperation(Project project,
+            HttpServletRequest request, JSONObject engineConfig) throws Exception {
+        
+        String columnNames = request.getParameter("columnNames");
+        return new ColumnReorderOperation(
+                JSONUtilities.toStringList(
+                        ParsingUtilities.evaluateJsonStringToArray(columnNames)));
+    }
+}

main/src/com/google/gridworks/importers/XmlImportUtilities.java

@@ -54,6 +54,7 @@ public class XmlImportUtilities {
     static public class ImportColumnGroup extends ImportVertical {
         public Map<String, ImportColumnGroup> subgroups = new HashMap<String, ImportColumnGroup>();
         public Map<String, ImportColumn> columns = new HashMap<String, ImportColumn>();
+        public int nextRowIndex;
 
         @Override
         void tabulate() {
@@ -74,10 +75,12 @@ public class XmlImportUtilities {
      */
     static public class ImportColumn extends ImportVertical {
         public int      cellIndex;
+        public int      nextRowIndex;
         public boolean  blankOnFirstRow;
 
-        public ImportColumn(){}
+        public ImportColumn() {}
-        public ImportColumn(String name){ //required for testing
+        
+        public ImportColumn(String name) { //required for testing
             super.name = name;
         }
 
@@ -93,7 +96,6 @@ public class XmlImportUtilities {
      */
     static public class ImportRecord {
         public List<List<Cell>> rows = new LinkedList<List<Cell>>();
-        public List<Integer> columnEmptyRowIndices = new ArrayList<Integer>();
     }
 
     static public String[] detectPathFromTag(InputStream inputStream, String tag) {
@@ -319,6 +321,7 @@ public class XmlImportUtilities {
                 }
             }
         } catch (Exception e) {
+            e.printStackTrace();
             // silent
         }
     }
@@ -437,18 +440,22 @@ public class XmlImportUtilities {
         if (record.rows.size() > 0) {
             for (List<Cell> row : record.rows) {
                 Row realRow = new Row(row.size());
+                int cellCount = 0;
 
                 for (int c = 0; c < row.size(); c++) {
                     Cell cell = row.get(c);
                     if (cell != null) {
                         realRow.setCell(c, cell);
+                        cellCount++;
                     }
                 }
                 
+                if (cellCount > 0) {
                     project.rows.add(realRow);
                 }
             }
         }
+    }
 
     static protected String composeName(String prefix, String localName) {
         return prefix != null && prefix.length() > 0 ? (prefix + ":" + localName) : localName;
@@ -473,14 +480,7 @@ public class XmlImportUtilities {
                 columnGroup,
                 composeName(parser.getPrefix(), parser.getLocalName()));
         
-        int commonStartingRowIndex = 0;
+        thisColumnGroup.nextRowIndex = Math.max(thisColumnGroup.nextRowIndex, columnGroup.nextRowIndex);
-        for (ImportColumn column : thisColumnGroup.columns.values()) {
-            if (column.cellIndex < record.columnEmptyRowIndices.size()) {
-                commonStartingRowIndex = Math.max(
-                        commonStartingRowIndex,
-                        record.columnEmptyRowIndices.get(column.cellIndex));
-            }
-        }
         
         int attributeCount = parser.getAttributeCount();
         for (int i = 0; i < attributeCount; i++) {
@@ -491,8 +491,7 @@ public class XmlImportUtilities {
                     thisColumnGroup,
                     record,
                     composeName(parser.getAttributePrefix(i), parser.getAttributeLocalName(i)),
-                    text,
+                    text
-                    commonStartingRowIndex
                 );
             }
         }
@@ -515,8 +514,7 @@ public class XmlImportUtilities {
                         thisColumnGroup,
                         record,
                         null,
-                        parser.getText(),
+                        parser.getText()
-                        commonStartingRowIndex
                     );
                 }
             } else if (eventType == XMLStreamConstants.END_ELEMENT) {
@@ -524,16 +522,14 @@ public class XmlImportUtilities {
             }
         }
         
-        if (commonStartingRowIndex < record.rows.size()) {
+        int nextRowIndex = thisColumnGroup.nextRowIndex;
-            List<Cell> startingRow = record.rows.get(commonStartingRowIndex);
+        for (ImportColumn column2 : thisColumnGroup.columns.values()) {
-
+            nextRowIndex = Math.max(nextRowIndex, column2.nextRowIndex);
-            for (ImportColumn c : thisColumnGroup.columns.values()) {
-                int cellIndex = c.cellIndex;
-                if (cellIndex >= startingRow.size() || startingRow.get(cellIndex) == null) {
-                    c.blankOnFirstRow = true;
-                }
         }
+        for (ImportColumnGroup columnGroup2 : thisColumnGroup.subgroups.values()) {
+            nextRowIndex = Math.max(nextRowIndex, columnGroup2.nextRowIndex);
         }
+        thisColumnGroup.nextRowIndex = nextRowIndex;
     }
 
     static protected void addCell(
@@ -541,8 +537,7 @@ public class XmlImportUtilities {
         ImportColumnGroup columnGroup,
         ImportRecord record,
         String columnLocalName,
-        String text,
+        String text
-        int commonStartingRowIndex
     ) {
         if (text == null || ((String) text).isEmpty()) {
             return;
@@ -553,26 +548,21 @@ public class XmlImportUtilities {
         ImportColumn column = getColumn(project, columnGroup, columnLocalName);
         int cellIndex = column.cellIndex;
         
-        while (cellIndex >= record.columnEmptyRowIndices.size()) {
+        int rowIndex = Math.max(columnGroup.nextRowIndex, column.nextRowIndex);
-            record.columnEmptyRowIndices.add(commonStartingRowIndex);
-        }
-        int rowIndex = record.columnEmptyRowIndices.get(cellIndex);
-
         while (rowIndex >= record.rows.size()) {
             record.rows.add(new ArrayList<Cell>());
         }
-        List<Cell> row = record.rows.get(rowIndex);
         
+        List<Cell> row = record.rows.get(rowIndex);
         while (cellIndex >= row.size()) {
             row.add(null);
         }
         
         logger.trace("Adding cell with value : \"" + value + "\" to row : " + rowIndex + " at cell index : " + (cellIndex-1));
         
-        row.set(cellIndex-1, new Cell(value, null));
+        row.set(cellIndex, new Cell(value, null));
-
-        record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1);
         
+        column.nextRowIndex = rowIndex + 1;
         column.nonBlankCount++;
     }
 
@@ -604,6 +594,7 @@ public class XmlImportUtilities {
             (localName == null ? columnGroup.name : (columnGroup.name + " - " + localName));
 
         newColumn.cellIndex = project.columnModel.allocateNewCellIndex();
+        newColumn.nextRowIndex = columnGroup.nextRowIndex;
         
         return newColumn;
     }
@@ -635,6 +626,8 @@ public class XmlImportUtilities {
             (localName == null ? "Text" : localName) :
             (localName == null ? columnGroup.name : (columnGroup.name + " - " + localName));
 
+        newGroup.nextRowIndex = columnGroup.nextRowIndex;
+        
         return newGroup;
     }
 }

main/src/com/google/gridworks/importers/XmlImporter.java

@@ -52,7 +52,7 @@ public class XmlImporter implements StreamImporter {
             }
         }
 
-        if(recordPath == null)
+        if (recordPath == null)
             return;
         
         ImportColumnGroup rootColumnGroup = new ImportColumnGroup();

main/src/com/google/gridworks/model/RecordModel.java

@@ -164,7 +164,7 @@ public class RecordModel implements Jsonizable {
                 int c = 0;
                 for (int i = 0; i < group.columnSpan; i++) {
                     int columnIndex = group.startColumnIndex + i;
-                    if (columnIndex != group.keyColumnIndex) {
+                    if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) {
                         int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
                         keyedGroup.cellIndices[c++] = cellIndex;
                     }

main/src/com/google/gridworks/model/changes/ColumnReorderChange.java

@@ -0,0 +1,114 @@
+package com.google.gridworks.model.changes;
+
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import com.google.gridworks.history.Change;
+import com.google.gridworks.model.Column;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.util.Pool;
+
+public class ColumnReorderChange extends ColumnChange {
+    final protected List<String>    _columnNames;
+    protected List<Column>          _oldColumns;
+    protected List<Column>          _newColumns;
+    
+    public ColumnReorderChange(List<String> columnNames) {
+        _columnNames = columnNames;
+    }
+    
+    public void apply(Project project) {
+        synchronized (project) {
+            if (_newColumns == null) {
+                _newColumns = new ArrayList<Column>();
+                _oldColumns = new ArrayList<Column>(project.columnModel.columns);
+                
+                for (String n : _columnNames) {
+                    Column column = project.columnModel.getColumnByName(n);
+                    if (column != null) {
+                        _newColumns.add(column);
+                    }
+                }
+            }
+            
+            project.columnModel.columns.clear();
+            project.columnModel.columns.addAll(_newColumns);
+            project.update();
+        }
+    }
+
+    public void revert(Project project) {
+        synchronized (project) {
+            project.columnModel.columns.clear();
+            project.columnModel.columns.addAll(_oldColumns);
+            project.update();
+        }
+    }
+
+    public void save(Writer writer, Properties options) throws IOException {
+        writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
+        for (String n : _columnNames) {
+            writer.write(n);
+            writer.write('\n');
+        }
+        writer.write("oldColumnCount="); writer.write(Integer.toString(_oldColumns.size())); writer.write('\n');
+        for (Column c : _oldColumns) {
+            c.save(writer);
+            writer.write('\n');
+        }
+        writer.write("newColumnCount="); writer.write(Integer.toString(_newColumns.size())); writer.write('\n');
+        for (Column c : _newColumns) {
+            c.save(writer);
+            writer.write('\n');
+        }
+        writer.write("/ec/\n"); // end of change marker
+    }
+    
+    static public Change load(LineNumberReader reader, Pool pool) throws Exception {
+        List<String> columnNames = new ArrayList<String>();
+        List<Column> oldColumns = new ArrayList<Column>();
+        List<Column> newColumns = new ArrayList<Column>();
+        
+        String line;
+        while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
+            int equal = line.indexOf('=');
+            CharSequence field = line.subSequence(0, equal);
+            
+            if ("columnNameCount".equals(field)) {
+                int count = Integer.parseInt(line.substring(equal + 1));
+                for (int i = 0; i < count; i++) {
+                    line = reader.readLine();
+                    if (line != null) {
+                        columnNames.add(line);
+                    }
+                }
+            } else if ("oldColumnCount".equals(field)) {
+                int count = Integer.parseInt(line.substring(equal + 1));
+                for (int i = 0; i < count; i++) {
+                    line = reader.readLine();
+                    if (line != null) {
+                        oldColumns.add(Column.load(line));
+                    }
+                }
+            } else if ("newColumnCount".equals(field)) {
+                int count = Integer.parseInt(line.substring(equal + 1));
+                for (int i = 0; i < count; i++) {
+                    line = reader.readLine();
+                    if (line != null) {
+                        newColumns.add(Column.load(line));
+                    }
+                }
+            }
+        }
+        
+        ColumnReorderChange change = new ColumnReorderChange(columnNames);
+        change._oldColumns = oldColumns;
+        change._newColumns = newColumns;
+        
+        return change;
+    }
+}

main/src/com/google/gridworks/operations/column/ColumnReorderOperation.java

@@ -0,0 +1,60 @@
+package com.google.gridworks.operations.column;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONWriter;
+
+import com.google.gridworks.history.HistoryEntry;
+import com.google.gridworks.model.AbstractOperation;
+import com.google.gridworks.model.Project;
+import com.google.gridworks.model.changes.ColumnReorderChange;
+import com.google.gridworks.operations.OperationRegistry;
+import com.google.gridworks.util.JSONUtilities;
+
+public class ColumnReorderOperation extends AbstractOperation {
+    static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
+        List<String> columnNames = new ArrayList<String>();
+        
+        JSONUtilities.getStringList(obj, "columnNames", columnNames);
+        
+        return new ColumnReorderOperation(columnNames);
+    }
+    
+    final protected List<String> _columnNames;
+    
+    public ColumnReorderOperation(List<String> columnNames) {
+        _columnNames = columnNames;
+    }
+
+    public void write(JSONWriter writer, Properties options)
+            throws JSONException {
+        
+        writer.object();
+        writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
+        writer.key("description"); writer.value(getBriefDescription(null));
+        writer.key("columnNames"); writer.array();
+        for (String n : _columnNames) {
+            writer.value(n);
+        }
+        writer.endArray();
+        writer.endObject();
+    }
+
+    protected String getBriefDescription(Project project) {
+        return "Reorder columns";
+    }
+
+   protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
+        return new HistoryEntry(
+            historyEntryID,
+            project, 
+            "Reorder columns", 
+            this, 
+            new ColumnReorderChange(_columnNames)
+        );
+    }
+}

main/src/com/google/gridworks/util/JSONUtilities.java

@@ -1,5 +1,6 @@
 package com.google.gridworks.util;
 
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.List;
@@ -140,4 +141,15 @@ public class JSONUtilities {
         
         return a2;
     }
+    
+    static public List<String> toStringList(JSONArray a) throws JSONException {
+        int l = a.length();
+        
+        List<String> list = new ArrayList<String>();
+        for (int i = 0; i < l; i++) {
+            list.add(a.getString(i));
+        }
+        
+        return list;
+    }
 }

main/tests/server/src/com/google/gridworks/tests/importers/XmlImportUtilitiesStub.java

@@ -26,7 +26,7 @@ public class XmlImportUtilitiesStub extends XmlImportUtilities {
         super.processRecord(project, parser, rootColumnGroup);
     }
 
-    public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex){
+    public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex) {
-        super.addCell(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
+        super.addCell(project, columnGroup, record, columnLocalName, text);
     }
 }

main/tests/server/src/com/google/gridworks/tests/importers/XmlImportUtilitiesTests.java

@@ -341,17 +341,17 @@ public class XmlImportUtilitiesTests extends GridworksTest {
 
         Assert.assertNotNull(record);
         Assert.assertNotNull(record.rows);
-        Assert.assertNotNull(record.columnEmptyRowIndices);
+        //Assert.assertNotNull(record.columnEmptyRowIndices);
         Assert.assertEquals(record.rows.size(), 1);
-        Assert.assertEquals(record.columnEmptyRowIndices.size(), 2);
+        //Assert.assertEquals(record.columnEmptyRowIndices.size(), 2);
         Assert.assertNotNull(record.rows.get(0));
-        Assert.assertNotNull(record.columnEmptyRowIndices.get(0));
+        //Assert.assertNotNull(record.columnEmptyRowIndices.get(0));
-        Assert.assertNotNull(record.columnEmptyRowIndices.get(1));
+        //Assert.assertNotNull(record.columnEmptyRowIndices.get(1));
         Assert.assertEquals(record.rows.get(0).size(), 2);
         Assert.assertNotNull(record.rows.get(0).get(0));
         Assert.assertEquals(record.rows.get(0).get(0).value, "Author1, The");
-        Assert.assertEquals(record.columnEmptyRowIndices.get(0).intValue(),0);
+        //Assert.assertEquals(record.columnEmptyRowIndices.get(0).intValue(),0);
-        Assert.assertEquals(record.columnEmptyRowIndices.get(1).intValue(),1);
+        //Assert.assertEquals(record.columnEmptyRowIndices.get(1).intValue(),1);
 
     }
 

main/webapp/modules/core/MOD-INF/controller.js

@@ -57,6 +57,7 @@ function registerCommands() {
     GS.registerCommand(module, "split-column", new Packages.com.google.gridworks.commands.column.SplitColumnCommand());
     GS.registerCommand(module, "extend-data", new Packages.com.google.gridworks.commands.column.ExtendDataCommand());
     GS.registerCommand(module, "add-column-by-fetching-urls", new Packages.com.google.gridworks.commands.column.AddColumnByFetchingURLsCommand());
+    GS.registerCommand(module, "reorder-columns", new Packages.com.google.gridworks.commands.column.ReorderColumnsCommand());
 
     GS.registerCommand(module, "denormalize", new Packages.com.google.gridworks.commands.row.DenormalizeCommand());
 
@@ -120,6 +121,7 @@ function registerOperations() {
     OR.registerOperation(module, "column-split", Packages.com.google.gridworks.operations.column.ColumnSplitOperation);
     OR.registerOperation(module, "extend-data", Packages.com.google.gridworks.operations.column.ExtendDataOperation);
     OR.registerOperation(module, "column-addition-by-fetching-urls", Packages.com.google.gridworks.operations.column.ColumnAdditionByFetchingURLsOperation);
+    OR.registerOperation(module, "column-reorder", Packages.com.google.gridworks.operations.column.ColumnReorderOperation);
     
     OR.registerOperation(module, "row-removal", Packages.com.google.gridworks.operations.row.RowRemovalOperation);
     OR.registerOperation(module, "row-star", Packages.com.google.gridworks.operations.row.RowStarOperation);
@@ -228,6 +230,7 @@ function init() {
             "scripts/dialogs/scatterplot-dialog.js",
             "scripts/dialogs/extend-data-preview-dialog.js",
             "scripts/dialogs/templating-exporter-dialog.js",
+            "scripts/dialogs/column-reordering-dialog.js",
 
             "scripts/protograph/schema-alignment.js",
             "scripts/protograph/schema-alignment-ui-node.js",
@@ -266,6 +269,7 @@ function init() {
             "styles/dialogs/scatterplot-dialog.css",
             "styles/dialogs/freebase-loading-dialog.css",
             "styles/dialogs/extend-data-preview-dialog.css",
+            "styles/dialogs/column-reordering-dialog.css",
             
             "styles/reconciliation/recon-dialog.css",
             "styles/reconciliation/standard-service-panel.css",

main/webapp/modules/core/scripts/dialogs/column-reordering-dialog.html

@@ -0,0 +1,17 @@
+<div class="dialog-frame" style="width: 600px;">
+  <div class="dialog-header" bind="dialogHeader">Re-order Columns</div>
+  <div class="dialog-body" bind="dialogBody"><div class="grid-layout grid-layout-for-ui layout-normal layout-full"><table>
+    <tr>
+      <td>Drag columns to re-order</td>
+      <td>Drop columns here to remove</td>
+    </tr>
+    <tr>
+      <td width="50%"><div class="column-reordering-dialog-column-container" bind="columnContainer"></div></td>
+      <td><div class="column-reordering-dialog-column-container" bind="trashContainer"></div></td>
+    </tr>
+  </table></div></div>
+  <div class="dialog-footer" bind="dialogFooter">
+    <button bind="okButton">&nbsp;&nbsp;OK&nbsp;&nbsp;</button>
+    <button bind="cancelButton">Cancel</button>
+  </div>
+</div>

main/webapp/modules/core/scripts/dialogs/column-reordering-dialog.js

@@ -0,0 +1,48 @@
+function ColumnReorderingDialog() {
+    this._createDialog();
+}
+
+ColumnReorderingDialog.prototype._createDialog = function() {
+    var self = this;
+    var dialog = $(DOM.loadHTML("core", "scripts/dialogs/column-reordering-dialog.html"));
+    this._elmts = DOM.bind(dialog);
+    this._elmts.cancelButton.click(function() { self._dismiss(); });
+    this._elmts.okButton.click(function() { self._commit(); });
+    
+    this._level = DialogSystem.showDialog(dialog);
+    
+    for (var i = 0; i < theProject.columnModel.columns.length; i++) {
+        var column = theProject.columnModel.columns[i];
+        var name = column.name;
+        
+        $('<div>')
+            .addClass("column-reordering-dialog-column")
+            .text(name)
+            .attr("column", name)
+            .appendTo(this._elmts.columnContainer);
+    }
+    
+    dialog.find('.column-reordering-dialog-column-container')
+        .sortable({
+            connectWith: '.column-reordering-dialog-column-container'
+        })
+        .disableSelection();
+};
+
+ColumnReorderingDialog.prototype._dismiss = function() {
+    DialogSystem.dismissUntil(this._level - 1);
+};
+
+ColumnReorderingDialog.prototype._commit = function() {
+    var columnNames = this._elmts.columnContainer.find('div').map(function() { return this.getAttribute("column"); }).get();
+    
+    Gridworks.postCoreProcess(
+        "reorder-columns",
+        null,
+        { "columnNames" : JSON.stringify(columnNames) }, 
+        { modelsChanged: true },
+        { includeEngine: false }
+    );
+    
+    this._dismiss();
+};

main/webapp/modules/core/scripts/views/data-table/data-table-view.js

@@ -502,6 +502,16 @@ DataTableView.prototype._createMenuForAllColumns = function(elmt) {
                 }
             ]
         },
+        {   label: "Edit Columns",
+            submenu: [
+                {
+                    label: "Re-order Columns",
+                    click: function() {
+                        new ColumnReorderingDialog();
+                    }
+                }
+            ]
+        },
         {   label: "View",
             submenu: [
                 {
@@ -522,7 +532,7 @@ DataTableView.prototype._createMenuForAllColumns = function(elmt) {
                 }
             ]
         }
-    ], elmt, { width: "80px", horizontal: false });
+    ], elmt, { width: "120px", horizontal: false });
 };
 
 DataTableView.prototype._createSortingMenu = function(elmt) {

main/webapp/modules/core/styles/dialogs/column-reordering-dialog.css

@@ -0,0 +1,14 @@
+.column-reordering-dialog-column {
+    border: 1px solid #aaa;
+    background: #eee;
+    padding: 3px;
+    margin: 2px;
+    cursor: move;
+}
+
+.column-reordering-dialog-column-container {
+    height: 500px;
+    border: 1px solid #eee;
+    padding: 10px;
+    overflow: auto;
+}