s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed xml importer: subgroups should now line up properly by rows.

Browse Source 
Added command to reorder columns using drag and drop.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@1227 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-08-25 06:17:08 +00:00
parent 1106d07282
commit 367796488e
14 changed files with 359 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
main/src/com/google/gridworks/commands/column/ReorderColumnsCommand.java Normal file
View File

@ -0,0 +1,25 @@
package com.google.gridworks.commands.column;
import javax.servlet.http.HttpServletRequest;
import org.json.JSONObject;
import com.google.gridworks.commands.EngineDependentCommand;
import com.google.gridworks.model.AbstractOperation;
import com.google.gridworks.model.Project;
import com.google.gridworks.operations.column.ColumnReorderOperation;
import com.google.gridworks.util.JSONUtilities;
import com.google.gridworks.util.ParsingUtilities;
public class ReorderColumnsCommand extends EngineDependentCommand {
@Override
protected AbstractOperation createOperation(Project project,
HttpServletRequest request, JSONObject engineConfig) throws Exception {
String columnNames = request.getParameter("columnNames");
return new ColumnReorderOperation(
JSONUtilities.toStringList(
ParsingUtilities.evaluateJsonStringToArray(columnNames)));
}
}

91
main/src/com/google/gridworks/importers/XmlImportUtilities.java
View File

@ -54,6 +54,7 @@ public class XmlImportUtilities {
static public class ImportColumnGroup extends ImportVertical {
public Map<String, ImportColumnGroup> subgroups = new HashMap<String, ImportColumnGroup>();
public Map<String, ImportColumn> columns = new HashMap<String, ImportColumn>();
public int nextRowIndex;
@Override
void tabulate() {
@ -73,11 +74,13 @@ public class XmlImportUtilities {
*
*/
static public class ImportColumn extends ImportVertical {
public int cellIndex;
public boolean blankOnFirstRow;
public int cellIndex;
public int nextRowIndex;
public boolean blankOnFirstRow;
public ImportColumn(){}
public ImportColumn(String name){ //required for testing
public ImportColumn() {}
public ImportColumn(String name) { //required for testing
super.name = name;
}
@ -93,7 +96,6 @@ public class XmlImportUtilities {
*/
static public class ImportRecord {
public List<List<Cell>> rows = new LinkedList<List<Cell>>();
public List<Integer> columnEmptyRowIndices = new ArrayList<Integer>();
}
static public String[] detectPathFromTag(InputStream inputStream, String tag) {
@ -319,6 +321,7 @@ public class XmlImportUtilities {
}
}
} catch (Exception e) {
e.printStackTrace();
// silent
}
}
@ -437,15 +440,19 @@ public class XmlImportUtilities {
if (record.rows.size() > 0) {
for (List<Cell> row : record.rows) {
Row realRow = new Row(row.size());
int cellCount = 0;
for (int c = 0; c < row.size(); c++) {
Cell cell = row.get(c);
if (cell != null) {
realRow.setCell(c, cell);
cellCount++;
}
}
project.rows.add(realRow);
if (cellCount > 0) {
project.rows.add(realRow);
}
}
}
}
@ -472,16 +479,9 @@ public class XmlImportUtilities {
project,
columnGroup,
composeName(parser.getPrefix(), parser.getLocalName()));
int commonStartingRowIndex = 0;
for (ImportColumn column : thisColumnGroup.columns.values()) {
if (column.cellIndex < record.columnEmptyRowIndices.size()) {
commonStartingRowIndex = Math.max(
commonStartingRowIndex,
record.columnEmptyRowIndices.get(column.cellIndex));
}
}
thisColumnGroup.nextRowIndex = Math.max(thisColumnGroup.nextRowIndex, columnGroup.nextRowIndex);
int attributeCount = parser.getAttributeCount();
for (int i = 0; i < attributeCount; i++) {
String text = parser.getAttributeValue(i).trim();
@ -491,8 +491,7 @@ public class XmlImportUtilities {
thisColumnGroup,
record,
composeName(parser.getAttributePrefix(i), parser.getAttributeLocalName(i)),
text,
commonStartingRowIndex
text
);
}
}
@ -515,25 +514,22 @@ public class XmlImportUtilities {
thisColumnGroup,
record,
null,
parser.getText(),
commonStartingRowIndex
parser.getText()
);
}
} else if (eventType == XMLStreamConstants.END_ELEMENT) {
break;
}
}
if (commonStartingRowIndex < record.rows.size()) {
List<Cell> startingRow = record.rows.get(commonStartingRowIndex);
for (ImportColumn c : thisColumnGroup.columns.values()) {
int cellIndex = c.cellIndex;
if (cellIndex >= startingRow.size() || startingRow.get(cellIndex) == null) {
c.blankOnFirstRow = true;
}
}
int nextRowIndex = thisColumnGroup.nextRowIndex;
for (ImportColumn column2 : thisColumnGroup.columns.values()) {
nextRowIndex = Math.max(nextRowIndex, column2.nextRowIndex);
}
for (ImportColumnGroup columnGroup2 : thisColumnGroup.subgroups.values()) {
nextRowIndex = Math.max(nextRowIndex, columnGroup2.nextRowIndex);
}
thisColumnGroup.nextRowIndex = nextRowIndex;
}
static protected void addCell(
@ -541,38 +537,32 @@ public class XmlImportUtilities {
ImportColumnGroup columnGroup,
ImportRecord record,
String columnLocalName,
String text,
int commonStartingRowIndex
String text
) {
if (text == null || ((String) text).isEmpty()) {
return;
}
Serializable value = ImporterUtilities.parseCellValue(text);
ImportColumn column = getColumn(project, columnGroup, columnLocalName);
int cellIndex = column.cellIndex;
while (cellIndex >= record.columnEmptyRowIndices.size()) {
record.columnEmptyRowIndices.add(commonStartingRowIndex);
}
int rowIndex = record.columnEmptyRowIndices.get(cellIndex);
int rowIndex = Math.max(columnGroup.nextRowIndex, column.nextRowIndex);
while (rowIndex >= record.rows.size()) {
record.rows.add(new ArrayList<Cell>());
}
List<Cell> row = record.rows.get(rowIndex);
while (cellIndex >= row.size()) {
row.add(null);
}
logger.trace("Adding cell with value : \"" + value + "\" to row : " + rowIndex + " at cell index : " + (cellIndex-1));
row.set(cellIndex-1, new Cell(value, null));
record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1);
row.set(cellIndex, new Cell(value, null));
column.nextRowIndex = rowIndex + 1;
column.nonBlankCount++;
}
@ -604,7 +594,8 @@ public class XmlImportUtilities {
(localName == null ? columnGroup.name : (columnGroup.name + " - " + localName));
newColumn.cellIndex = project.columnModel.allocateNewCellIndex();
newColumn.nextRowIndex = columnGroup.nextRowIndex;
return newColumn;
}
@ -635,6 +626,8 @@ public class XmlImportUtilities {
(localName == null ? "Text" : localName) :
(localName == null ? columnGroup.name : (columnGroup.name + " - " + localName));
newGroup.nextRowIndex = columnGroup.nextRowIndex;
return newGroup;
}
}

6
main/src/com/google/gridworks/importers/XmlImporter.java
View File

@ -52,14 +52,14 @@ public class XmlImporter implements StreamImporter {
}
}
if(recordPath == null)
if (recordPath == null)
return;
ImportColumnGroup rootColumnGroup = new ImportColumnGroup();
XmlImportUtilities.importXml(pis, project, recordPath, rootColumnGroup);
XmlImportUtilities.createColumnsFromImport(project, rootColumnGroup);
project.columnModel.update();
}

2
main/src/com/google/gridworks/model/RecordModel.java
View File

@ -164,7 +164,7 @@ public class RecordModel implements Jsonizable {
int c = 0;
for (int i = 0; i < group.columnSpan; i++) {
int columnIndex = group.startColumnIndex + i;
if (columnIndex != group.keyColumnIndex) {
if (columnIndex != group.keyColumnIndex && columnIndex < columnModel.columns.size()) {
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
keyedGroup.cellIndices[c++] = cellIndex;
}

114
main/src/com/google/gridworks/model/changes/ColumnReorderChange.java Normal file
View File

@ -0,0 +1,114 @@
package com.google.gridworks.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import com.google.gridworks.history.Change;
import com.google.gridworks.model.Column;
import com.google.gridworks.model.Project;
import com.google.gridworks.util.Pool;
public class ColumnReorderChange extends ColumnChange {
final protected List<String> _columnNames;
protected List<Column> _oldColumns;
protected List<Column> _newColumns;
public ColumnReorderChange(List<String> columnNames) {
_columnNames = columnNames;
}
public void apply(Project project) {
synchronized (project) {
if (_newColumns == null) {
_newColumns = new ArrayList<Column>();
_oldColumns = new ArrayList<Column>(project.columnModel.columns);
for (String n : _columnNames) {
Column column = project.columnModel.getColumnByName(n);
if (column != null) {
_newColumns.add(column);
}
}
}
project.columnModel.columns.clear();
project.columnModel.columns.addAll(_newColumns);
project.update();
}
}
public void revert(Project project) {
synchronized (project) {
project.columnModel.columns.clear();
project.columnModel.columns.addAll(_oldColumns);
project.update();
}
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String n : _columnNames) {
writer.write(n);
writer.write('\n');
}
writer.write("oldColumnCount="); writer.write(Integer.toString(_oldColumns.size())); writer.write('\n');
for (Column c : _oldColumns) {
c.save(writer);
writer.write('\n');
}
writer.write("newColumnCount="); writer.write(Integer.toString(_newColumns.size())); writer.write('\n');
for (Column c : _newColumns) {
c.save(writer);
writer.write('\n');
}
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
List<String> columnNames = new ArrayList<String>();
List<Column> oldColumns = new ArrayList<Column>();
List<Column> newColumns = new ArrayList<Column>();
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
if ("columnNameCount".equals(field)) {
int count = Integer.parseInt(line.substring(equal + 1));
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
columnNames.add(line);
}
}
} else if ("oldColumnCount".equals(field)) {
int count = Integer.parseInt(line.substring(equal + 1));
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
oldColumns.add(Column.load(line));
}
}
} else if ("newColumnCount".equals(field)) {
int count = Integer.parseInt(line.substring(equal + 1));
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
newColumns.add(Column.load(line));
}
}
}
}
ColumnReorderChange change = new ColumnReorderChange(columnNames);
change._oldColumns = oldColumns;
change._newColumns = newColumns;
return change;
}
}

60
main/src/com/google/gridworks/operations/column/ColumnReorderOperation.java Normal file
View File

@ -0,0 +1,60 @@
package com.google.gridworks.operations.column;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.google.gridworks.history.HistoryEntry;
import com.google.gridworks.model.AbstractOperation;
import com.google.gridworks.model.Project;
import com.google.gridworks.model.changes.ColumnReorderChange;
import com.google.gridworks.operations.OperationRegistry;
import com.google.gridworks.util.JSONUtilities;
public class ColumnReorderOperation extends AbstractOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
List<String> columnNames = new ArrayList<String>();
JSONUtilities.getStringList(obj, "columnNames", columnNames);
return new ColumnReorderOperation(columnNames);
}
final protected List<String> _columnNames;
public ColumnReorderOperation(List<String> columnNames) {
_columnNames = columnNames;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
writer.key("columnNames"); writer.array();
for (String n : _columnNames) {
writer.value(n);
}
writer.endArray();
writer.endObject();
}
protected String getBriefDescription(Project project) {
return "Reorder columns";
}
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
return new HistoryEntry(
historyEntryID,
project,
"Reorder columns",
this,
new ColumnReorderChange(_columnNames)
);
}
}

12
main/src/com/google/gridworks/util/JSONUtilities.java
View File

@ -1,5 +1,6 @@
package com.google.gridworks.util;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
@ -140,4 +141,15 @@ public class JSONUtilities {
return a2;
}
static public List<String> toStringList(JSONArray a) throws JSONException {
int l = a.length();
List<String> list = new ArrayList<String>();
for (int i = 0; i < l; i++) {
list.add(a.getString(i));
}
return list;
}
}

4
main/tests/server/src/com/google/gridworks/tests/importers/XmlImportUtilitiesStub.java
View File

@ -26,7 +26,7 @@ public class XmlImportUtilitiesStub extends XmlImportUtilities {
super.processRecord(project, parser, rootColumnGroup);
}
public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex){
super.addCell(project, columnGroup, record, columnLocalName, text, commonStartingRowIndex);
public void addCellWrapper(Project project, ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, String text, int commonStartingRowIndex) {
super.addCell(project, columnGroup, record, columnLocalName, text);
}
}

12
main/tests/server/src/com/google/gridworks/tests/importers/XmlImportUtilitiesTests.java
View File

@ -341,17 +341,17 @@ public class XmlImportUtilitiesTests extends GridworksTest {
Assert.assertNotNull(record);
Assert.assertNotNull(record.rows);
Assert.assertNotNull(record.columnEmptyRowIndices);
//Assert.assertNotNull(record.columnEmptyRowIndices);
Assert.assertEquals(record.rows.size(), 1);
Assert.assertEquals(record.columnEmptyRowIndices.size(), 2);
//Assert.assertEquals(record.columnEmptyRowIndices.size(), 2);
Assert.assertNotNull(record.rows.get(0));
Assert.assertNotNull(record.columnEmptyRowIndices.get(0));
Assert.assertNotNull(record.columnEmptyRowIndices.get(1));
//Assert.assertNotNull(record.columnEmptyRowIndices.get(0));
//Assert.assertNotNull(record.columnEmptyRowIndices.get(1));
Assert.assertEquals(record.rows.get(0).size(), 2);
Assert.assertNotNull(record.rows.get(0).get(0));
Assert.assertEquals(record.rows.get(0).get(0).value, "Author1, The");
Assert.assertEquals(record.columnEmptyRowIndices.get(0).intValue(),0);
Assert.assertEquals(record.columnEmptyRowIndices.get(1).intValue(),1);
//Assert.assertEquals(record.columnEmptyRowIndices.get(0).intValue(),0);
//Assert.assertEquals(record.columnEmptyRowIndices.get(1).intValue(),1);
}

4
main/webapp/modules/core/MOD-INF/controller.js
View File

@ -57,6 +57,7 @@ function registerCommands() {
GS.registerCommand(module, "split-column", new Packages.com.google.gridworks.commands.column.SplitColumnCommand());
GS.registerCommand(module, "extend-data", new Packages.com.google.gridworks.commands.column.ExtendDataCommand());
GS.registerCommand(module, "add-column-by-fetching-urls", new Packages.com.google.gridworks.commands.column.AddColumnByFetchingURLsCommand());
GS.registerCommand(module, "reorder-columns", new Packages.com.google.gridworks.commands.column.ReorderColumnsCommand());
GS.registerCommand(module, "denormalize", new Packages.com.google.gridworks.commands.row.DenormalizeCommand());
@ -120,6 +121,7 @@ function registerOperations() {
OR.registerOperation(module, "column-split", Packages.com.google.gridworks.operations.column.ColumnSplitOperation);
OR.registerOperation(module, "extend-data", Packages.com.google.gridworks.operations.column.ExtendDataOperation);
OR.registerOperation(module, "column-addition-by-fetching-urls", Packages.com.google.gridworks.operations.column.ColumnAdditionByFetchingURLsOperation);
OR.registerOperation(module, "column-reorder", Packages.com.google.gridworks.operations.column.ColumnReorderOperation);
OR.registerOperation(module, "row-removal", Packages.com.google.gridworks.operations.row.RowRemovalOperation);
OR.registerOperation(module, "row-star", Packages.com.google.gridworks.operations.row.RowStarOperation);
@ -228,6 +230,7 @@ function init() {
"scripts/dialogs/scatterplot-dialog.js",
"scripts/dialogs/extend-data-preview-dialog.js",
"scripts/dialogs/templating-exporter-dialog.js",
"scripts/dialogs/column-reordering-dialog.js",
"scripts/protograph/schema-alignment.js",
"scripts/protograph/schema-alignment-ui-node.js",
@ -266,6 +269,7 @@ function init() {
"styles/dialogs/scatterplot-dialog.css",
"styles/dialogs/freebase-loading-dialog.css",
"styles/dialogs/extend-data-preview-dialog.css",
"styles/dialogs/column-reordering-dialog.css",
"styles/reconciliation/recon-dialog.css",
"styles/reconciliation/standard-service-panel.css",

17
main/webapp/modules/core/scripts/dialogs/column-reordering-dialog.html Normal file
View File

@ -0,0 +1,17 @@
<div class="dialog-frame" style="width: 600px;">
<div class="dialog-header" bind="dialogHeader">Re-order Columns</div>
<div class="dialog-body" bind="dialogBody"><div class="grid-layout grid-layout-for-ui layout-normal layout-full"><table>
<tr>
<td>Drag columns to re-order</td>
<td>Drop columns here to remove</td>
</tr>
<tr>
<td width="50%"><div class="column-reordering-dialog-column-container" bind="columnContainer"></div></td>
<td><div class="column-reordering-dialog-column-container" bind="trashContainer"></div></td>
</tr>
</table></div></div>
<div class="dialog-footer" bind="dialogFooter">
<button bind="okButton">&nbsp;&nbsp;OK&nbsp;&nbsp;</button>
<button bind="cancelButton">Cancel</button>
</div>
</div>

48
main/webapp/modules/core/scripts/dialogs/column-reordering-dialog.js Normal file
View File

@ -0,0 +1,48 @@
function ColumnReorderingDialog() {
this._createDialog();
}
ColumnReorderingDialog.prototype._createDialog = function() {
var self = this;
var dialog = $(DOM.loadHTML("core", "scripts/dialogs/column-reordering-dialog.html"));
this._elmts = DOM.bind(dialog);
this._elmts.cancelButton.click(function() { self._dismiss(); });
this._elmts.okButton.click(function() { self._commit(); });
this._level = DialogSystem.showDialog(dialog);
for (var i = 0; i < theProject.columnModel.columns.length; i++) {
var column = theProject.columnModel.columns[i];
var name = column.name;
$('<div>')
.addClass("column-reordering-dialog-column")
.text(name)
.attr("column", name)
.appendTo(this._elmts.columnContainer);
}
dialog.find('.column-reordering-dialog-column-container')
.sortable({
connectWith: '.column-reordering-dialog-column-container'
})
.disableSelection();
};
ColumnReorderingDialog.prototype._dismiss = function() {
DialogSystem.dismissUntil(this._level - 1);
};
ColumnReorderingDialog.prototype._commit = function() {
var columnNames = this._elmts.columnContainer.find('div').map(function() { return this.getAttribute("column"); }).get();
Gridworks.postCoreProcess(
"reorder-columns",
null,
{ "columnNames" : JSON.stringify(columnNames) },
{ modelsChanged: true },
{ includeEngine: false }
);
this._dismiss();
};

12
main/webapp/modules/core/scripts/views/data-table/data-table-view.js
View File

@ -502,6 +502,16 @@ DataTableView.prototype._createMenuForAllColumns = function(elmt) {
}
]
},
{ label: "Edit Columns",
submenu: [
{
label: "Re-order Columns",
click: function() {
new ColumnReorderingDialog();
}
}
]
},
{ label: "View",
submenu: [
{
@ -522,7 +532,7 @@ DataTableView.prototype._createMenuForAllColumns = function(elmt) {
}
]
}
], elmt, { width: "80px", horizontal: false });
], elmt, { width: "120px", horizontal: false });
};
DataTableView.prototype._createSortingMenu = function(elmt) {

14
main/webapp/modules/core/styles/dialogs/column-reordering-dialog.css Normal file
View File

@ -0,0 +1,14 @@
.column-reordering-dialog-column {
border: 1px solid #aaa;
background: #eee;
padding: 3px;
margin: 2px;
cursor: move;
}
.column-reordering-dialog-column-container {
height: 500px;
border: 1px solid #eee;
padding: 10px;
overflow: auto;
}