diff --git a/src/main/java/com/metaweb/gridworks/importers/XmlImportUtilities.java b/src/main/java/com/metaweb/gridworks/importers/XmlImportUtilities.java index 41dcb5419..dc681ac11 100644 --- a/src/main/java/com/metaweb/gridworks/importers/XmlImportUtilities.java +++ b/src/main/java/com/metaweb/gridworks/importers/XmlImportUtilities.java @@ -17,6 +17,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import com.metaweb.gridworks.model.Cell; +import com.metaweb.gridworks.model.ColumnGroup; import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.model.Column; @@ -27,16 +28,38 @@ public class XmlImportUtilities { int count; } - static public class ImportColumnGroup { + static protected abstract class ImportVertical { public String name = ""; + public int nonBlankCount; - public Map subgroups = new HashMap(); - public Map columns = new HashMap(); + abstract void tabulate(); } - static public class ImportColumn { + static public class ImportColumnGroup extends ImportVertical { + public Map subgroups = new HashMap(); + public Map columns = new HashMap(); + + @Override + void tabulate() { + for (ImportColumn c : columns.values()) { + c.tabulate(); + nonBlankCount = Math.max(nonBlankCount, c.nonBlankCount); + } + for (ImportColumnGroup g : subgroups.values()) { + g.tabulate(); + nonBlankCount = Math.max(nonBlankCount, g.nonBlankCount); + } + } + } + + static public class ImportColumn extends ImportVertical { public int cellIndex; - public String name; + public boolean blankOnFirstRow; + + @Override + void tabulate() { + // already done the tabulation elsewhere + } } static public class ImportRecord { @@ -44,6 +67,58 @@ public class XmlImportUtilities { List columnEmptyRowIndices = new ArrayList(); } + static public String[] detectPathFromTag(InputStream inputStream, String tag) { + List candidates = new ArrayList(); + + try { + XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(inputStream); + + while (parser.hasNext()) { + int eventType = parser.next(); + if (eventType == XMLStreamConstants.START_ELEMENT) { + List path = detectRecordElement(parser, tag); + if (path != null) { + String[] path2 = new String[path.size()]; + + path.toArray(path2); + + return path2; + } + } + } + } catch (Exception e) { + // silent + // e.printStackTrace(); + } + + return null; + } + + static protected List detectRecordElement(XMLStreamReader parser, String tag) throws XMLStreamException { + String localName = parser.getLocalName(); + String fullName = composeName(parser.getPrefix(), localName); + if (tag.equals(parser.getLocalName()) || tag.equals(fullName)) { + List path = new LinkedList(); + path.add(localName); + + return path; + } + + while (parser.hasNext()) { + int eventType = parser.next(); + if (eventType == XMLStreamConstants.END_ELEMENT) { + break; + } else if (eventType == XMLStreamConstants.START_ELEMENT) { + List path = detectRecordElement(parser, tag); + if (path != null) { + path.add(0, localName); + return path; + } + } + } + return null; + } + static public String[] detectRecordElement(InputStream inputStream) { List candidates = new ArrayList(); @@ -77,19 +152,30 @@ public class XmlImportUtilities { } static protected RecordElementCandidate detectRecordElement(XMLStreamReader parser, String[] path) { - List candidateList = new ArrayList(); + List descendantCandidates = new ArrayList(); - Map candidates = new HashMap(); + Map immediateChildCandidateMap = new HashMap(); + int textNodeCount = 0; + int childElementNodeCount = 0; try { while (parser.hasNext()) { int eventType = parser.next(); if (eventType == XMLStreamConstants.END_ELEMENT) { break; + } else if (eventType == XMLStreamConstants.CHARACTERS) { + if (parser.getText().trim().length() > 0) { + textNodeCount++; + } } else if (eventType == XMLStreamConstants.START_ELEMENT) { + childElementNodeCount++; + String tagName = parser.getLocalName(); - candidates.put(tagName, candidates.containsKey(tagName) ? candidates.get(tagName) + 1 : 1); + immediateChildCandidateMap.put( + tagName, + immediateChildCandidateMap.containsKey(tagName) ? + immediateChildCandidateMap.get(tagName) + 1 : 1); String[] path2 = new String[path.length + 1]; System.arraycopy(path, 0, path2, 0, path.length); @@ -97,7 +183,7 @@ public class XmlImportUtilities { RecordElementCandidate c = detectRecordElement(parser, path2); if (c != null) { - candidateList.add(c); + descendantCandidates.add(c); } } } @@ -106,9 +192,14 @@ public class XmlImportUtilities { // e.printStackTrace(); } - if (candidates.size() > 0) { - List ourCandidateList = new ArrayList(candidates.size()); - for (Entry entry : candidates.entrySet()) { + if (textNodeCount > 0 && childElementNodeCount > 0) { + // This is a mixed element + return null; + } + + if (immediateChildCandidateMap.size() > 0) { + List immediateChildCandidates = new ArrayList(immediateChildCandidateMap.size()); + for (Entry entry : immediateChildCandidateMap.entrySet()) { int count = entry.getValue(); if (count > 1) { String[] path2 = new String[path.length + 1]; @@ -118,26 +209,32 @@ public class XmlImportUtilities { RecordElementCandidate candidate = new RecordElementCandidate(); candidate.path = path2; candidate.count = count; - ourCandidateList.add(candidate); + immediateChildCandidates.add(candidate); } } - if (ourCandidateList.size() > 0) { - sortRecordElementCandidates(ourCandidateList); + if (immediateChildCandidates.size() > 0 && immediateChildCandidates.size() < 5) { + // There are some promising immediate child elements, but not many, + // that can serve as record elements. - RecordElementCandidate ourCandidate = ourCandidateList.get(0); - if (ourCandidate.count > 10) { + sortRecordElementCandidates(immediateChildCandidates); + + RecordElementCandidate ourCandidate = immediateChildCandidates.get(0); + if (ourCandidate.count / immediateChildCandidates.size() > 5) { return ourCandidate; } - candidateList.add(ourCandidate); + descendantCandidates.add(ourCandidate); } } - if (candidateList.size() > 0) { - sortRecordElementCandidates(candidateList); + if (descendantCandidates.size() > 0) { + sortRecordElementCandidates(descendantCandidates); - return candidateList.get(0); + RecordElementCandidate candidate = descendantCandidates.get(0); + if (candidate.count / descendantCandidates.size() > 5) { + return candidate; + } } return null; @@ -175,14 +272,44 @@ public class XmlImportUtilities { Project project, ImportColumnGroup columnGroup ) { - for (ImportColumn c : columnGroup.columns.values()) { + int startColumnIndex = project.columnModel.columns.size(); + + List columns = new ArrayList(columnGroup.columns.values()); + Collections.sort(columns, new Comparator() { + public int compare(ImportColumn o1, ImportColumn o2) { + if (o1.blankOnFirstRow != o2.blankOnFirstRow) { + return o1.blankOnFirstRow ? 1 : -1; + } + + int c = o2.nonBlankCount - o1.nonBlankCount; + return c != 0 ? c : (o1.name.length() - o2.name.length()); + } + }); + + for (int i = 0; i < columns.size(); i++) { + ImportColumn c = columns.get(i); + Column column = new com.metaweb.gridworks.model.Column(c.cellIndex, c.name); project.columnModel.columns.add(column); } - for (ImportColumnGroup g : columnGroup.subgroups.values()) { + List subgroups = new ArrayList(columnGroup.subgroups.values()); + Collections.sort(subgroups, new Comparator() { + public int compare(ImportColumnGroup o1, ImportColumnGroup o2) { + int c = o2.nonBlankCount - o1.nonBlankCount; + return c != 0 ? c : (o1.name.length() - o2.name.length()); + } + }); + + for (ImportColumnGroup g : subgroups) { createColumnsFromImport(project, g); } + + int endColumnIndex = project.columnModel.columns.size(); + int span = endColumnIndex - startColumnIndex; + if (span > 1 && span < project.columnModel.columns.size()) { + project.columnModel.addColumnGroup(startColumnIndex, span, startColumnIndex); + } } static protected void findRecord( @@ -231,28 +358,39 @@ public class XmlImportUtilities { processSubRecord(project, parser, rootColumnGroup, record); - for (List row : record.rows) { - Row realRow = new Row(row.size()); - - for (int c = 0; c < row.size(); c++) { - Cell cell = row.get(c); - if (cell != null) { - realRow.setCell(c, cell); + if (record.rows.size() > 0) { + for (List row : record.rows) { + Row realRow = new Row(row.size()); + + for (int c = 0; c < row.size(); c++) { + Cell cell = row.get(c); + if (cell != null) { + realRow.setCell(c, cell); + } } + + project.rows.add(realRow); } - - project.rows.add(realRow); } } + static protected String composeName(String prefix, String localName) { + return prefix != null && prefix.length() > 0 ? (prefix + ":" + localName) : localName; + } + static protected void processSubRecord( Project project, XMLStreamReader parser, ImportColumnGroup columnGroup, ImportRecord record ) throws XMLStreamException { + ImportColumnGroup thisColumnGroup = getColumnGroup( + project, + columnGroup, + composeName(parser.getPrefix(), parser.getLocalName())); + int commonStartingRowIndex = 0; - for (ImportColumn column : columnGroup.columns.values()) { + for (ImportColumn column : thisColumnGroup.columns.values()) { if (column.cellIndex < record.columnEmptyRowIndices.size()) { commonStartingRowIndex = Math.max( commonStartingRowIndex, @@ -260,43 +398,58 @@ public class XmlImportUtilities { } } + int attributeCount = parser.getAttributeCount(); + for (int i = 0; i < attributeCount; i++) { + String text = parser.getAttributeValue(i).trim(); + if (text.length() > 0) { + addCell( + project, + thisColumnGroup, + record, + composeName(parser.getAttributePrefix(i), parser.getAttributeLocalName(i)), + text, + commonStartingRowIndex + ); + } + } + while (parser.hasNext()) { int eventType = parser.next(); if (eventType == XMLStreamConstants.START_ELEMENT) { - ImportColumnGroup thisColumnGroup = getColumnGroup(project, columnGroup, parser.getLocalName()); - - int attributeCount = parser.getAttributeCount(); - for (int i = 0; i < attributeCount; i++) { - addCell( - project, - thisColumnGroup, - record, - parser.getAttributeLocalName(i), - parser.getAttributeValue(i), - commonStartingRowIndex - ); - } - processSubRecord( project, parser, thisColumnGroup, record ); - } else if (eventType == XMLStreamConstants.CDATA || + } else if (//eventType == XMLStreamConstants.CDATA || eventType == XMLStreamConstants.CHARACTERS) { - addCell( - project, - columnGroup, - record, - null, - parser.getText(), - commonStartingRowIndex - ); + String text = parser.getText().trim(); + if (text.length() > 0) { + addCell( + project, + thisColumnGroup, + record, + null, + parser.getText(), + commonStartingRowIndex + ); + } } else if (eventType == XMLStreamConstants.END_ELEMENT) { break; } } + + if (commonStartingRowIndex < record.rows.size()) { + List startingRow = record.rows.get(commonStartingRowIndex); + + for (ImportColumn c : thisColumnGroup.columns.values()) { + int cellIndex = c.cellIndex; + if (cellIndex >= startingRow.size() || startingRow.get(cellIndex) == null) { + c.blankOnFirstRow = true; + } + } + } } static protected void addCell( @@ -304,13 +457,15 @@ public class XmlImportUtilities { ImportColumnGroup columnGroup, ImportRecord record, String columnLocalName, - Serializable value, + String text, int commonStaringRowIndex ) { - if (value == null || (value instanceof String && ((String) value).isEmpty())) { + if (text == null || ((String) text).isEmpty()) { return; } + Serializable value = ImporterUtilities.parseCellValue(text); + ImportColumn column = getColumn(project, columnGroup, columnLocalName); int cellIndex = column.cellIndex; @@ -331,6 +486,8 @@ public class XmlImportUtilities { row.set(cellIndex, new Cell(value, null)); record.columnEmptyRowIndices.set(cellIndex, rowIndex + 1); + + column.nonBlankCount++; } static protected ImportColumn getColumn( diff --git a/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java b/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java index 0a4690ecf..995d14f95 100644 --- a/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java +++ b/src/main/java/com/metaweb/gridworks/importers/XmlImporter.java @@ -23,9 +23,13 @@ public class XmlImporter implements Importer { throw new NotImplementedException(); } - public void read(InputStream inputStream, Project project, - Properties options, int skip, int limit) throws Exception { - + public void read( + InputStream inputStream, + Project project, + Properties options, + int skip, + int limit + ) throws Exception { BufferedInputStream bis = new BufferedInputStream(inputStream); String[] recordPath = null; @@ -36,14 +40,22 @@ public class XmlImporter implements Importer { int c = bis.read(buffer); bis.reset(); - recordPath = XmlImportUtilities.detectRecordElement( - new ByteArrayInputStream(buffer, 0, c)); + if (options.containsKey("importer-record-tag")) { + recordPath = XmlImportUtilities.detectPathFromTag( + new ByteArrayInputStream(buffer, 0, c), + options.getProperty("importer-record-tag")); + } else { + recordPath = XmlImportUtilities.detectRecordElement( + new ByteArrayInputStream(buffer, 0, c)); + } } ImportColumnGroup rootColumnGroup = new ImportColumnGroup(); XmlImportUtilities.importXml(bis, project, recordPath, rootColumnGroup); XmlImportUtilities.createColumnsFromImport(project, rootColumnGroup); + + project.columnModel.update(); } } diff --git a/src/main/java/com/metaweb/gridworks/model/ColumnModel.java b/src/main/java/com/metaweb/gridworks/model/ColumnModel.java index d5999e9b8..d8280be89 100644 --- a/src/main/java/com/metaweb/gridworks/model/ColumnModel.java +++ b/src/main/java/com/metaweb/gridworks/model/ColumnModel.java @@ -53,9 +53,27 @@ public class ColumnModel implements Jsonizable { public int getKeyColumnIndex() { return _keyColumnIndex; } + + public void addColumnGroup(int startColumnIndex, int span, int keyColumnIndex) { + for (ColumnGroup g : columnGroups) { + if (g.startColumnIndex == startColumnIndex && g.columnSpan == span) { + if (g.keyColumnIndex == keyColumnIndex) { + return; + } else { + columnGroups.remove(g); + break; + } + } + } + + ColumnGroup cg = new ColumnGroup(startColumnIndex, span, keyColumnIndex); + + columnGroups.add(cg); + + } public void update() { - generateMaps(); + internalInitialize(); } public Column getColumnByName(String name) { diff --git a/src/main/webapp/scripts/views/data-table-view.js b/src/main/webapp/scripts/views/data-table-view.js index a25636f6a..719095867 100644 --- a/src/main/webapp/scripts/views/data-table-view.js +++ b/src/main/webapp/scripts/views/data-table-view.js @@ -161,7 +161,9 @@ DataTableView.prototype._renderDataTable = function(table) { c += (columnGroup.columnSpan - 1); - nextLayer = nextLayer.concat(columnGroup.subgroups); + if ("subgroups" in columnGroup) { + nextLayer = nextLayer.concat(columnGroup.subgroups); + } } } } @@ -172,12 +174,13 @@ DataTableView.prototype._renderDataTable = function(table) { renderColumnGroups(nextLayer, []); } }; - /* - renderColumnGroups( - columnGroups, - [ theProject.columnModel.keyCellIndex ] - ); - */ + + if (columnGroups.length > 0) { + renderColumnGroups( + columnGroups, + [ theProject.columnModel.keyCellIndex ] + ); + } /*------------------------------------------------------------ * Column Headers with Menus