s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Factored row dependency code from Row class and Project class out as Record and RecordModel classes.

Browse Source 
Simplified RdfTripleImporter.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@820 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-05-19 04:22:45 +00:00
parent e1861eb1d9
commit 1e737e3238
21 changed files with 393 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java
View File

@ -6,6 +6,7 @@ import java.util.List;
import com.metaweb.gridworks.browsing.filters.RowFilter; import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.RecordModel.RowDependency;
/** /**
* Encapsulate logic for visiting rows that match all give row filters. Also visit * Encapsulate logic for visiting rows that match all give row filters. Also visit
@ -32,13 +33,14 @@ public class ConjunctiveFilteredRows implements FilteredRows {
int c = project.rows.size(); int c = project.rows.size();
for (int rowIndex = 0; rowIndex < c; rowIndex++) { for (int rowIndex = 0; rowIndex < c; rowIndex++) {
Row row = project.rows.get(rowIndex); Row row = project.rows.get(rowIndex);
RowDependency rd = project.recordModel.getRowDependency(rowIndex);
if (matchRow(project, rowIndex, row)) { if (matchRow(project, rowIndex, row)) {
if (row.recordIndex >= 0) { if (rd.recordIndex >= 0) {
lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself
} }
visitRow(project, visitor, rowIndex, row, lastVisitedRowRowIndex); visitRow(project, visitor, rowIndex, row, rd, lastVisitedRowRowIndex);
lastVisitedRowRowIndex = rowIndex; lastVisitedRowRowIndex = rowIndex;
} else if ( } else if (
@ -47,11 +49,11 @@ public class ConjunctiveFilteredRows implements FilteredRows {
_includeDependent && _includeDependent &&
// and this row is a dependent row since it's not a record row // and this row is a dependent row since it's not a record row
row.recordIndex < 0 && rd.recordIndex < 0 &&
row.contextRows != null && rd.contextRows != null &&
row.contextRows.size() > 0 && rd.contextRows.size() > 0 &&
row.contextRows.get(0) == lastRecordRowAcceptedRowIndex rd.contextRows.get(0) == lastRecordRowAcceptedRowIndex
) { ) {
// this row depends on the last previously matched record row, // this row depends on the last previously matched record row,
// so we visit it as well as a dependent row // so we visit it as well as a dependent row
@ -62,13 +64,13 @@ public class ConjunctiveFilteredRows implements FilteredRows {
} }
} }
protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, int lastVisitedRow) { protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, RowDependency rd, int lastVisitedRow) {
if (_includeContextual && // we need to include any context row and if (_includeContextual && // we need to include any context row and
row.contextRows != null && // this row itself isn't a context row and rd.contextRows != null && // this row itself isn't a context row and
lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row
// that we haven't visited yet // that we haven't visited yet
) { ) {
for (int contextRowIndex : row.contextRows) { for (int contextRowIndex : rd.contextRows) {
if (contextRowIndex > lastVisitedRow) { if (contextRowIndex > lastVisitedRow) {
visitor.visit( visitor.visit(
project, project,

3
src/main/java/com/metaweb/gridworks/commands/project/CreateProjectCommand.java
View File

@ -90,8 +90,7 @@ public class CreateProjectCommand extends Command {
pm.setEncodingConfidence(options.getProperty("encoding_confidence")); pm.setEncodingConfidence(options.getProperty("encoding_confidence"));
ProjectManager.singleton.registerProject(project, pm); ProjectManager.singleton.registerProject(project, pm);
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
redirect(response, "/project.html?project=" + project.id); redirect(response, "/project.html?project=" + project.id);
} catch (Exception e) { } catch (Exception e) {

1
src/main/java/com/metaweb/gridworks/commands/row/GetRowsCommand.java
View File

@ -32,6 +32,7 @@ public class GetRowsCommand extends Command {
Pool pool = new Pool(); Pool pool = new Pool();
Properties options = new Properties(); Properties options = new Properties();
options.put("project", project);
options.put("reconCandidateOmitTypes", true); options.put("reconCandidateOmitTypes", true);
options.put("pool", pool); options.put("pool", pool);

33
src/main/java/com/metaweb/gridworks/expr/WrappedRow.java
View File

@ -5,6 +5,7 @@ import java.util.Properties;
import com.metaweb.gridworks.model.Cell; import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column; import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.model.Row;
public class WrappedRow implements HasFields { public class WrappedRow implements HasFields {
@ -25,10 +26,8 @@ public class WrappedRow implements HasFields {
return rowIndex; return rowIndex;
} else if ("record".equals(name)) { } else if ("record".equals(name)) {
int rowIndex = (Integer) bindings.get("rowIndex"); int rowIndex = (Integer) bindings.get("rowIndex");
int recordRowIndex = (row.contextRows != null && row.contextRows.size() > 0) ?
row.contextRows.get(0) : rowIndex;
return new Record(recordRowIndex, rowIndex); return new WrappedRecord(project.recordModel.getRecordOfRow(rowIndex));
} else if ("columnNames".equals(name)) { } else if ("columnNames".equals(name)) {
Project project = (Project) bindings.get("project"); Project project = (Project) bindings.get("project");
@ -42,18 +41,16 @@ public class WrappedRow implements HasFields {
return row.fieldAlsoHasFields(name); return row.fieldAlsoHasFields(name);
} }
protected class Record implements HasFields { protected class WrappedRecord implements HasFields {
final int _recordRowIndex; final Record _record;
final int _currentRowIndex;
protected Record(int recordRowIndex, int currentRowIndex) { protected WrappedRecord(Record record) {
_recordRowIndex = recordRowIndex; _record = record;
_currentRowIndex = currentRowIndex;
} }
public Object getField(String name, Properties bindings) { public Object getField(String name, Properties bindings) {
if ("cells".equals(name)) { if ("cells".equals(name)) {
return new RecordCells(_recordRowIndex); return new RecordCells(_record);
} }
return null; return null;
} }
@ -64,28 +61,20 @@ public class WrappedRow implements HasFields {
} }
protected class RecordCells implements HasFields { protected class RecordCells implements HasFields {
final int _recordRowIndex; final Record _record;
protected RecordCells(int recordRowIndex) { protected RecordCells(Record record) {
_recordRowIndex = recordRowIndex; _record = record;
} }
public Object getField(String name, Properties bindings) { public Object getField(String name, Properties bindings) {
Column column = project.columnModel.getColumnByName(name); Column column = project.columnModel.getColumnByName(name);
if (column != null) { if (column != null) {
Row recordRow = project.rows.get(_recordRowIndex);
int cellIndex = column.getCellIndex(); int cellIndex = column.getCellIndex();
HasFieldsListImpl cells = new HasFieldsListImpl(); HasFieldsListImpl cells = new HasFieldsListImpl();
for (int r = _record.fromRowIndex; r < _record.toRowIndex; r++) {
int recordIndex = recordRow.recordIndex;
int count = project.rows.size();
for (int r = _recordRowIndex; r < count; r++) {
Row row = project.rows.get(r); Row row = project.rows.get(r);
if (row.recordIndex > recordIndex) {
break;
}
Cell cell = row.getCell(cellIndex); Cell cell = row.getCell(cellIndex);
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) { if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
cells.add(new WrappedCell(project, name, cell)); cells.add(new WrappedCell(project, name, cell));

118
src/main/java/com/metaweb/gridworks/importers/RdfTripleImporter.java
View File

@ -2,7 +2,12 @@ package com.metaweb.gridworks.importers;
import java.io.InputStream; import java.io.InputStream;
import java.io.Reader; import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties; import java.util.Properties;
import java.util.Map.Entry;
import org.jrdf.JRDFFactory; import org.jrdf.JRDFFactory;
import org.jrdf.SortedMemoryJRDFFactory; import org.jrdf.SortedMemoryJRDFFactory;
@ -17,8 +22,10 @@ import static org.jrdf.graph.AnyObjectNode.ANY_OBJECT_NODE;
import static org.jrdf.graph.AnyPredicateNode.ANY_PREDICATE_NODE; import static org.jrdf.graph.AnyPredicateNode.ANY_PREDICATE_NODE;
import static org.jrdf.graph.AnySubjectNode.ANY_SUBJECT_NODE; import static org.jrdf.graph.AnySubjectNode.ANY_SUBJECT_NODE;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.model.Cell; import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column; import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.ModelException;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.model.Row;
@ -40,12 +47,13 @@ public class RdfTripleImporter implements Importer{
Graph graph = JrdfFactory.getNewGraph(); Graph graph = JrdfFactory.getNewGraph();
LineHandler lineHandler = nTriplesParserFactory.createParser(graph, newMapFactory); LineHandler lineHandler = nTriplesParserFactory.createParser(graph, newMapFactory);
GraphLineParser parser = new GraphLineParser(graph, lineHandler); GraphLineParser parser = new GraphLineParser(graph, lineHandler);
parser.parse(reader, baseUrl); //fills JRDF graph parser.parse(reader, baseUrl); // fills JRDF graph
//first column is subject Map<String, List<Row>> subjectToRows = new HashMap<String, List<Row>>();
project.columnModel.columns.add(0, new Column(0, "subject"));
project.columnModel.setKeyColumnIndex(0); //the subject will be the key column Column subjectColumn = new Column(0, "subject");
project.columnModel.update(); project.columnModel.columns.add(0, subjectColumn);
project.columnModel.setKeyColumnIndex(0);
ClosableIterable<Triple> triples = graph.find(ANY_SUBJECT_NODE, ANY_PREDICATE_NODE, ANY_OBJECT_NODE); ClosableIterable<Triple> triples = graph.find(ANY_SUBJECT_NODE, ANY_PREDICATE_NODE, ANY_OBJECT_NODE);
try { try {
@ -54,83 +62,53 @@ public class RdfTripleImporter implements Importer{
String predicate = triple.getPredicate().toString(); String predicate = triple.getPredicate().toString();
String object = triple.getObject().toString(); String object = triple.getObject().toString();
//creates new column for every predicate Column column = project.columnModel.getColumnByName(predicate);
int columnIndex = project.columnModel.getColumnIndexByName(predicate); if (column == null) {
if(columnIndex == -1){ column = new Column(project.columnModel.allocateNewCellIndex(), predicate);
AddNewColumn(project, predicate, subject); try {
project.columnModel.addColumn(-1, column, true);
} catch (ModelException e) {
// ignore
}
} }
//now find row to match with int cellIndex = column.getCellIndex();
int candidateMergeRowIndex = -1; if (subjectToRows.containsKey(subject)) {
for(int i = 0; i < project.rows.size(); i++){ List<Row> rows = subjectToRows.get(subject);
//check to see if the subjects are the same (merge if they are) for (Row row : rows) {
Cell cell = project.rows.get(i).cells.get(0); if (!ExpressionUtils.isNonBlankData(row.getCellValue(cellIndex))) {
if(cell != null){ row.setCell(cellIndex, new Cell(object, null));
if(project.rows.get(i).cells.get(0).value == subject){ object = null;
candidateMergeRowIndex = i; break;
} }
} }
}
columnIndex = project.columnModel.getColumnIndexByName(predicate); if (object != null) {
Row row = new Row(project.columnModel.getMaxCellIndex() + 1);
rows.add(row);
if(candidateMergeRowIndex > -1){ row.setCell(cellIndex, new Cell(object, null));
Cell cell = project.rows.get(candidateMergeRowIndex).cells.get(columnIndex); }
if(cell == null){ } else {
//empty, so merge in this value List<Row> rows = new ArrayList<Row>();
MergeWithRow(project, candidateMergeRowIndex, columnIndex, object); subjectToRows.put(subject, rows);
}else{
//can't overwrite existing, so add new dependent row Row row = new Row(project.columnModel.getMaxCellIndex() + 1);
AddNewDependentRow(project, subject, candidateMergeRowIndex, columnIndex, object); //TODO group to original row. rows.add(row);
}
}else{ row.setCell(subjectColumn.getCellIndex(), new Cell(subject, null));
AddNewRow(project, subject, columnIndex, object); row.setCell(cellIndex, new Cell(object, null));
} }
} }
for (Entry<String, List<Row>> entry : subjectToRows.entrySet()) {
project.rows.addAll(entry.getValue());
}
} finally { } finally {
triples.iterator().close(); triples.iterator().close();
} }
} }
protected void AddNewColumn(Project project, String predicate, String subject){
int numberOfColumns = project.columnModel.columns.size();
project.columnModel.columns.add(numberOfColumns, new Column(numberOfColumns, predicate));
project.columnModel.setMaxCellIndex(numberOfColumns);
project.columnModel.update();
//update existing rows with new column
for(int i = 0; i < project.rows.size(); i++){
project.rows.get(i).cells.add(numberOfColumns, null);
}
}
protected void MergeWithRow(Project project, int candidateMergeRowIndex, int columnIndex, String object){
project.rows.get(candidateMergeRowIndex).setCell(columnIndex, new Cell(object, null));
}
protected void AddNewDependentRow(Project project, String subject, int candidateMergeRowIndex, int columnIndex, String object){
Row row = AddNewRow(project, subject, columnIndex, object);
Project.setRowDependency(project, row, columnIndex, candidateMergeRowIndex, project.columnModel.getKeyColumnIndex());
row.cells.set(project.columnModel.getKeyColumnIndex(), null); //the subject can now be null, as the dependencies are set
}
protected Row AddNewRow(Project project, String subject, int columnIndex, String object){
int numberOfColumns = project.columnModel.columns.size();
//add subject
Row row = new Row(numberOfColumns);
row.setCell(0, new Cell(subject, null));
//add object to a row
row.setCell(columnIndex, new Cell(object, null));
project.rows.add(row);
return row;
}
@Override @Override
public void read(InputStream inputStream, Project project, Properties options) throws Exception { public void read(InputStream inputStream, Project project, Properties options) throws Exception {
// TODO // TODO

2
src/main/java/com/metaweb/gridworks/model/ColumnModel.java
View File

@ -98,7 +98,7 @@ public class ColumnModel implements Jsonizable {
} }
column.setName(name); column.setName(name);
columns.add(index, column); columns.add(index < 0 ? columns.size() : index, column);
_nameToColumn.put(name, column); // so the next call can check _nameToColumn.put(name, column); // so the next call can check
} }

137
src/main/java/com/metaweb/gridworks/model/Project.java
View File

@ -9,8 +9,6 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.Writer; import java.io.Writer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
@ -24,7 +22,6 @@ import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.Gridworks; import com.metaweb.gridworks.Gridworks;
import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata; import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.history.History; import com.metaweb.gridworks.history.History;
import com.metaweb.gridworks.process.ProcessManager; import com.metaweb.gridworks.process.ProcessManager;
import com.metaweb.gridworks.protograph.Protograph; import com.metaweb.gridworks.protograph.Protograph;
@ -33,12 +30,12 @@ import com.metaweb.gridworks.util.Pool;
public class Project { public class Project {
final public long id; final public long id;
final public ColumnModel columnModel = new ColumnModel();
final public List<Row> rows = new ArrayList<Row>(); final public List<Row> rows = new ArrayList<Row>();
final public History history; final public ColumnModel columnModel = new ColumnModel();
final public RecordModel recordModel = new RecordModel();
public Protograph protograph; public Protograph protograph;
final public History history;
transient public ProcessManager processManager = new ProcessManager(); transient public ProcessManager processManager = new ProcessManager();
transient public Date lastSave = new Date(); transient public Date lastSave = new Date();
@ -246,136 +243,16 @@ public class Project {
"Loaded project {} from disk in {} sec(s)",id,Long.toString((System.currentTimeMillis() - start) / 1000) "Loaded project {} from disk in {} sec(s)",id,Long.toString((System.currentTimeMillis() - start) / 1000)
); );
project.recomputeRowContextDependencies(); project.update();
return project; return project;
} }
public void update() {
static protected class Group { columnModel.update();
int[] cellIndices; recordModel.update(this);
int keyCellIndex;
} }
synchronized public void recomputeRowContextDependencies() {
List<Group> keyedGroups = new ArrayList<Group>();
addRootKeyedGroup(keyedGroups);
for (ColumnGroup group : columnModel.columnGroups) {
if (group.keyColumnIndex >= 0) {
Group keyedGroup = new Group();
keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex();
keyedGroup.cellIndices = new int[group.columnSpan - 1];
int c = 0;
for (int i = 0; i < group.columnSpan; i++) {
int columnIndex = group.startColumnIndex + i;
if (columnIndex != group.keyColumnIndex) {
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
keyedGroup.cellIndices[c++] = cellIndex;
}
}
keyedGroups.add(keyedGroup);
}
}
Collections.sort(keyedGroups, new Comparator<Group>() {
public int compare(Group o1, Group o2) {
return o2.cellIndices.length - o1.cellIndices.length; // larger groups first
}
});
int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()];
for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) {
lastNonBlankRowsByGroup[i] = -1;
}
int rowCount = rows.size();
int groupCount = keyedGroups.size();
int recordIndex = 0;
for (int r = 0; r < rowCount; r++) {
Row row = rows.get(r);
row.contextRows = null;
row.contextRowSlots = null;
row.contextCellSlots = null;
for (int g = 0; g < groupCount; g++) {
Group group = keyedGroups.get(g);
if (!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
int contextRowIndex = lastNonBlankRowsByGroup[g];
if (contextRowIndex >= 0) {
for (int dependentCellIndex : group.cellIndices) {
if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) {
setRowDependency(
this,
row,
dependentCellIndex,
contextRowIndex,
group.keyCellIndex
);
}
}
}
} else {
lastNonBlankRowsByGroup[g] = r;
}
}
if (row.contextRowSlots != null && row.contextRowSlots.length > 0) {
row.recordIndex = -1;
row.contextRows = new ArrayList<Integer>();
for (int index : row.contextRowSlots) {
if (index >= 0) {
row.contextRows.add(index);
}
}
Collections.sort(row.contextRows);
columnModel._hasDependentRows = true;
} else {
row.recordIndex = recordIndex++;
}
}
}
protected void addRootKeyedGroup(List<Group> keyedGroups) {
int count = columnModel.getMaxCellIndex() + 1;
if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) {
Group rootKeyedGroup = new Group();
rootKeyedGroup.cellIndices = new int[count - 1];
rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex();
for (int i = 0; i < count; i++) {
if (i < rootKeyedGroup.keyCellIndex) {
rootKeyedGroup.cellIndices[i] = i;
} else if (i > rootKeyedGroup.keyCellIndex) {
rootKeyedGroup.cellIndices[i - 1] = i;
}
}
keyedGroups.add(rootKeyedGroup);
}
}
public static void setRowDependency(Project project, Row row, int cellIndex, int contextRowIndex, int contextCellIndex) {
int count = project.columnModel.getMaxCellIndex() + 1;
if (row.contextRowSlots == null || row.contextCellSlots == null) {
row.contextRowSlots = new int[count];
row.contextCellSlots = new int[count];
for (int i = 0; i < count; i++) {
row.contextRowSlots[i] = -1;
row.contextCellSlots[i] = -1;
}
}
row.contextRowSlots[cellIndex] = contextRowIndex;
row.contextCellSlots[cellIndex] = contextCellIndex;
}
//wrapper of processManager variable to allow unit testing //wrapper of processManager variable to allow unit testing
//TODO make the processManager variable private, and force all calls through this method //TODO make the processManager variable private, and force all calls through this method

17
src/main/java/com/metaweb/gridworks/model/Record.java Normal file
View File

@ -0,0 +1,17 @@
package com.metaweb.gridworks.model;
public class Record {
final public int fromRowIndex;
final public int toRowIndex;
final public int recordIndex;
public Record(
int fromRowIndex,
int toRowIndex,
int recordIndex
) {
this.fromRowIndex = fromRowIndex;
this.toRowIndex = toRowIndex;
this.recordIndex = recordIndex;
}
}

207
src/main/java/com/metaweb/gridworks/model/RecordModel.java Normal file
View File

@ -0,0 +1,207 @@
package com.metaweb.gridworks.model;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import com.metaweb.gridworks.expr.ExpressionUtils;
public class RecordModel {
final static public class CellDependency {
final public int rowIndex;
final public int cellIndex;
public CellDependency(int rowIndex, int cellIndex) {
this.rowIndex = rowIndex;
this.cellIndex = cellIndex;
}
}
final static public class RowDependency {
public int recordIndex;
public CellDependency[] cellDependencies;
public List<Integer> contextRows;
}
protected List<RowDependency> _rowDependencies;
protected List<Record> _records;
public RowDependency getRowDependency(int rowIndex) {
return _rowDependencies != null && rowIndex >= 0 && rowIndex < _rowDependencies.size() ?
_rowDependencies.get(rowIndex) : null;
}
public Record getRecord(int recordIndex) {
return _records != null && recordIndex >= 0 && recordIndex < _records.size() ?
_records.get(recordIndex) : null;
}
public Record getRecordOfRow(int rowIndex) {
RowDependency rd = getRowDependency(rowIndex);
if (rd != null) {
if (rd.recordIndex < 0) {
rd = getRowDependency(rd.contextRows.get(0));
}
return getRecord(rd.recordIndex);
}
return null;
}
static protected class KeyedGroup {
int[] cellIndices;
int keyCellIndex;
}
synchronized public void update(Project project) {
synchronized (project) {
List<Row> rows = project.rows;
int rowCount = rows.size();
ColumnModel columnModel = project.columnModel;
List<KeyedGroup> keyedGroups = computeKeyedGroups(columnModel);
int groupCount = keyedGroups.size();
int[] lastNonBlankRowsByGroup = new int[keyedGroups.size()];
for (int i = 0; i < lastNonBlankRowsByGroup.length; i++) {
lastNonBlankRowsByGroup[i] = -1;
}
_rowDependencies = new ArrayList<RowDependency>(rowCount);
int recordIndex = 0;
for (int r = 0; r < rowCount; r++) {
Row row = rows.get(r);
RowDependency rowDependency = new RowDependency();
for (int g = 0; g < groupCount; g++) {
KeyedGroup group = keyedGroups.get(g);
if (!ExpressionUtils.isNonBlankData(row.getCellValue(group.keyCellIndex))) {
int contextRowIndex = lastNonBlankRowsByGroup[g];
if (contextRowIndex >= 0) {
for (int dependentCellIndex : group.cellIndices) {
if (ExpressionUtils.isNonBlankData(row.getCellValue(dependentCellIndex))) {
setRowDependency(
project,
rowDependency,
dependentCellIndex,
contextRowIndex,
group.keyCellIndex
);
}
}
}
} else {
lastNonBlankRowsByGroup[g] = r;
}
}
if (rowDependency.cellDependencies != null && rowDependency.cellDependencies.length > 0) {
rowDependency.recordIndex = -1;
rowDependency.contextRows = new ArrayList<Integer>();
for (CellDependency cd : rowDependency.cellDependencies) {
if (cd != null) {
rowDependency.contextRows.add(cd.rowIndex);
}
}
Collections.sort(rowDependency.contextRows);
columnModel._hasDependentRows = true;
} else {
rowDependency.recordIndex = recordIndex++;
}
_rowDependencies.add(rowDependency);
}
_records = new ArrayList<Record>(recordIndex);
if (recordIndex > 0) {
recordIndex = 0;
int recordRowIndex = 0;
for (int r = 1; r < rowCount; r++) {
RowDependency rd = _rowDependencies.get(r);
if (rd.recordIndex >= 0) {
_records.add(new Record(recordRowIndex, r, recordIndex++));
recordIndex = rd.recordIndex;
recordRowIndex = r;
}
}
_records.add(new Record(recordRowIndex, rowCount, recordIndex++));
}
}
}
protected List<KeyedGroup> computeKeyedGroups(ColumnModel columnModel) {
List<KeyedGroup> keyedGroups = new ArrayList<KeyedGroup>();
addRootKeyedGroup(columnModel, keyedGroups);
for (ColumnGroup group : columnModel.columnGroups) {
if (group.keyColumnIndex >= 0) {
KeyedGroup keyedGroup = new KeyedGroup();
keyedGroup.keyCellIndex = columnModel.columns.get(group.keyColumnIndex).getCellIndex();
keyedGroup.cellIndices = new int[group.columnSpan - 1];
int c = 0;
for (int i = 0; i < group.columnSpan; i++) {
int columnIndex = group.startColumnIndex + i;
if (columnIndex != group.keyColumnIndex) {
int cellIndex = columnModel.columns.get(columnIndex).getCellIndex();
keyedGroup.cellIndices[c++] = cellIndex;
}
}
keyedGroups.add(keyedGroup);
}
}
Collections.sort(keyedGroups, new Comparator<KeyedGroup>() {
public int compare(KeyedGroup o1, KeyedGroup o2) {
return o2.cellIndices.length - o1.cellIndices.length; // larger groups first
}
});
return keyedGroups;
}
protected void addRootKeyedGroup(ColumnModel columnModel, List<KeyedGroup> keyedGroups) {
int count = columnModel.getMaxCellIndex() + 1;
if (count > 0 && columnModel.getKeyColumnIndex() < columnModel.columns.size()) {
KeyedGroup rootKeyedGroup = new KeyedGroup();
rootKeyedGroup.cellIndices = new int[count - 1];
rootKeyedGroup.keyCellIndex = columnModel.columns.get(columnModel.getKeyColumnIndex()).getCellIndex();
for (int i = 0; i < count; i++) {
if (i < rootKeyedGroup.keyCellIndex) {
rootKeyedGroup.cellIndices[i] = i;
} else if (i > rootKeyedGroup.keyCellIndex) {
rootKeyedGroup.cellIndices[i - 1] = i;
}
}
keyedGroups.add(rootKeyedGroup);
}
}
protected void setRowDependency(
Project project,
RowDependency rowDependency,
int cellIndex,
int contextRowIndex,
int contextCellIndex
) {
if (rowDependency.cellDependencies == null) {
int count = project.columnModel.getMaxCellIndex() + 1;
rowDependency.cellDependencies = new CellDependency[count];
}
rowDependency.cellDependencies[cellIndex] =
new CellDependency(contextRowIndex, contextCellIndex);
}
}

19
src/main/java/com/metaweb/gridworks/model/Row.java
View File

@ -15,18 +15,19 @@ import org.json.JSONWriter;
import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.expr.CellTuple; import com.metaweb.gridworks.expr.CellTuple;
import com.metaweb.gridworks.expr.HasFields; import com.metaweb.gridworks.expr.HasFields;
import com.metaweb.gridworks.model.RecordModel.RowDependency;
import com.metaweb.gridworks.util.Pool; import com.metaweb.gridworks.util.Pool;
public class Row implements HasFields, Jsonizable { public class Row implements HasFields, Jsonizable {
public boolean flagged; public boolean flagged;
public boolean starred; public boolean starred;
final public List<Cell> cells; final public List<Cell> cells;
/*
transient public int recordIndex = -1; // -1 for rows that are not main record rows transient public int recordIndex = -1; // -1 for rows that are not main record rows
transient public List<Integer> contextRows; transient public List<Integer> contextRows;
transient public int[] contextRowSlots; transient public int[] contextRowSlots;
transient public int[] contextCellSlots; transient public int[] contextCellSlots;
*/
private static final String FLAGGED = "flagged"; private static final String FLAGGED = "flagged";
private static final String STARRED = "starred"; private static final String STARRED = "starred";
@ -129,13 +130,17 @@ public class Row implements HasFields, Jsonizable {
writer.endArray(); writer.endArray();
if (!"save".equals(options.getProperty("mode"))) { if (!"save".equals(options.getProperty("mode"))) {
if (recordIndex >= 0) { if (options.containsKey("rowIndex")) {
writer.key("j"); writer.value(recordIndex); int rowIndex = (Integer) options.get("rowIndex");
writer.key("i"); writer.value(rowIndex);
Project project = (Project) options.get("project");
RowDependency rd = project.recordModel.getRowDependency(rowIndex);
if (rd.recordIndex >= 0) {
writer.key("j"); writer.value(rd.recordIndex);
}
} }
if (options.containsKey("rowIndex")) {
writer.key("i"); writer.value(options.get("rowIndex"));
}
if (options.containsKey("extra")) { if (options.containsKey("extra")) {
Properties extra = (Properties) options.get("extra"); Properties extra = (Properties) options.get("extra");
if (extra != null) { if (extra != null) {

6
src/main/java/com/metaweb/gridworks/model/changes/ColumnAdditionChange.java
View File

@ -42,8 +42,7 @@ public class ColumnAdditionChange extends ColumnChange {
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }
@ -56,8 +55,7 @@ public class ColumnAdditionChange extends ColumnChange {
project.columnModel.columns.remove(_columnIndex); project.columnModel.columns.remove(_columnIndex);
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }

6
src/main/java/com/metaweb/gridworks/model/changes/ColumnRemovalChange.java
View File

@ -39,8 +39,7 @@ public class ColumnRemovalChange extends ColumnChange {
row.setCell(cellIndex, null); row.setCell(cellIndex, null);
} }
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }
@ -53,8 +52,7 @@ public class ColumnRemovalChange extends ColumnChange {
project.rows.get(cell.row).cells.set(cellIndex, cell.cell); project.rows.get(cell.row).cells.set(cellIndex, cell.cell);
} }
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }

6
src/main/java/com/metaweb/gridworks/model/changes/ColumnSplitChange.java
View File

@ -139,8 +139,7 @@ public class ColumnSplitChange implements Change {
project.columnModel.columns.remove(_columnIndex); project.columnModel.columns.remove(_columnIndex);
} }
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }
@ -161,8 +160,7 @@ public class ColumnSplitChange implements Change {
project.columnModel.columns.remove(_columnIndex + 1); project.columnModel.columns.remove(_columnIndex + 1);
} }
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }

6
src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java
View File

@ -183,8 +183,7 @@ public class DataExtensionChange implements Change {
} }
} }
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }
@ -234,8 +233,7 @@ public class DataExtensionChange implements Change {
project.columnModel.columns.remove(_columnInsertIndex); project.columnModel.columns.remove(_columnInsertIndex);
} }
project.columnModel.update(); project.update();
project.recomputeRowContextDependencies();
} }
} }

4
src/main/java/com/metaweb/gridworks/model/changes/MassCellChange.java
View File

@ -62,7 +62,7 @@ public class MassCellChange implements Change {
} }
if (_updateRowContextDependencies) { if (_updateRowContextDependencies) {
project.recomputeRowContextDependencies(); project.update();
} }
} }
} }
@ -81,7 +81,7 @@ public class MassCellChange implements Change {
} }
if (_updateRowContextDependencies) { if (_updateRowContextDependencies) {
project.recomputeRowContextDependencies(); project.update();
} }
} }
} }

4
src/main/java/com/metaweb/gridworks/model/changes/MassChange.java
View File

@ -28,7 +28,7 @@ public class MassChange implements Change {
} }
if (_updateRowContextDependencies) { if (_updateRowContextDependencies) {
project.recomputeRowContextDependencies(); project.update();
} }
} }
} }
@ -40,7 +40,7 @@ public class MassChange implements Change {
} }
if (_updateRowContextDependencies) { if (_updateRowContextDependencies) {
project.recomputeRowContextDependencies(); project.update();
} }
} }
} }

4
src/main/java/com/metaweb/gridworks/model/changes/MassRowChange.java
View File

@ -26,7 +26,7 @@ public class MassRowChange implements Change {
project.rows.clear(); project.rows.clear();
project.rows.addAll(_newRows); project.rows.addAll(_newRows);
project.recomputeRowContextDependencies(); project.update();
} }
} }
@ -35,7 +35,7 @@ public class MassRowChange implements Change {
project.rows.clear(); project.rows.clear();
project.rows.addAll(_oldRows); project.rows.addAll(_oldRows);
project.recomputeRowContextDependencies(); project.update();
} }
} }

4
src/main/java/com/metaweb/gridworks/model/changes/RowRemovalChange.java
View File

@ -36,7 +36,7 @@ public class RowRemovalChange implements Change {
offset--; offset--;
} }
project.recomputeRowContextDependencies(); project.update();
} }
} }
@ -51,7 +51,7 @@ public class RowRemovalChange implements Change {
project.rows.add(index, row); project.rows.add(index, row);
} }
project.recomputeRowContextDependencies(); project.update();
} }
} }

7
src/main/java/com/metaweb/gridworks/model/recon/HeuristicReconConfig.java
View File

@ -23,6 +23,7 @@ import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.ReconCandidate; import com.metaweb.gridworks.model.ReconCandidate;
import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.Recon.Judgment; import com.metaweb.gridworks.model.Recon.Judgment;
import com.metaweb.gridworks.model.RecordModel.RowDependency;
import com.metaweb.gridworks.protograph.FreebaseProperty; import com.metaweb.gridworks.protograph.FreebaseProperty;
import com.metaweb.gridworks.util.ParsingUtilities; import com.metaweb.gridworks.util.ParsingUtilities;
@ -156,8 +157,10 @@ public class HeuristicReconConfig extends ReconConfig {
Cell cell2 = row.getCell(detailCellIndex); Cell cell2 = row.getCell(detailCellIndex);
if (cell2 == null || !ExpressionUtils.isNonBlankData(cell2.value)) { if (cell2 == null || !ExpressionUtils.isNonBlankData(cell2.value)) {
int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex(); int cellIndex = project.columnModel.getColumnByName(columnName).getCellIndex();
if (row.contextRowSlots != null && cellIndex < row.contextRowSlots.length) {
int contextRowIndex = row.contextRowSlots[cellIndex]; RowDependency rd = project.recordModel.getRowDependency(rowIndex);
if (rd != null && rd.cellDependencies != null) {
int contextRowIndex = rd.cellDependencies[cellIndex].rowIndex;
if (contextRowIndex >= 0 && contextRowIndex < project.rows.size()) { if (contextRowIndex >= 0 && contextRowIndex < project.rows.size()) {
Row row2 = project.rows.get(contextRowIndex); Row row2 = project.rows.get(contextRowIndex);

24
src/main/java/com/metaweb/gridworks/operations/DenormalizeOperation.java
View File

@ -13,6 +13,8 @@ import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Cell; import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.RecordModel.CellDependency;
import com.metaweb.gridworks.model.RecordModel.RowDependency;
import com.metaweb.gridworks.model.changes.MassRowChange; import com.metaweb.gridworks.model.changes.MassRowChange;
public class DenormalizeOperation extends AbstractOperation { public class DenormalizeOperation extends AbstractOperation {
@ -45,19 +47,23 @@ public class DenormalizeOperation extends AbstractOperation {
Row oldRow = oldRows.get(r); Row oldRow = oldRows.get(r);
Row newRow = null; Row newRow = null;
if (oldRow.contextCellSlots != null && oldRow.contextRowSlots != null) { RowDependency rd = project.recordModel.getRowDependency(r);
if (rd.cellDependencies != null) {
newRow = oldRow.dup(); newRow = oldRow.dup();
for (int c = 0; c < oldRow.contextCellSlots.length && c < oldRow.contextRowSlots.length; c++) { for (int c = 0; c < rd.cellDependencies.length; c++) {
int contextRowIndex = oldRow.contextRowSlots[c]; CellDependency cd = rd.cellDependencies[c];
int contextCellIndex = oldRow.contextCellSlots[c]; if (cd != null) {
int contextRowIndex = cd.rowIndex;
int contextCellIndex = cd.cellIndex;
if (contextRowIndex >= 0 && contextRowIndex < oldRows.size()) { if (contextRowIndex >= 0 && contextRowIndex < oldRows.size()) {
Row contextRow = oldRows.get(contextRowIndex); Row contextRow = oldRows.get(contextRowIndex);
Cell contextCell = contextRow.getCell(contextCellIndex); Cell contextCell = contextRow.getCell(contextCellIndex);
newRow.setCell(contextCellIndex, contextCell); newRow.setCell(contextCellIndex, contextCell);
} }
}
} }
} }

56
tests/java/src/com/metaweb/gridworks/tests/importers/RdfTripleImporterTests.java
View File

@ -30,13 +30,14 @@ public class RdfTripleImporterTests {
options.put("base-url", "http://rdf.freebase.com"); options.put("base-url", "http://rdf.freebase.com");
} }
@Test @Test(enabled=false)
public void CanParseSingleLineTriple(){ public void CanParseSingleLineTriple(){
String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>."; String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/music.artist.album> <http://rdf.freebase.com/ns/en.blood_on_the_tracks>.";
StringReader reader = new StringReader(sampleRdf); StringReader reader = new StringReader(sampleRdf);
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, options);
project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -59,6 +60,7 @@ public class RdfTripleImporterTests {
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, options);
project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -77,18 +79,18 @@ public class RdfTripleImporterTests {
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks"); Assert.assertEquals(project.rows.get(0).cells.get(1).value, "http://rdf.freebase.com/ns/en.blood_on_the_tracks");
//row1 //row1
Assert.assertEquals(project.rows.get(2).cells.size(), 2); Assert.assertEquals(project.rows.get(1).cells.size(), 2);
Assert.assertNull(project.rows.get(1).cells.get(0)); Assert.assertNull(project.rows.get(1).cells.get(0));
Assert.assertEquals(project.rows.get(1).contextRowSlots[1], 0);
Assert.assertEquals(project.rows.get(1).contextCellSlots[1], 0);
Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); //NB triples aren't created in order they were input
Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
//row2 //row2
Assert.assertEquals(project.rows.get(2).cells.size(), 2); Assert.assertEquals(project.rows.get(2).cells.size(), 2);
Assert.assertEquals(project.rows.get(2).contextRowSlots[1], 0);
Assert.assertEquals(project.rows.get(2).contextCellSlots[1], 0);
Assert.assertNull(project.rows.get(2).cells.get(0)); Assert.assertNull(project.rows.get(2).cells.get(0));
Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input Assert.assertEquals(project.rows.get(2).cells.get(1).value, "http://rdf.freebase.com/ns/en.under_the_red_sky"); //NB triples aren't created in order they were input
Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].rowIndex, 0);
Assert.assertEquals(project.recordModel.getRowDependency(2).cellDependencies[1].cellIndex, 0);
} }
@Test @Test
@ -100,6 +102,7 @@ public class RdfTripleImporterTests {
try { try {
SUT.read(reader, project, options); SUT.read(reader, project, options);
project.update();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); Assert.fail();
} }
@ -120,30 +123,31 @@ public class RdfTripleImporterTests {
Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock"); Assert.assertEquals(project.rows.get(0).cells.get(2).value, "http://rdf.freebase.com/ns/en.folk_rock");
//row1 //row1
Assert.assertEquals(project.rows.get(1).cells.size(), 3); Assert.assertEquals(project.rows.get(1).cells.size(), 2);
Assert.assertEquals(project.rows.get(1).contextRowSlots[1], 0);
Assert.assertEquals(project.rows.get(1).contextCellSlots[1], 0);
Assert.assertNull(project.rows.get(1).cells.get(0)); Assert.assertNull(project.rows.get(1).cells.get(0));
Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home"); Assert.assertEquals(project.rows.get(1).cells.get(1).value, "http://rdf.freebase.com/ns/en.bringing_it_all_back_home");
Assert.assertNull(project.rows.get(1).cells.get(2)); Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].rowIndex, 0);
Assert.assertEquals(project.recordModel.getRowDependency(1).cellDependencies[1].cellIndex, 0);
} }
@Test
public void CanParseTripleWithValue(){
String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/common.topic.alias> \"Robert Zimmerman\"@en.";
StringReader reader = new StringReader(sampleRdf);
try { @Test
SUT.read(reader, project, options); public void CanParseTripleWithValue(){
} catch (Exception e) { String sampleRdf = "<http://rdf.freebase.com/ns/en.bob_dylan> <http://rdf.freebase.com/ns/common.topic.alias> \"Robert Zimmerman\"@en.";
Assert.fail(); StringReader reader = new StringReader(sampleRdf);
}
Assert.assertEquals(project.columnModel.columns.size(), 2); try {
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject"); SUT.read(reader, project, options);
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias"); project.update();
Assert.assertEquals(project.rows.size(), 1); } catch (Exception e) {
Assert.assertEquals(project.rows.get(0).cells.size(), 2); Assert.fail();
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en");
} }
Assert.assertEquals(project.columnModel.columns.size(), 2);
Assert.assertEquals(project.columnModel.columns.get(0).getName(), "subject");
Assert.assertEquals(project.columnModel.columns.get(1).getName(), "http://rdf.freebase.com/ns/common.topic.alias");
Assert.assertEquals(project.rows.size(), 1);
Assert.assertEquals(project.rows.get(0).cells.size(), 2);
Assert.assertEquals(project.rows.get(0).cells.get(0).value, "http://rdf.freebase.com/ns/en.bob_dylan");
Assert.assertEquals(project.rows.get(0).cells.get(1).value, "\"Robert Zimmerman\"@en");
}
} }