s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

More row/record model refactoring work. Everything should still be working almost as before, except contextual rows are not shown in row-based mode.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@823 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-05-20 00:13:19 +00:00
parent ec7ac81246
commit 28ca652dea
47 changed files with 930 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRecords.java Normal file
View File

@ -0,0 +1,41 @@
package com.metaweb.gridworks.browsing;
import java.util.LinkedList;
import java.util.List;
import com.metaweb.gridworks.browsing.filters.RecordFilter;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
/**
* Encapsulate logic for visiting records that match all given record filters.
*/
public class ConjunctiveFilteredRecords implements FilteredRecords {
final protected List<RecordFilter> _recordFilters = new LinkedList<RecordFilter>();
public void add(RecordFilter recordFilter) {
_recordFilters.add(recordFilter);
}
@Override
public void accept(Project project, RecordVisitor visitor) {
int c = project.recordModel.getRecordCount();
for (int r = 0; r < c; r++) {
Record record = project.recordModel.getRecord(r);
if (matchRecord(project, record)) {
if (visitor.visit(project, record)) {
return;
}
}
}
}
protected boolean matchRecord(Project project, Record record) {
for (RecordFilter recordFilter : _recordFilters) {
if (!recordFilter.filterRecord(project, record)) {
return false;
}
}
return true;
}
}

61
src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java
View File

@ -6,7 +6,6 @@ import java.util.List;
import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.RecordModel.RowDependency;
/**
* Encapsulate logic for visiting rows that match all give row filters. Also visit
@ -14,77 +13,23 @@ import com.metaweb.gridworks.model.RecordModel.RowDependency;
*/
public class ConjunctiveFilteredRows implements FilteredRows {
final protected List<RowFilter> _rowFilters = new LinkedList<RowFilter>();
final protected boolean _includeContextual;
final protected boolean _includeDependent;
public ConjunctiveFilteredRows(boolean includeContextual, boolean includeDependent) {
_includeContextual = includeContextual;
_includeDependent = includeDependent;
}
public void add(RowFilter rowFilter) {
_rowFilters.add(rowFilter);
}
public void accept(Project project, RowVisitor visitor) {
int lastVisitedRowRowIndex = -1;
int lastRecordRowAcceptedRowIndex = -1;
int c = project.rows.size();
for (int rowIndex = 0; rowIndex < c; rowIndex++) {
Row row = project.rows.get(rowIndex);
RowDependency rd = project.recordModel.getRowDependency(rowIndex);
if (matchRow(project, rowIndex, row)) {
if (rd.recordIndex >= 0) {
lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself
}
visitRow(project, visitor, rowIndex, row, rd, lastVisitedRowRowIndex);
lastVisitedRowRowIndex = rowIndex;
} else if (
// this row doesn't match by itself but ...
// we want to include dependent rows
_includeDependent &&
// and this row is a dependent row since it's not a record row
rd.recordIndex < 0 &&
rd.contextRows != null &&
rd.contextRows.size() > 0 &&
rd.contextRows.get(0) == lastRecordRowAcceptedRowIndex
) {
// this row depends on the last previously matched record row,
// so we visit it as well as a dependent row
visitor.visit(project, rowIndex, row, false, true);
lastVisitedRowRowIndex = rowIndex;
visitRow(project, visitor, rowIndex, row);
}
}
}
protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, RowDependency rd, int lastVisitedRow) {
if (_includeContextual && // we need to include any context row and
rd.contextRows != null && // this row itself isn't a context row and
lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row
// that we haven't visited yet
) {
for (int contextRowIndex : rd.contextRows) {
if (contextRowIndex > lastVisitedRow) {
visitor.visit(
project,
contextRowIndex,
project.rows.get(contextRowIndex),
true, // is visited as a context row
false // is not visited as a dependent row
);
lastVisitedRow = contextRowIndex;
}
}
}
visitor.visit(project, rowIndex, row, false, false);
protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row) {
visitor.visit(project, rowIndex, row);
}
protected boolean matchRow(Project project, int rowIndex, Row row) {

121
src/main/java/com/metaweb/gridworks/browsing/Engine.java
View File

@ -15,42 +15,102 @@ import com.metaweb.gridworks.browsing.facets.ListFacet;
import com.metaweb.gridworks.browsing.facets.RangeFacet;
import com.metaweb.gridworks.browsing.facets.ScatterplotFacet;
import com.metaweb.gridworks.browsing.facets.TextSearchFacet;
import com.metaweb.gridworks.browsing.filters.RecordFilter;
import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
/**
* Faceted browsing engine.
*/
public class Engine implements Jsonizable {
static public enum Mode {
RowBased,
RecordBased
}
public final static String INCLUDE_DEPENDENT = "includeDependent";
public final static String MODE = "mode";
public final static String MODE_ROW_BASED = "row-based";
public final static String MODE_RECORD_BASED = "record-based";
protected Project _project;
protected List<Facet> _facets = new LinkedList<Facet>();
protected boolean _includeDependent;
public final static String INCLUDE_DEPENDENT = "includeDependent";
protected Mode _mode = Mode.RowBased;
public Engine(Project project) {
_project = project;
}
public FilteredRows getAllRows() {
return new ConjunctiveFilteredRows(false, false);
public Mode getMode() {
return _mode;
}
public FilteredRows getAllFilteredRows(boolean includeContextual) {
return getFilteredRows(null, includeContextual);
public FilteredRows getAllRows() {
return new FilteredRows() {
@Override
public void accept(Project project, RowVisitor visitor) {
int c = project.rows.size();
for (int rowIndex = 0; rowIndex < c; rowIndex++) {
Row row = project.rows.get(rowIndex);
visitor.visit(project, rowIndex, row);
}
}
};
}
public FilteredRows getAllFilteredRows() {
return getFilteredRows(null);
}
public FilteredRows getFilteredRows(Facet except, boolean includeContextual) {
ConjunctiveFilteredRows cfr = new ConjunctiveFilteredRows(includeContextual, _includeDependent);
for (Facet facet : _facets) {
if (facet != except) {
RowFilter rowFilter = facet.getRowFilter();
if (rowFilter != null) {
cfr.add(rowFilter);
public FilteredRows getFilteredRows(Facet except) {
if (_mode == Mode.RecordBased) {
return new FilteredRecordsAsFilteredRows(getFilteredRecords(except));
} else if (_mode == Mode.RowBased) {
ConjunctiveFilteredRows cfr = new ConjunctiveFilteredRows();
for (Facet facet : _facets) {
if (facet != except) {
RowFilter rowFilter = facet.getRowFilter();
if (rowFilter != null) {
cfr.add(rowFilter);
}
}
}
return cfr;
}
throw new InternalError("Unknown mode.");
}
public FilteredRecords getAllRecords() {
return new FilteredRecords() {
@Override
public void accept(Project project, RecordVisitor visitor) {
int c = project.recordModel.getRecordCount();
for (int r = 0; r < c; r++) {
visitor.visit(project, project.recordModel.getRecord(r));
}
}
};
}
public FilteredRecords getFilteredRecords() {
return getFilteredRecords(null);
}
public FilteredRecords getFilteredRecords(Facet except) {
if (_mode == Mode.RecordBased) {
ConjunctiveFilteredRecords cfr = new ConjunctiveFilteredRecords();
for (Facet facet : _facets) {
if (facet != except) {
RecordFilter recordFilter = facet.getRecordFilter();
if (recordFilter != null) {
cfr.add(recordFilter);
}
}
}
}
return cfr;
return cfr;
}
throw new InternalError("This method should not be called when the engine is not in record mode.");
}
public void initializeFromJSON(JSONObject o) throws Exception {
@ -84,17 +144,32 @@ public class Engine implements Jsonizable {
}
}
// for backward compatibility
if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) {
_includeDependent = o.getBoolean(INCLUDE_DEPENDENT);
_mode = o.getBoolean(INCLUDE_DEPENDENT) ? Mode.RecordBased : Mode.RowBased;
}
if (o.has(MODE) && !o.isNull(MODE)) {
_mode = MODE_ROW_BASED.equals(o.getString(MODE)) ? Mode.RowBased : Mode.RecordBased;
}
}
public void computeFacets() throws JSONException {
for (Facet facet : _facets) {
FilteredRows filteredRows = getFilteredRows(facet, false);
facet.computeChoices(_project, filteredRows);
}
if (_mode == Mode.RowBased) {
for (Facet facet : _facets) {
FilteredRows filteredRows = getFilteredRows(facet);
facet.computeChoices(_project, filteredRows);
}
} else if (_mode == Mode.RecordBased) {
for (Facet facet : _facets) {
FilteredRecords filteredRecords = getFilteredRecords(facet);
facet.computeChoices(_project, filteredRecords);
}
} else {
throw new InternalError("Unknown mode.");
}
}
public void write(JSONWriter writer, Properties options)
@ -107,7 +182,7 @@ public class Engine implements Jsonizable {
facet.write(writer, options);
}
writer.endArray();
writer.key(INCLUDE_DEPENDENT); writer.value(_includeDependent);
writer.key(MODE); writer.value(_mode == Mode.RowBased ? MODE_ROW_BASED : MODE_RECORD_BASED);
writer.endObject();
}
}

18
src/main/java/com/metaweb/gridworks/browsing/FilteredRecords.java Normal file
View File

@ -0,0 +1,18 @@
package com.metaweb.gridworks.browsing;
import com.metaweb.gridworks.model.Project;
/**
* Interface for anything that can decide which records match and which don't
* based on some particular criteria.
*/
public interface FilteredRecords {
/**
* Go through the records of the given project, determine which match and which don't,
* and call visitor.visit() on those that match
*
* @param project
* @param visitor
*/
public void accept(Project project, RecordVisitor visitor);
}

17
src/main/java/com/metaweb/gridworks/browsing/FilteredRecordsAsFilteredRows.java Normal file
View File

@ -0,0 +1,17 @@
package com.metaweb.gridworks.browsing;
import com.metaweb.gridworks.model.Project;
public class FilteredRecordsAsFilteredRows implements FilteredRows {
final protected FilteredRecords _filteredRecords;
public FilteredRecordsAsFilteredRows(FilteredRecords filteredRecords) {
_filteredRecords = filteredRecords;
}
@Override
public void accept(Project project, RowVisitor visitor) {
_filteredRecords.accept(project, new RowVisitorAsRecordVisitor(visitor));
}
}

15
src/main/java/com/metaweb/gridworks/browsing/RecordVisitor.java Normal file
View File

@ -0,0 +1,15 @@
package com.metaweb.gridworks.browsing;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
/**
* Interface for visiting records one by one. The records visited are only those that match some
* particular criteria, such as facets' constraints.
*/
public interface RecordVisitor {
public boolean visit(
Project project,
Record record
);
}

10
src/main/java/com/metaweb/gridworks/browsing/RowVisitor.java
View File

@ -5,18 +5,12 @@ import com.metaweb.gridworks.model.Row;
/**
* Interface for visiting rows one by one. The rows visited are only those that match some
* particular criteria, such as facets' constraints, or those that are related to the matched
* rows. The related rows can be those that the matched rows depend on, or those that depend
* on the matched rows.
* particular criteria, such as facets' constraints.
*/
public interface RowVisitor {
public boolean visit(
Project project,
int rowIndex, // zero-based row index
Row row,
boolean contextual, // true if this row is included because it's the context row
// of a matched row, that is, a matched row depends on it
boolean dependent // true if this row is included because it depends on a matched row,
// that is, it depends on a matched row
Row row
);
}

22
src/main/java/com/metaweb/gridworks/browsing/RowVisitorAsRecordVisitor.java Normal file
View File

@ -0,0 +1,22 @@
package com.metaweb.gridworks.browsing;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
public class RowVisitorAsRecordVisitor implements RecordVisitor {
final protected RowVisitor _rowVisitor;
public RowVisitorAsRecordVisitor(RowVisitor rowVisitor) {
_rowVisitor = rowVisitor;
}
@Override
public boolean visit(Project project, Record record) {
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
if (_rowVisitor.visit(project, r, project.rows.get(r))) {
return true;
}
}
return false;
}
}

99
src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNominalRowGrouper.java
View File

@ -6,18 +6,29 @@ import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.browsing.DecoratedValue;
import com.metaweb.gridworks.browsing.RecordVisitor;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
import com.metaweb.gridworks.model.Row;
/**
* Visit matched rows and group them into facet choices based on the values computed
* from a given expression.
*/
public class ExpressionNominalRowGrouper implements RowVisitor {
public class ExpressionNominalRowGrouper implements RowVisitor, RecordVisitor {
static public class IndexedNominalFacetChoice extends NominalFacetChoice {
int _latestIndex;
public IndexedNominalFacetChoice(DecoratedValue decoratedValue, int latestIndex) {
super(decoratedValue);
_latestIndex = latestIndex;
}
}
/*
* Configuration
*/
@ -28,9 +39,11 @@ public class ExpressionNominalRowGrouper implements RowVisitor {
/*
* Computed results
*/
final public Map<Object, NominalFacetChoice> choices = new HashMap<Object, NominalFacetChoice>();
final public Map<Object, IndexedNominalFacetChoice> choices = new HashMap<Object, IndexedNominalFacetChoice>();
public int blankCount = 0;
public int errorCount = 0;
protected boolean hasBlank;
protected boolean hasError;
public ExpressionNominalRowGrouper(Evaluable evaluable, String columnName, int cellIndex) {
_evaluable = evaluable;
@ -38,10 +51,49 @@ public class ExpressionNominalRowGrouper implements RowVisitor {
_cellIndex = cellIndex;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
hasError = false;
hasBlank = false;
Properties bindings = ExpressionUtils.createBindings(project);
visitRow(project, rowIndex, row, bindings, rowIndex);
if (hasError) {
errorCount++;
}
if (hasBlank) {
blankCount++;
}
return false;
}
@Override
public boolean visit(Project project, Record record) {
hasError = false;
hasBlank = false;
Properties bindings = ExpressionUtils.createBindings(project);
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
Row row = project.rows.get(r);
visitRow(project, r, row, bindings, record.recordIndex);
}
if (hasError) {
errorCount++;
}
if (hasBlank) {
blankCount++;
}
return false;
}
protected void visitRow(Project project, int rowIndex, Row row, Properties bindings, int index) {
Cell cell = _cellIndex < 0 ? null : row.getCell(_cellIndex);
Properties bindings = ExpressionUtils.createBindings(project);
ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
Object value = _evaluable.evaluate(bindings);
@ -49,40 +101,43 @@ public class ExpressionNominalRowGrouper implements RowVisitor {
if (value.getClass().isArray()) {
Object[] a = (Object[]) value;
for (Object v : a) {
processValue(v);
processValue(v, rowIndex);
}
return false;
} else if (value instanceof Collection<?>) {
for (Object v : ExpressionUtils.toObjectCollection(value)) {
processValue(v);
processValue(v, rowIndex);
}
return false;
} // else, fall through
} else {
processValue(value, rowIndex);
}
} else {
processValue(value, rowIndex);
}
processValue(value);
return false;
}
protected void processValue(Object value) {
protected void processValue(Object value, int index) {
if (ExpressionUtils.isError(value)) {
errorCount++;
hasError = true;
} else if (ExpressionUtils.isNonBlankData(value)) {
String valueString = value.toString();
String label = value.toString();
IndexedNominalFacetChoice facetChoice = choices.get(valueString);
DecoratedValue dValue = new DecoratedValue(value, label);
if (choices.containsKey(valueString)) {
choices.get(valueString).count++;
if (facetChoice != null) {
if (facetChoice._latestIndex < index) {
facetChoice._latestIndex = index;
facetChoice.count++;
}
} else {
NominalFacetChoice choice = new NominalFacetChoice(dValue);
String label = value.toString();
DecoratedValue dValue = new DecoratedValue(value, label);
IndexedNominalFacetChoice choice =
new IndexedNominalFacetChoice(dValue, index);
choice.count = 1;
choices.put(valueString, choice);
}
} else {
blankCount++;
hasBlank = true;
}
}
}

104
src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java
View File

@ -3,18 +3,20 @@ package com.metaweb.gridworks.browsing.facets;
import java.util.Collection;
import java.util.Properties;
import com.metaweb.gridworks.browsing.RecordVisitor;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
import com.metaweb.gridworks.model.Row;
/**
* Visit matched rows and slot them into bins based on the numbers computed
* from a given expression.
*/
public class ExpressionNumericRowBinner implements RowVisitor {
public class ExpressionNumericRowBinner implements RowVisitor, RecordVisitor {
/*
* Configuration
*/
@ -35,10 +37,10 @@ public class ExpressionNumericRowBinner implements RowVisitor {
/*
* Scratchpad variables
*/
private boolean rowHasError;
private boolean rowHasBlank;
private boolean rowHasNumeric;
private boolean rowHasNonNumeric;
protected boolean hasError;
protected boolean hasBlank;
protected boolean hasNumeric;
protected boolean hasNonNumeric;
public ExpressionNumericRowBinner(Evaluable evaluable, String columnName, int cellIndex, NumericBinIndex index) {
_evaluable = evaluable;
@ -48,78 +50,100 @@ public class ExpressionNumericRowBinner implements RowVisitor {
bins = new int[_index.getBins().length];
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
Cell cell = row.getCell(_cellIndex);
@Override
public boolean visit(Project project, int rowIndex, Row row) {
resetFlags();
Properties bindings = ExpressionUtils.createBindings(project);
processRow(project, rowIndex, row, bindings);
updateCounts();
return false;
}
@Override
public boolean visit(Project project, Record record) {
resetFlags();
Properties bindings = ExpressionUtils.createBindings(project);
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
processRow(project, r, project.rows.get(r), bindings);
}
updateCounts();
return false;
}
protected void resetFlags() {
hasError = false;
hasBlank = false;
hasNumeric = false;
hasNonNumeric = false;
}
protected void updateCounts() {
if (hasError) {
errorCount++;
}
if (hasBlank) {
blankCount++;
}
if (hasNumeric) {
numericCount++;
}
if (hasNonNumeric) {
nonNumericCount++;
}
}
protected void processRow(Project project, int rowIndex, Row row, Properties bindings) {
Cell cell = row.getCell(_cellIndex);
ExpressionUtils.bind(bindings, row, rowIndex, _columnName, cell);
Object value = _evaluable.evaluate(bindings);
rowHasError = false;
rowHasBlank = false;
rowHasNumeric = false;
rowHasNonNumeric = false;
if (value != null) {
if (value.getClass().isArray()) {
Object[] a = (Object[]) value;
for (Object v : a) {
processValue(v);
}
updateCounts();
return false;
return;
} else if (value instanceof Collection<?>) {
for (Object v : ExpressionUtils.toObjectCollection(value)) {
processValue(v);
}
updateCounts();
return false;
return;
} // else, fall through
}
processValue(value);
updateCounts();
return false;
}
protected void updateCounts() {
if (rowHasError) {
errorCount++;
}
if (rowHasBlank) {
blankCount++;
}
if (rowHasNumeric) {
numericCount++;
}
if (rowHasNonNumeric) {
nonNumericCount++;
}
}
protected void processValue(Object value) {
if (ExpressionUtils.isError(value)) {
rowHasError = true;
hasError = true;
} else if (ExpressionUtils.isNonBlankData(value)) {
if (value instanceof Number) {
double d = ((Number) value).doubleValue();
if (!Double.isInfinite(d) && !Double.isNaN(d)) {
rowHasNumeric = true;
hasNumeric = true;
int bin = (int) Math.floor((d - _index.getMin()) / _index.getStep());
if (bin >= 0 && bin < bins.length) { // as a precaution
bins[bin]++;
}
} else {
rowHasError = true;
hasError = true;
}
} else {
rowHasNonNumeric = true;
hasNonNumeric = true;
}
} else {
rowHasBlank = true;
hasBlank = true;
}
}
}

6
src/main/java/com/metaweb/gridworks/browsing/facets/Facet.java
View File

@ -3,7 +3,9 @@ package com.metaweb.gridworks.browsing.facets;
import org.json.JSONObject;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.browsing.FilteredRecords;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.filters.RecordFilter;
import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.model.Project;
@ -13,7 +15,11 @@ import com.metaweb.gridworks.model.Project;
public interface Facet extends Jsonizable {
public RowFilter getRowFilter();
public RecordFilter getRecordFilter();
public void computeChoices(Project project, FilteredRows filteredRows);
public void computeChoices(Project project, FilteredRecords filteredRecords);
public void initializeFromJSON(Project project, JSONObject o) throws Exception;
}

81
src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java
View File

@ -10,8 +10,11 @@ import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.DecoratedValue;
import com.metaweb.gridworks.browsing.FilteredRecords;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.filters.AnyRowRecordFilter;
import com.metaweb.gridworks.browsing.filters.ExpressionEqualRowFilter;
import com.metaweb.gridworks.browsing.filters.RecordFilter;
import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.MetaParser;
@ -54,6 +57,7 @@ public class ListFacet implements Facet {
public ListFacet() {
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
@ -93,6 +97,7 @@ public class ListFacet implements Facet {
writer.endObject();
}
@Override
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
_name = o.getString("name");
_expression = o.getString("expression");
@ -141,6 +146,7 @@ public class ListFacet implements Facet {
_selectError = JSONUtilities.getBoolean(o, "selectError", false);
}
@Override
public RowFilter getRowFilter() {
return
_eval == null ||
@ -156,7 +162,14 @@ public class ListFacet implements Facet {
_selectError,
_invert);
}
@Override
public RecordFilter getRecordFilter() {
RowFilter rowFilter = getRowFilter();
return rowFilter == null ? null : new AnyRowRecordFilter(rowFilter);
}
@Override
public void computeChoices(Project project, FilteredRows filteredRows) {
if (_eval != null && _errorMessage == null) {
ExpressionNominalRowGrouper grouper =
@ -164,35 +177,51 @@ public class ListFacet implements Facet {
filteredRows.accept(project, grouper);
_choices.clear();
_choices.addAll(grouper.choices.values());
for (NominalFacetChoice choice : _selection) {
String valueString = choice.decoratedValue.value.toString();
if (grouper.choices.containsKey(valueString)) {
grouper.choices.get(valueString).selected = true;
} else {
/*
* A selected choice can have zero count if it is selected together
* with other choices, and some other facets' constraints eliminate
* all rows projected to this choice altogether. For example, if you
* select both "car" and "bicycle" in the "type of vehicle" facet, and
* then constrain the "wheels" facet to more than 2, then the "bicycle"
* choice now has zero count even if it's still selected. The grouper
* won't be able to detect the "bicycle" choice, so we need to inject
* that choice into the choice list ourselves.
*/
choice.count = 0;
_choices.add(choice);
}
}
_blankCount = grouper.blankCount;
_errorCount = grouper.errorCount;
postProcessGrouper(grouper);
}
}
@Override
public void computeChoices(Project project, FilteredRecords filteredRecords) {
if (_eval != null && _errorMessage == null) {
ExpressionNominalRowGrouper grouper =
new ExpressionNominalRowGrouper(_eval, _columnName, _cellIndex);
filteredRecords.accept(project, grouper);
postProcessGrouper(grouper);
}
}
protected void postProcessGrouper(ExpressionNominalRowGrouper grouper) {
_choices.clear();
_choices.addAll(grouper.choices.values());
for (NominalFacetChoice choice : _selection) {
String valueString = choice.decoratedValue.value.toString();
if (grouper.choices.containsKey(valueString)) {
grouper.choices.get(valueString).selected = true;
} else {
/*
* A selected choice can have zero count if it is selected together
* with other choices, and some other facets' constraints eliminate
* all rows projected to this choice altogether. For example, if you
* select both "car" and "bicycle" in the "type of vehicle" facet, and
* then constrain the "wheels" facet to more than 2, then the "bicycle"
* choice now has zero count even if it's still selected. The grouper
* won't be able to detect the "bicycle" choice, so we need to inject
* that choice into the choice list ourselves.
*/
choice.count = 0;
_choices.add(choice);
}
}
_blankCount = grouper.blankCount;
_errorCount = grouper.errorCount;
}
protected Object[] createMatches() {
Object[] a = new Object[_selection.size()];
for (int i = 0; i < a.length; i++) {

202
src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java
View File

@ -21,105 +21,34 @@ import com.metaweb.gridworks.model.Row;
* needs to compute the base bins of a numeric range facet, which remain unchanged
* as the user interacts with the facet.
*/
public class NumericBinIndex {
abstract public class NumericBinIndex {
private int _totalValueCount;
private int _numbericValueCount;
private double _min;
private double _max;
private double _step;
private int[] _bins;
protected int _totalValueCount;
protected int _numbericValueCount;
protected double _min;
protected double _max;
protected double _step;
protected int[] _bins;
private int _numericRowCount;
private int _nonNumericRowCount;
private int _blankRowCount;
private int _errorRowCount;
protected int _numericRowCount;
protected int _nonNumericRowCount;
protected int _blankRowCount;
protected int _errorRowCount;
protected boolean _hasError = false;
protected boolean _hasNonNumeric = false;
protected boolean _hasNumeric = false;
protected boolean _hasBlank = false;
abstract protected void iterate(Project project, String columnName, int cellIndex, Evaluable eval, List<Double> allValues);
public NumericBinIndex(Project project, String columnName, int cellIndex, Evaluable eval) {
Properties bindings = ExpressionUtils.createBindings(project);
_min = Double.POSITIVE_INFINITY;
_max = Double.NEGATIVE_INFINITY;
List<Double> allValues = new ArrayList<Double>();
for (int i = 0; i < project.rows.size(); i++) {
Row row = project.rows.get(i);
Cell cell = row.getCell(cellIndex);
ExpressionUtils.bind(bindings, row, i, columnName, cell);
Object value = eval.evaluate(bindings);
boolean rowHasError = false;
boolean rowHasNonNumeric = false;
boolean rowHasNumeric = false;
boolean rowHasBlank = false;
if (ExpressionUtils.isError(value)) {
rowHasError = true;
} else if (ExpressionUtils.isNonBlankData(value)) {
if (value.getClass().isArray()) {
Object[] a = (Object[]) value;
for (Object v : a) {
_totalValueCount++;
if (ExpressionUtils.isError(v)) {
rowHasError = true;
} else if (ExpressionUtils.isNonBlankData(v)) {
if (v instanceof Number) {
rowHasNumeric = true;
processValue(((Number) v).doubleValue(), allValues);
} else {
rowHasNonNumeric = true;
}
} else {
rowHasBlank = true;
}
}
} else if (value instanceof Collection<?>) {
for (Object v : ExpressionUtils.toObjectCollection(value)) {
_totalValueCount++;
if (ExpressionUtils.isError(v)) {
rowHasError = true;
} else if (ExpressionUtils.isNonBlankData(v)) {
if (v instanceof Number) {
rowHasNumeric = true;
processValue(((Number) v).doubleValue(), allValues);
} else {
rowHasNonNumeric = true;
}
} else {
rowHasBlank = true;
}
}
} else {
_totalValueCount++;
if (value instanceof Number) {
rowHasNumeric = true;
processValue(((Number) value).doubleValue(), allValues);
} else {
rowHasNonNumeric = true;
}
}
} else {
rowHasBlank = true;
}
if (rowHasError) {
_errorRowCount++;
}
if (rowHasBlank) {
_blankRowCount++;
}
if (rowHasNumeric) {
_numericRowCount++;
}
if (rowHasNonNumeric) {
_nonNumericRowCount++;
}
}
iterate(project, columnName, cellIndex, eval, allValues);
_numbericValueCount = allValues.size();
@ -203,6 +132,97 @@ public class NumericBinIndex {
return _errorRowCount;
}
protected void processRow(
Project project,
String columnName,
int cellIndex,
Evaluable eval,
List<Double> allValues,
int rowIndex,
Row row,
Properties bindings
) {
Cell cell = row.getCell(cellIndex);
ExpressionUtils.bind(bindings, row, rowIndex, columnName, cell);
Object value = eval.evaluate(bindings);
if (ExpressionUtils.isError(value)) {
_hasError = true;
} else if (ExpressionUtils.isNonBlankData(value)) {
if (value.getClass().isArray()) {
Object[] a = (Object[]) value;
for (Object v : a) {
_totalValueCount++;
if (ExpressionUtils.isError(v)) {
_hasError = true;
} else if (ExpressionUtils.isNonBlankData(v)) {
if (v instanceof Number) {
_hasNumeric = true;
processValue(((Number) v).doubleValue(), allValues);
} else {
_hasNonNumeric = true;
}
} else {
_hasBlank = true;
}
}
} else if (value instanceof Collection<?>) {
for (Object v : ExpressionUtils.toObjectCollection(value)) {
_totalValueCount++;
if (ExpressionUtils.isError(v)) {
_hasError = true;
} else if (ExpressionUtils.isNonBlankData(v)) {
if (v instanceof Number) {
_hasNumeric = true;
processValue(((Number) v).doubleValue(), allValues);
} else {
_hasNonNumeric = true;
}
} else {
_hasBlank = true;
}
}
} else {
_totalValueCount++;
if (value instanceof Number) {
_hasNumeric = true;
processValue(((Number) value).doubleValue(), allValues);
} else {
_hasNonNumeric = true;
}
}
} else {
_hasBlank = true;
}
}
protected void preprocessing() {
_hasBlank = false;
_hasError = false;
_hasNonNumeric = false;
_hasNumeric = false;
}
protected void postprocessing() {
if (_hasError) {
_errorRowCount++;
}
if (_hasBlank) {
_blankRowCount++;
}
if (_hasNumeric) {
_numericRowCount++;
}
if (_hasNonNumeric) {
_nonNumericRowCount++;
}
}
protected void processValue(double v, List<Double> allValues) {
if (!Double.isInfinite(v) && !Double.isNaN(v)) {
_min = Math.min(_min, v);

42
src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinRecordIndex.java Normal file
View File

@ -0,0 +1,42 @@
package com.metaweb.gridworks.browsing.facets;
import java.util.List;
import java.util.Properties;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
import com.metaweb.gridworks.model.Row;
public class NumericBinRecordIndex extends NumericBinIndex {
public NumericBinRecordIndex(Project project, String columnName,
int cellIndex, Evaluable eval) {
super(project, columnName, cellIndex, eval);
}
@Override
protected void iterate(
Project project, String columnName, int cellIndex,
Evaluable eval, List<Double> allValues) {
Properties bindings = ExpressionUtils.createBindings(project);
int count = project.recordModel.getRecordCount();
for (int r = 0; r < count; r++) {
Record record = project.recordModel.getRecord(r);
preprocessing();
for (int i = record.fromRowIndex; i < record.toRowIndex; i++) {
Row row = project.rows.get(i);
processRow(project, columnName, cellIndex, eval, allValues, i, row, bindings);
}
postprocessing();
}
}
}

36
src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinRowIndex.java Normal file
View File

@ -0,0 +1,36 @@
package com.metaweb.gridworks.browsing.facets;
import java.util.List;
import java.util.Properties;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
public class NumericBinRowIndex extends NumericBinIndex {
public NumericBinRowIndex(Project project, String columnName,
int cellIndex, Evaluable eval) {
super(project, columnName, cellIndex, eval);
}
@Override
protected void iterate(
Project project, String columnName, int cellIndex,
Evaluable eval, List<Double> allValues) {
Properties bindings = ExpressionUtils.createBindings(project);
for (int i = 0; i < project.rows.size(); i++) {
Row row = project.rows.get(i);
preprocessing();
processRow(project, columnName, cellIndex, eval, allValues, i, row, bindings);
postprocessing();
}
}
}

88
src/main/java/com/metaweb/gridworks/browsing/facets/RangeFacet.java
View File

@ -6,8 +6,11 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.FilteredRecords;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.filters.AnyRowRecordFilter;
import com.metaweb.gridworks.browsing.filters.ExpressionNumberComparisonRowFilter;
import com.metaweb.gridworks.browsing.filters.RecordFilter;
import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.MetaParser;
@ -165,45 +168,80 @@ public class RangeFacet implements Facet {
}
}
@Override
public RecordFilter getRecordFilter() {
RowFilter rowFilter = getRowFilter();
return rowFilter == null ? null : new AnyRowRecordFilter(rowFilter);
}
public void computeChoices(Project project, FilteredRows filteredRows) {
if (_eval != null && _errorMessage == null) {
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
String key = "numeric-bin:" + _expression;
String key = "numeric-bin:row-based:" + _expression;
NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
if (index == null) {
index = new NumericBinIndex(project, _columnName, _cellIndex, _eval);
index = new NumericBinRowIndex(project, _columnName, _cellIndex, _eval);
column.setPrecompute(key, index);
}
_min = index.getMin();
_max = index.getMax();
_step = index.getStep();
_baseBins = index.getBins();
_baseNumericCount = index.getNumericRowCount();
_baseNonNumericCount = index.getNonNumericRowCount();
_baseBlankCount = index.getBlankRowCount();
_baseErrorCount = index.getErrorRowCount();
if (_selected) {
_from = Math.max(_from, _min);
_to = Math.min(_to, _max);
} else {
_from = _min;
_to = _max;
}
retrieveDataFromBaseBinIndex(index);
ExpressionNumericRowBinner binner =
new ExpressionNumericRowBinner(_eval, _columnName, _cellIndex, index);
filteredRows.accept(project, binner);
_bins = binner.bins;
_numericCount = binner.numericCount;
_nonNumericCount = binner.nonNumericCount;
_blankCount = binner.blankCount;
_errorCount = binner.errorCount;
retrieveDataFromBinner(binner);
}
}
public void computeChoices(Project project, FilteredRecords filteredRecords) {
if (_eval != null && _errorMessage == null) {
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
String key = "numeric-bin:record-based:" + _expression;
NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
if (index == null) {
index = new NumericBinRecordIndex(project, _columnName, _cellIndex, _eval);
column.setPrecompute(key, index);
}
retrieveDataFromBaseBinIndex(index);
ExpressionNumericRowBinner binner =
new ExpressionNumericRowBinner(_eval, _columnName, _cellIndex, index);
filteredRecords.accept(project, binner);
retrieveDataFromBinner(binner);
}
}
protected void retrieveDataFromBaseBinIndex(NumericBinIndex index) {
_min = index.getMin();
_max = index.getMax();
_step = index.getStep();
_baseBins = index.getBins();
_baseNumericCount = index.getNumericRowCount();
_baseNonNumericCount = index.getNonNumericRowCount();
_baseBlankCount = index.getBlankRowCount();
_baseErrorCount = index.getErrorRowCount();
if (_selected) {
_from = Math.max(_from, _min);
_to = Math.min(_to, _max);
} else {
_from = _min;
_to = _max;
}
}
protected void retrieveDataFromBinner(ExpressionNumericRowBinner binner) {
_bins = binner.bins;
_numericCount = binner.numericCount;
_nonNumericCount = binner.nonNumericCount;
_blankCount = binner.blankCount;
_errorCount = binner.errorCount;
}
}

15
src/main/java/com/metaweb/gridworks/browsing/facets/ScatterplotDrawingRowVisitor.java
View File

@ -10,12 +10,14 @@ import java.awt.geom.Rectangle2D;
import java.awt.image.BufferedImage;
import java.awt.image.RenderedImage;
import com.metaweb.gridworks.browsing.RecordVisitor;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
import com.metaweb.gridworks.model.Row;
public class ScatterplotDrawingRowVisitor implements RowVisitor {
public class ScatterplotDrawingRowVisitor implements RowVisitor, RecordVisitor {
int col_x;
int col_y;
@ -85,7 +87,8 @@ public class ScatterplotDrawingRowVisitor implements RowVisitor {
g2.setPaint(color);
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
@Override
public boolean visit(Project project, int rowIndex, Row row) {
Cell cellx = row.getCell(col_x);
Cell celly = row.getCell(col_y);
if ((cellx != null && cellx.value != null && cellx.value instanceof Number) &&
@ -105,6 +108,14 @@ public class ScatterplotDrawingRowVisitor implements RowVisitor {
return false;
}
@Override
public boolean visit(Project project, Record record) {
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
visit(project, r, project.rows.get(r));
}
return false;
}
public RenderedImage getImage() {
return image;
}

70
src/main/java/com/metaweb/gridworks/browsing/facets/ScatterplotFacet.java
View File

@ -18,8 +18,11 @@ import org.json.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.browsing.FilteredRecords;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.filters.AnyRowRecordFilter;
import com.metaweb.gridworks.browsing.filters.DualExpressionsNumberComparisonRowFilter;
import com.metaweb.gridworks.browsing.filters.RecordFilter;
import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.MetaParser;
@ -269,19 +272,21 @@ public class ScatterplotFacet implements Facet {
}
}
@Override
public RecordFilter getRecordFilter() {
RowFilter rowFilter = getRowFilter();
return rowFilter == null ? null : new AnyRowRecordFilter(rowFilter);
}
public void computeChoices(Project project, FilteredRows filteredRows) {
if (eval_x != null && eval_y != null && errorMessage_x == null && errorMessage_y == null) {
Column column_x = project.columnModel.getColumnByCellIndex(columnIndex_x);
NumericBinIndex index_x = getBinIndex(project, column_x, eval_x, expression_x);
NumericBinIndex index_x = getBinIndex(project, column_x, eval_x, expression_x, "row-based");
min_x = index_x.getMin();
max_x = index_x.getMax();
Column column_y = project.columnModel.getColumnByCellIndex(columnIndex_y);
NumericBinIndex index_y = getBinIndex(project, column_y, eval_y, expression_y);
NumericBinIndex index_y = getBinIndex(project, column_y, eval_y, expression_y, "row-based");
min_y = index_y.getMin();
max_y = index_y.getMax();
retrieveDataFromBinIndices(index_x, index_y);
if (IMAGE_URI) {
if (index_x.isNumeric() && index_y.isNumeric()) {
@ -302,7 +307,45 @@ public class ScatterplotFacet implements Facet {
}
}
}
public void computeChoices(Project project, FilteredRecords filteredRecords) {
if (eval_x != null && eval_y != null && errorMessage_x == null && errorMessage_y == null) {
Column column_x = project.columnModel.getColumnByCellIndex(columnIndex_x);
NumericBinIndex index_x = getBinIndex(project, column_x, eval_x, expression_x, "record-based");
Column column_y = project.columnModel.getColumnByCellIndex(columnIndex_y);
NumericBinIndex index_y = getBinIndex(project, column_y, eval_y, expression_y, "record-based");
retrieveDataFromBinIndices(index_x, index_y);
if (IMAGE_URI) {
if (index_x.isNumeric() && index_y.isNumeric()) {
ScatterplotDrawingRowVisitor drawer = new ScatterplotDrawingRowVisitor(
columnIndex_x, columnIndex_y, min_x, max_x, min_y, max_y,
size, dim_x, dim_y, rotation, dot, color
);
filteredRecords.accept(project, drawer);
try {
image = serializeImage(drawer.getImage());
} catch (IOException e) {
logger.warn("Exception caught while generating the image", e);
}
} else {
image = EMPTY_IMAGE;
}
}
}
}
protected void retrieveDataFromBinIndices(NumericBinIndex index_x, NumericBinIndex index_y) {
min_x = index_x.getMin();
max_x = index_x.getMax();
min_y = index_y.getMin();
max_y = index_y.getMax();
}
public static String serializeImage(RenderedImage image) throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream(4096);
ImageIO.write(image, "png", output);
@ -328,7 +371,11 @@ public class ScatterplotFacet implements Facet {
}
public static NumericBinIndex getBinIndex(Project project, Column column, Evaluable eval, String expression) {
String key = "numeric-bin:" + expression;
return getBinIndex(project, column, eval, expression, "row-based");
}
public static NumericBinIndex getBinIndex(Project project, Column column, Evaluable eval, String expression, String mode) {
String key = "numeric-bin:" + mode + ":" + expression;
if (eval == null) {
try {
eval = MetaParser.parse(expression);
@ -338,7 +385,10 @@ public class ScatterplotFacet implements Facet {
}
NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
if (index == null) {
index = new NumericBinIndex(project, column.getName(), column.getCellIndex(), eval);
index = "row-based".equals(mode) ?
new NumericBinRowIndex(project, column.getName(), column.getCellIndex(), eval) :
new NumericBinRecordIndex(project, column.getName(), column.getCellIndex(), eval);
column.setPrecompute(key, index);
}
return index;

18
src/main/java/com/metaweb/gridworks/browsing/facets/TextSearchFacet.java
View File

@ -7,8 +7,11 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.FilteredRecords;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.filters.AnyRowRecordFilter;
import com.metaweb.gridworks.browsing.filters.ExpressionStringComparisonRowFilter;
import com.metaweb.gridworks.browsing.filters.RecordFilter;
import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.gel.ast.VariableExpr;
@ -33,6 +36,7 @@ public class TextSearchFacet implements Facet {
public TextSearchFacet() {
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
@ -45,6 +49,7 @@ public class TextSearchFacet implements Facet {
writer.endObject();
}
@Override
public void initializeFromJSON(Project project, JSONObject o) throws Exception {
_name = o.getString("name");
_columnName = o.getString("columnName");
@ -72,6 +77,7 @@ public class TextSearchFacet implements Facet {
}
}
@Override
public RowFilter getRowFilter() {
if (_query == null || _query.length() == 0) {
return null;
@ -96,7 +102,19 @@ public class TextSearchFacet implements Facet {
}
}
@Override
public RecordFilter getRecordFilter() {
RowFilter rowFilter = getRowFilter();
return rowFilter == null ? null : new AnyRowRecordFilter(rowFilter);
}
@Override
public void computeChoices(Project project, FilteredRows filteredRows) {
// nothing to do
}
@Override
public void computeChoices(Project project, FilteredRecords filteredRecords) {
// nothing to do
}
}

22
src/main/java/com/metaweb/gridworks/browsing/filters/AnyRowRecordFilter.java Normal file
View File

@ -0,0 +1,22 @@
package com.metaweb.gridworks.browsing.filters;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
public class AnyRowRecordFilter implements RecordFilter {
final protected RowFilter _rowFilter;
public AnyRowRecordFilter(RowFilter rowFilter) {
_rowFilter = rowFilter;
}
@Override
public boolean filterRecord(Project project, Record record) {
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
if (_rowFilter.filterRow(project, r, project.rows.get(r))) {
return true;
}
}
return false;
}
}

12
src/main/java/com/metaweb/gridworks/browsing/filters/RecordFilter.java Normal file
View File

@ -0,0 +1,12 @@
package com.metaweb.gridworks.browsing.filters;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
/**
* Interface for judging if a particular record matches or doesn't match some
* particular criterion, such as a facet constraint.
*/
public interface RecordFilter {
public boolean filterRecord(Project project, Record record);
}

4
src/main/java/com/metaweb/gridworks/clustering/binning/BinningClusterer.java
View File

@ -66,7 +66,7 @@ public class BinningClusterer extends Clusterer {
}
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_colindex);
if (cell != null && cell.value != null) {
Object v = cell.value;
@ -128,7 +128,7 @@ public class BinningClusterer extends Clusterer {
public void computeClusters(Engine engine) {
BinningRowVisitor visitor = new BinningRowVisitor(_keyer,_config);
FilteredRows filteredRows = engine.getAllFilteredRows(true);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, visitor);
Map<String,Map<String,Integer>> map = visitor.getMap();

6
src/main/java/com/metaweb/gridworks/clustering/knn/kNNClusterer.java
View File

@ -80,7 +80,7 @@ public class kNNClusterer extends Clusterer {
}
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_colindex);
if (cell != null && cell.value != null) {
Object v = cell.value;
@ -121,7 +121,7 @@ public class kNNClusterer extends Clusterer {
_clusterer = new NGramClusterer(_distance, _blockingNgramSize);
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_colindex);
if (cell != null && cell.value != null) {
Object v = cell.value;
@ -145,7 +145,7 @@ public class kNNClusterer extends Clusterer {
public void computeClusters(Engine engine) {
//VPTreeClusteringRowVisitor visitor = new VPTreeClusteringRowVisitor(_distance,_config);
BlockingClusteringRowVisitor visitor = new BlockingClusteringRowVisitor(_distance,_config);
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, visitor);
_clusters = visitor.getClusters();

2
src/main/java/com/metaweb/gridworks/commands/browsing/GetScatterplotCommand.java
View File

@ -161,7 +161,7 @@ public class GetScatterplotCommand extends Command {
}
{
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, drawer);
}

3
src/main/java/com/metaweb/gridworks/commands/column/GetColumnsInfoCommand.java
View File

@ -10,6 +10,7 @@ import org.json.JSONException;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.facets.NumericBinIndex;
import com.metaweb.gridworks.browsing.facets.NumericBinRowIndex;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.MetaParser;
@ -59,7 +60,7 @@ public class GetColumnsInfoCommand extends Command {
}
NumericBinIndex index = (NumericBinIndex) column.getPrecompute(key);
if (index == null) {
index = new NumericBinIndex(project, column.getName(), column.getCellIndex(), eval);
index = new NumericBinRowIndex(project, column.getName(), column.getCellIndex(), eval);
column.setPrecompute(key, index);
}
return index;

112
src/main/java/com/metaweb/gridworks/commands/row/GetRowsCommand.java
View File

@ -11,10 +11,14 @@ import org.json.JSONException;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRecords;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RecordVisitor;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.browsing.Engine.Mode;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Record;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.util.Pool;
@ -42,56 +46,71 @@ public class GetRowsCommand extends Command {
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
{
RowAccumulator acc = new RowAccumulator(start, limit) {
JSONWriter writer;
Properties options;
Properties extra;
RowAccumulator acc = new RowAccumulator(start, limit) {
JSONWriter writer;
Properties options;
public RowAccumulator init(JSONWriter writer, Properties options) {
this.writer = writer;
this.options = options;
public RowAccumulator init(JSONWriter writer, Properties options) {
this.writer = writer;
this.options = options;
this.extra = new Properties();
this.extra.put("contextual", true);
return this;
return this;
}
@Override
public boolean internalVisit(Project project, int rowIndex, Row row) {
try {
options.put("rowIndex", rowIndex);
row.write(writer, options);
} catch (JSONException e) {
}
return false;
}
@Override
protected boolean internalVisit(Project project, Record record) {
options.put("recordIndex", record.recordIndex);
@Override
public boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) {
for (int r = record.fromRowIndex; r < record.toRowIndex; r++) {
try {
/*
* Whatever that's in the "extra" field will be written out
* by the row as well. This is how we can customize what the row
* writes, in a limited way.
*/
if (contextual) {
options.put("extra", extra);
} else {
options.remove("extra");
}
options.put("rowIndex", rowIndex);
Row row = project.rows.get(r);
options.put("rowIndex", r);
row.write(writer, options);
} catch (JSONException e) {
}
return false;
}
}.init(writer, options);
FilteredRows filteredRows = engine.getAllFilteredRows(true);
options.remove("recordIndex");
}
return false;
}
}.init(writer, options);
if (engine.getMode() == Mode.RowBased) {
FilteredRows filteredRows = engine.getAllFilteredRows();
writer.key("mode"); writer.value("row-based");
writer.key("rows"); writer.array();
filteredRows.accept(project, acc);
writer.endArray();
writer.key("filtered"); writer.value(acc.total);
writer.key("total"); writer.value(project.rows.size());
} else {
FilteredRecords filteredRecords = engine.getFilteredRecords();
writer.key("mode"); writer.value("record-based");
writer.key("rows"); writer.array();
filteredRecords.accept(project, acc);
writer.endArray();
writer.key("filtered"); writer.value(acc.total);
writer.key("total"); writer.value(project.recordModel.getRecordCount());
}
writer.key("start"); writer.value(start);
writer.key("limit"); writer.value(limit);
writer.key("total"); writer.value(project.rows.size());
writer.key("pool"); pool.write(writer, options);
writer.endObject();
@ -100,7 +119,7 @@ public class GetRowsCommand extends Command {
}
}
static protected class RowAccumulator implements RowVisitor {
static protected class RowAccumulator implements RowVisitor, RecordVisitor {
final public int start;
final public int limit;
@ -111,19 +130,32 @@ public class GetRowsCommand extends Command {
this.limit = limit;
}
public boolean visit(Project project, int rowIndex, Row row, boolean contextual, boolean dependent) {
public boolean visit(Project project, int rowIndex, Row row) {
boolean r = false;
if (total >= start && total < start + limit) {
r = internalVisit(rowIndex, row, contextual, dependent);
}
if (!contextual) {
total++;
r = internalVisit(project, rowIndex, row);
}
total++;
return r;
}
protected boolean internalVisit(int rowIndex, Row row, boolean contextual, boolean dependent) {
@Override
public boolean visit(Project project, Record record) {
boolean r = false;
if (total >= start && total < start + limit) {
r = internalVisit(project, record);
}
total++;
return r;
}
protected boolean internalVisit(Project project, int rowIndex, Row row) {
return false;
}
protected boolean internalVisit(Project project, Record record) {
return false;
}
}

4
src/main/java/com/metaweb/gridworks/exporters/HtmlTableExporter.java
View File

@ -56,7 +56,7 @@ public class HtmlTableExporter implements Exporter {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean contextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
try {
writer.write("<tr>");
@ -83,7 +83,7 @@ public class HtmlTableExporter implements Exporter {
}
}.init(writer);
FilteredRows filteredRows = engine.getAllFilteredRows(true);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
}

4
src/main/java/com/metaweb/gridworks/exporters/TsvExporter.java
View File

@ -48,7 +48,7 @@ public class TsvExporter implements Exporter {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean contextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
boolean first = true;
try {
for (Column column : project.columnModel.columns) {
@ -82,7 +82,7 @@ public class TsvExporter implements Exporter {
}
}.init(writer);
FilteredRows filteredRows = engine.getAllFilteredRows(true);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
}
}

4
src/main/java/com/metaweb/gridworks/exporters/XlsExporter.java
View File

@ -64,7 +64,7 @@ public class XlsExporter implements Exporter {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean contextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
org.apache.poi.ss.usermodel.Row r = sheet.createRow(rowCount++);
int cellCount = 0;
@ -105,7 +105,7 @@ public class XlsExporter implements Exporter {
}
}.init(s, rowCount);
FilteredRows filteredRows = engine.getAllFilteredRows(true);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, visitor);
}

4
src/main/java/com/metaweb/gridworks/model/RecordModel.java
View File

@ -32,6 +32,10 @@ public class RecordModel {
_rowDependencies.get(rowIndex) : null;
}
public int getRecordCount() {
return _records.size();
}
public Record getRecord(int recordIndex) {
return _records != null && recordIndex >= 0 && recordIndex < _records.size() ?
_records.get(recordIndex) : null;

16
src/main/java/com/metaweb/gridworks/model/Row.java
View File

@ -15,19 +15,13 @@ import org.json.JSONWriter;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.expr.CellTuple;
import com.metaweb.gridworks.expr.HasFields;
import com.metaweb.gridworks.model.RecordModel.RowDependency;
import com.metaweb.gridworks.util.Pool;
public class Row implements HasFields, Jsonizable {
public boolean flagged;
public boolean starred;
final public List<Cell> cells;
/*
transient public int recordIndex = -1; // -1 for rows that are not main record rows
transient public List<Integer> contextRows;
transient public int[] contextRowSlots;
transient public int[] contextCellSlots;
*/
private static final String FLAGGED = "flagged";
private static final String STARRED = "starred";
@ -134,10 +128,10 @@ public class Row implements HasFields, Jsonizable {
int rowIndex = (Integer) options.get("rowIndex");
writer.key("i"); writer.value(rowIndex);
Project project = (Project) options.get("project");
RowDependency rd = project.recordModel.getRowDependency(rowIndex);
if (rd.recordIndex >= 0) {
writer.key("j"); writer.value(rd.recordIndex);
if (options.containsKey("recordIndex")) {
int recordIndex = (Integer) options.get("recordIndex");
writer.key("j"); writer.value(recordIndex);
}
}

4
src/main/java/com/metaweb/gridworks/operations/ColumnAdditionOperation.java
View File

@ -104,7 +104,7 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
List<CellAtRow> cellsAtRows = new ArrayList<CellAtRow>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, cellsAtRows));
String description = createDescription(column, cellsAtRows);
@ -135,7 +135,7 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
Cell newCell = null;

4
src/main/java/com/metaweb/gridworks/operations/ColumnSplitOperation.java
View File

@ -148,7 +148,7 @@ public class ColumnSplitOperation extends EngineDependentOperation {
List<Integer> rowIndices = new ArrayList<Integer>(project.rows.size());
List<List<Serializable>> tuples = new ArrayList<List<Serializable>>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
RowVisitor rowVisitor;
if ("lengths".equals(_mode)) {
rowVisitor = new ColumnSplitRowVisitor(column.getCellIndex(), columnNames, rowIndices, tuples) {
@ -234,7 +234,7 @@ public class ColumnSplitOperation extends EngineDependentOperation {
this.tuples = tuples;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Object value = row.getCellValue(cellIndex);
if (ExpressionUtils.isNonBlankData(value)) {
String s = value instanceof String ? ((String) value) : value.toString();

2
src/main/java/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java
View File

@ -36,7 +36,7 @@ abstract public class EngineDependentMassCellOperation extends EngineDependentOp
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
try {
filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID));
} catch (Exception e) {

13
src/main/java/com/metaweb/gridworks/operations/ExtendDataOperation.java
View File

@ -140,7 +140,7 @@ public class ExtendDataOperation extends EngineDependentOperation {
_cellIndex = column.getCellIndex();
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, new RowVisitor() {
List<Integer> _rowIndices;
@ -148,13 +148,12 @@ public class ExtendDataOperation extends EngineDependentOperation {
_rowIndices = rowIndices;
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
if (!includeContextual) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex);
}
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex);
}
return false;
}
}.init(rowIndices));

2
src/main/java/com/metaweb/gridworks/operations/MassEditOperation.java
View File

@ -196,7 +196,7 @@ public class MassEditOperation extends EngineDependentMassCellOperation {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
Cell newCell = null;

2
src/main/java/com/metaweb/gridworks/operations/ReconDiscardJudgmentsOperation.java
View File

@ -74,7 +74,7 @@ public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOpera
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
if (cell != null && cell.recon != null) {
Recon newRecon;

2
src/main/java/com/metaweb/gridworks/operations/ReconJudgeSimilarCellsOperation.java
View File

@ -162,7 +162,7 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {
String value = cell.value instanceof String ?

2
src/main/java/com/metaweb/gridworks/operations/ReconMarkNewTopicsOperation.java
View File

@ -84,7 +84,7 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
if (cell != null) {
Recon recon = null;

2
src/main/java/com/metaweb/gridworks/operations/ReconMatchBestCandidatesOperation.java
View File

@ -75,7 +75,7 @@ public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOp
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell != null && cell.recon != null) {

2
src/main/java/com/metaweb/gridworks/operations/ReconMatchSpecificTopicOperation.java
View File

@ -108,7 +108,7 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
if (cell != null) {
long reconID = cell.recon != null ? cell.recon.id : 0;

4
src/main/java/com/metaweb/gridworks/operations/ReconOperation.java
View File

@ -174,9 +174,9 @@ public class ReconOperation extends EngineDependentOperation {
_entries = new ArrayList<ReconEntry>(_project.rows.size());
_cellIndex = column.getCellIndex();
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, new RowVisitor() {
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
if (_cellIndex < row.cells.size()) {
Cell cell = row.cells.get(_cellIndex);
if (cell != null && ExpressionUtils.isNonBlankData(cell.value)) {

4
src/main/java/com/metaweb/gridworks/operations/RowFlagOperation.java
View File

@ -57,7 +57,7 @@ public class RowFlagOperation extends EngineDependentOperation {
List<Change> changes = new ArrayList<Change>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, changes));
return new HistoryEntry(
@ -78,7 +78,7 @@ public class RowFlagOperation extends EngineDependentOperation {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
if (row.flagged != _flagged) {
RowFlagChange change = new RowFlagChange(rowIndex, _flagged);

9
src/main/java/com/metaweb/gridworks/operations/RowRemovalOperation.java
View File

@ -49,7 +49,7 @@ public class RowRemovalOperation extends EngineDependentOperation {
List<Integer> rowIndices = new ArrayList<Integer>();
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, rowIndices));
return new HistoryEntry(
@ -70,10 +70,9 @@ public class RowRemovalOperation extends EngineDependentOperation {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
if (!includeContextual) {
rowIndices.add(rowIndex);
}
public boolean visit(Project project, int rowIndex, Row row) {
rowIndices.add(rowIndex);
return false;
}
}.init(rowIndices);

4
src/main/java/com/metaweb/gridworks/operations/RowStarOperation.java
View File

@ -57,7 +57,7 @@ public class RowStarOperation extends EngineDependentOperation {
List<Change> changes = new ArrayList<Change>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(false);
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, changes));
return new HistoryEntry(
@ -78,7 +78,7 @@ public class RowStarOperation extends EngineDependentOperation {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
if (row.starred != _starred) {
RowStarChange change = new RowStarChange(rowIndex, _starred);

2
src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java
View File

@ -119,7 +119,7 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
Cell newCell = null;

23
src/main/webapp/scripts/views/data-table-view.js
View File

@ -51,13 +51,14 @@ DataTableView.prototype.render = function() {
DataTableView.prototype._renderSummaryText = function(elmt) {
var summaryText;
var units = theProject.rowModel.mode == "row-based" ? "rows" : "records";
var from = (theProject.rowModel.start + 1);
var to = Math.min(theProject.rowModel.filtered, theProject.rowModel.start + theProject.rowModel.limit);
if (theProject.rowModel.filtered == theProject.rowModel.total) {
summaryText = from + ' &ndash; ' + to + ' of <span class="viewPanel-summary-row-count">' + (theProject.rowModel.total) + '</span> rows';
summaryText = from + ' &ndash; ' + to + ' of <span class="viewPanel-summary-row-count">' + (theProject.rowModel.total) + '</span> ' + units;
} else {
summaryText = from + ' &ndash; ' + to + ' of <span class="viewPanel-summary-row-count">' +
(theProject.rowModel.filtered) + '</span> matching rows (' + (theProject.rowModel.total) + ' total)';
(theProject.rowModel.filtered) + '</span> matching ' + units + ' (' + (theProject.rowModel.total) + ' total)';
}
$('<span>').html(summaryText).appendTo(elmt);
};
@ -285,15 +286,15 @@ DataTableView.prototype._renderDataTable = function(table) {
});
var tdIndex = tr.insertCell(tr.cells.length);
if ("j" in row) {
$(tr).addClass("record");
$('<div></div>').html((row.j + 1) + ".").appendTo(tdIndex);
if (theProject.rowModel.mode == "record-based") {
if ("j" in row) {
$(tr).addClass("record");
$('<div></div>').html((row.j + 1) + ".").appendTo(tdIndex);
} else {
$('<div></div>').html("&nbsp;").appendTo(tdIndex);
}
} else {
$('<div></div>').html("&nbsp;").appendTo(tdIndex);
}
if ("contextual" in row && row.contextual) {
$(tr).addClass("contextual");
$('<div></div>').html((row.i + 1) + ".").appendTo(tdIndex);
}
$(tr).addClass(even ? "even" : "odd");
@ -314,7 +315,7 @@ DataTableView.prototype._renderDataTable = function(table) {
for (var r = 0; r < rows.length; r++) {
var row = rows[r];
var tr = table.insertRow(table.rows.length);
if ("j" in row) {
if (theProject.rowModel.mode == "row-based" || "j" in row) {
even = !even;
}
renderRow(tr, r, row, even);