Encapsulated a bunch of fields.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@33 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-02-03 21:57:38 +00:00
parent 97fd9422f6
commit 21f2403146
20 changed files with 195 additions and 110 deletions

View File

@ -11,11 +11,39 @@ import org.json.JSONWriter;
public class ProjectMetadata implements Serializable, Jsonizable { public class ProjectMetadata implements Serializable, Jsonizable {
private static final long serialVersionUID = 7959027046468240844L; private static final long serialVersionUID = 7959027046468240844L;
public String name; private final Date _created = new Date();
public String password; private String _name;
public Date created = new Date(); private String _password;
public Date modified = new Date(); private Date _modified = new Date();
public Date getCreated() {
return _created;
}
public void setName(String name) {
this._name = name;
}
public String getName() {
return _name;
}
public void setPassword(String password) {
this._password = password;
}
public String getPassword() {
return _password;
}
public Date getModified() {
return _modified;
}
public void updateModified() {
_modified = new Date();
}
@Override @Override
public void write(JSONWriter writer, Properties options) public void write(JSONWriter writer, Properties options)
throws JSONException { throws JSONException {
@ -23,9 +51,9 @@ public class ProjectMetadata implements Serializable, Jsonizable {
SimpleDateFormat sdf = (SimpleDateFormat) SimpleDateFormat.getDateTimeInstance(); SimpleDateFormat sdf = (SimpleDateFormat) SimpleDateFormat.getDateTimeInstance();
writer.object(); writer.object();
writer.key("name"); writer.value(name); writer.key("name"); writer.value(getName());
writer.key("created"); writer.value(sdf.format(created)); writer.key("created"); writer.value(sdf.format(getCreated()));
writer.key("modified"); writer.value(sdf.format(modified)); writer.key("modified"); writer.value(sdf.format(_modified));
writer.endObject(); writer.endObject();
} }
} }

View File

@ -10,16 +10,16 @@ import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row; import com.metaweb.gridworks.model.Row;
public class NumericBinIndex { public class NumericBinIndex {
public double min; private double _min;
public double max; private double _max;
public double step; private double _step;
public int[] bins; private int[] _bins;
public NumericBinIndex(Project project, int cellIndex, Evaluable eval) { public NumericBinIndex(Project project, int cellIndex, Evaluable eval) {
Properties bindings = new Properties(); Properties bindings = new Properties();
min = Double.POSITIVE_INFINITY; _min = Double.POSITIVE_INFINITY;
max = Double.NEGATIVE_INFINITY; _max = Double.NEGATIVE_INFINITY;
List<Double> allValues = new ArrayList<Double>(); List<Double> allValues = new ArrayList<Double>();
for (int i = 0; i < project.rows.size(); i++) { for (int i = 0; i < project.rows.size(); i++) {
@ -49,40 +49,55 @@ public class NumericBinIndex {
} }
} }
if (min >= max) { if (getMin() >= getMax()) {
step = 0; _step = 0;
bins = new int[0]; _bins = new int[0];
return; return;
} }
double diff = max - min; double diff = getMax() - getMin();
_step = 1;
if (diff > 10) { if (diff > 10) {
step = 1; while (getStep() * 100 < diff) {
while (step * 100 < diff) { _step *= 10;
step *= 10;
} }
} else { } else {
step = 1; while (getStep() * 100 > diff) {
while (step * 100 > diff) { _step /= 10;
step /= 10;
} }
} }
min = Math.floor(min / step) * step; _min = (Math.floor(_min / _step) * _step);
max = Math.ceil(max / step) * step; _max = (Math.ceil(_max / _step) * _step);
int binCount = 1 + (int) Math.ceil((max - min) / step); int binCount = 1 + (int) Math.ceil((getMax() - getMin()) / getStep());
bins = new int[binCount]; _bins = new int[binCount];
for (double d : allValues) { for (double d : allValues) {
int bin = (int) Math.round((d - min) / step); int bin = (int) Math.round((d - getMin()) / getStep());
bins[bin]++; getBins()[bin]++;
} }
} }
public double getMin() {
return _min;
}
public double getMax() {
return _max;
}
public double getStep() {
return _step;
}
public int[] getBins() {
return _bins;
}
protected void processValue(double v, List<Double> allValues) { protected void processValue(double v, List<Double> allValues) {
min = Math.min(min, v); _min = Math.min(getMin(), v);
max = Math.max(max, v); _max = Math.max(getMax(), v);
allValues.add(v); allValues.add(v);
} }
} }

View File

@ -115,9 +115,9 @@ public class RangeFacet implements Facet {
column.setPrecompute(key, index); column.setPrecompute(key, index);
} }
_min = index.min; _min = index.getMin();
_max = index.max; _max = index.getMax();
_step = index.step; _step = index.getStep();
_bins = index.bins; _bins = index.getBins();
} }
} }

View File

@ -7,11 +7,9 @@ import org.json.JSONObject;
import org.json.JSONWriter; import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.FilteredRows; import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.filters.ExpressionNumberComparisonRowFilter;
import com.metaweb.gridworks.browsing.filters.ExpressionStringComparisonRowFilter; import com.metaweb.gridworks.browsing.filters.ExpressionStringComparisonRowFilter;
import com.metaweb.gridworks.browsing.filters.RowFilter; import com.metaweb.gridworks.browsing.filters.RowFilter;
import com.metaweb.gridworks.expr.Evaluable; import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.Parser;
import com.metaweb.gridworks.expr.VariableExpr; import com.metaweb.gridworks.expr.VariableExpr;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;

View File

@ -39,7 +39,7 @@ public class ApproveNewReconcileCommand extends Command {
return; return;
} }
String columnName = column.headerLabel; String columnName = column.getHeaderLabel();
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size()); List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(); FilteredRows filteredRows = engine.getAllFilteredRows();
@ -58,9 +58,10 @@ public class ApproveNewReconcileCommand extends Command {
if (cellIndex < row.cells.size()) { if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex); Cell cell = row.cells.get(cellIndex);
Cell newCell = new Cell(); Cell newCell = new Cell(
newCell.value = cell.value; cell.value,
newCell.recon = cell.recon != null ? cell.recon.dup() : new Recon(); cell.recon != null ? cell.recon.dup() : new Recon()
);
newCell.recon.match = null; newCell.recon.match = null;
newCell.recon.judgment = Judgment.New; newCell.recon.judgment = Judgment.New;

View File

@ -38,7 +38,7 @@ public class ApproveReconcileCommand extends Command {
return; return;
} }
String columnName = column.headerLabel; String columnName = column.getHeaderLabel();
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size()); List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(); FilteredRows filteredRows = engine.getAllFilteredRows();
@ -57,9 +57,10 @@ public class ApproveReconcileCommand extends Command {
if (cellIndex < row.cells.size()) { if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex); Cell cell = row.cells.get(cellIndex);
if (cell.recon != null && cell.recon.candidates.size() > 0) { if (cell.recon != null && cell.recon.candidates.size() > 0) {
Cell newCell = new Cell(); Cell newCell = new Cell(
newCell.value = cell.value; cell.value,
newCell.recon = cell.recon.dup(); cell.recon.dup()
);
newCell.recon.match = newCell.recon.candidates.get(0); newCell.recon.match = newCell.recon.candidates.get(0);
newCell.recon.judgment = Judgment.Approve; newCell.recon.judgment = Judgment.Approve;

View File

@ -23,8 +23,8 @@ public class CreateProjectFromUploadCommand extends Command {
String content = readFileUpload(request, properties); String content = readFileUpload(request, properties);
ProjectMetadata pm = new ProjectMetadata(); ProjectMetadata pm = new ProjectMetadata();
pm.name = properties.getProperty("project-name"); pm.setName(properties.getProperty("project-name"));
pm.password = properties.getProperty("project-password"); pm.setPassword(properties.getProperty("project-password"));
Project project = ProjectManager.singleton.createProject(pm); Project project = ProjectManager.singleton.createProject(pm);
@ -63,9 +63,7 @@ public class CreateProjectFromUploadCommand extends Command {
cell = cell.substring(1, cell.length() - 1); cell = cell.substring(1, cell.length() - 1);
} }
Column column = new Column(); Column column = new Column(c, cell);
column.cellIndex = c;
column.headerLabel = cell;
project.columnModel.columns.add(column); project.columnModel.columns.add(column);
} }
@ -76,8 +74,7 @@ public class CreateProjectFromUploadCommand extends Command {
if ((sep.charAt(0) == ',') ? parseCSVIntoRow(row, line) : parseTSVIntoRow(row, line)) { if ((sep.charAt(0) == ',') ? parseCSVIntoRow(row, line) : parseTSVIntoRow(row, line)) {
project.rows.add(row); project.rows.add(row);
project.columnModel.maxCellIndex = project.columnModel.setMaxCellIndex(Math.max(project.columnModel.getMaxCellIndex(), row.cells.size()));
Math.max(project.columnModel.maxCellIndex, row.cells.size());
} }
} }
} }
@ -94,8 +91,7 @@ public class CreateProjectFromUploadCommand extends Command {
for (int c = 0; c < cells.length; c++) { for (int c = 0; c < cells.length; c++) {
String text = cells[c]; String text = cells[c];
Cell cell = new Cell(); Cell cell = new Cell(parseCellValue(text), null);
cell.value = parseCellValue(text);
row.cells.add(cell); row.cells.add(cell);
@ -133,8 +129,7 @@ public class CreateProjectFromUploadCommand extends Command {
} }
} }
Cell cell = new Cell(); Cell cell = new Cell(parseCellValue(text), null);
cell.value = parseCellValue(text);
row.cells.add(cell); row.cells.add(cell);

View File

@ -37,7 +37,7 @@ public class DiscardReconcileCommand extends Command {
return; return;
} }
String columnName = column.headerLabel; String columnName = column.getHeaderLabel();
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size()); List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows(); FilteredRows filteredRows = engine.getAllFilteredRows();
@ -56,9 +56,7 @@ public class DiscardReconcileCommand extends Command {
if (cellIndex < row.cells.size()) { if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex); Cell cell = row.cells.get(cellIndex);
Cell newCell = new Cell(); Cell newCell = new Cell(cell.value, null);
newCell.value = cell.value;
newCell.recon = null;
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange); cellChanges.add(cellChange);

View File

@ -41,7 +41,7 @@ public class DoTextTransformCommand extends Command {
return; return;
} }
String columnName = column.headerLabel; String columnName = column.getHeaderLabel();
String expression = request.getParameter("expression"); String expression = request.getParameter("expression");
Evaluable eval = new Parser(expression).getExpression(); Evaluable eval = new Parser(expression).getExpression();
@ -73,9 +73,7 @@ public class DoTextTransformCommand extends Command {
bindings.put("cell", cell); bindings.put("cell", cell);
bindings.put("value", cell.value); bindings.put("value", cell.value);
Cell newCell = new Cell(); Cell newCell = new Cell(eval.evaluate(bindings), cell.recon);
newCell.value = eval.evaluate(bindings);
newCell.recon = cell.recon;
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell); CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange); cellChanges.add(cellChange);

View File

@ -35,7 +35,7 @@ public class ReconcileCommand extends Command {
return; return;
} }
String columnName = column.headerLabel; String columnName = column.getHeaderLabel();
String typeID = request.getParameter("type"); String typeID = request.getParameter("type");
List<ReconEntry> entries = new ArrayList<ReconEntry>(project.rows.size()); List<ReconEntry> entries = new ArrayList<ReconEntry>(project.rows.size());

View File

@ -2,7 +2,6 @@ package com.metaweb.gridworks.history;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
@ -39,7 +38,7 @@ public class History implements Serializable, Jsonizable {
} }
protected void setModified() { protected void setModified() {
ProjectManager.singleton.getProjectMetadata(_projectID).modified = new Date(); ProjectManager.singleton.getProjectMetadata(_projectID).updateModified();
} }
public List<HistoryEntry> getLastPastEntries(int count) { public List<HistoryEntry> getLastPastEntries(int count) {

View File

@ -13,7 +13,8 @@ public class HistoryProcess extends Process {
final protected Project _project; final protected Project _project;
final protected long _lastDoneID; final protected long _lastDoneID;
final protected String _description; final protected String _description;
boolean _done = false;
protected boolean _done = false;
public HistoryProcess(Project project, long lastDoneID) { public HistoryProcess(Project project, long lastDoneID) {
_project = project; _project = project;

View File

@ -12,8 +12,13 @@ import com.metaweb.gridworks.expr.HasFields;
public class Cell implements Serializable, HasFields, Jsonizable { public class Cell implements Serializable, HasFields, Jsonizable {
private static final long serialVersionUID = -5891067829205458102L; private static final long serialVersionUID = -5891067829205458102L;
public Object value; final public Object value;
public Recon recon; final public Recon recon;
public Cell(Object value, Recon recon) {
this.value = value;
this.recon = recon;
}
@Override @Override
public Object getField(String name, Properties bindings) { public Object getField(String name, Properties bindings) {

View File

@ -13,20 +13,40 @@ import com.metaweb.gridworks.Jsonizable;
public class Column implements Serializable, Jsonizable { public class Column implements Serializable, Jsonizable {
private static final long serialVersionUID = -1063342490951563563L; private static final long serialVersionUID = -1063342490951563563L;
public int cellIndex; final private int _cellIndex;
public String headerLabel; final private String _originalHeaderLabel;
public Class valueType; private String _headerLabel;
transient protected Map<String, Object> _precomputes; transient protected Map<String, Object> _precomputes;
public Column(int cellIndex, String headerLabel) {
_cellIndex = cellIndex;
_originalHeaderLabel = _headerLabel = headerLabel;
}
public int getCellIndex() {
return _cellIndex;
}
public String getOriginalHeaderLabel() {
return _originalHeaderLabel;
}
public void setHeaderLabel(String headerLabel) {
this._headerLabel = headerLabel;
}
public String getHeaderLabel() {
return _headerLabel;
}
@Override @Override
public void write(JSONWriter writer, Properties options) public void write(JSONWriter writer, Properties options)
throws JSONException { throws JSONException {
writer.object(); writer.object();
writer.key("cellIndex"); writer.value(cellIndex); writer.key("cellIndex"); writer.value(getCellIndex());
writer.key("headerLabel"); writer.value(headerLabel); writer.key("headerLabel"); writer.value(getHeaderLabel());
writer.key("valueType"); writer.value(valueType == null ? null : valueType.getSimpleName());
writer.endObject(); writer.endObject();
} }

View File

@ -15,13 +15,15 @@ import com.metaweb.gridworks.Jsonizable;
public class ColumnGroup implements Serializable, Jsonizable { public class ColumnGroup implements Serializable, Jsonizable {
private static final long serialVersionUID = 2161780779920066118L; private static final long serialVersionUID = 2161780779920066118L;
public int[] cellIndices; // must be in order from smallest to largest final public int[] cellIndices; // must be in order from smallest to largest
public int keyCellIndex; // could be -1 if there is no key cell final public int keyCellIndex; // could be -1 if there is no key cell
transient public ColumnGroup parentGroup; transient public ColumnGroup parentGroup;
transient public List<ColumnGroup> subgroups; transient public List<ColumnGroup> subgroups;
public ColumnGroup() { public ColumnGroup(int[] cellIndices, int keyCellIndex) {
this.cellIndices = cellIndices;
this.keyCellIndex = keyCellIndex;
internalInitialize(); internalInitialize();
} }

View File

@ -19,11 +19,11 @@ import com.metaweb.gridworks.Jsonizable;
public class ColumnModel implements Serializable, Jsonizable { public class ColumnModel implements Serializable, Jsonizable {
private static final long serialVersionUID = 7679639795211544511L; private static final long serialVersionUID = 7679639795211544511L;
public List<Column> columns = new LinkedList<Column>(); final public List<Column> columns = new LinkedList<Column>();
public int maxCellIndex; final public List<ColumnGroup> columnGroups = new LinkedList<ColumnGroup>();
public List<ColumnGroup> columnGroups = new LinkedList<ColumnGroup>(); private int _maxCellIndex;
public int keyCellIndex; private int _keyCellIndex;
transient protected Map<String, Column> _nameToColumn; transient protected Map<String, Column> _nameToColumn;
transient protected Map<Integer, Column> _cellIndexToColumn; transient protected Map<Integer, Column> _cellIndexToColumn;
@ -33,6 +33,23 @@ public class ColumnModel implements Serializable, Jsonizable {
internalInitialize(); internalInitialize();
} }
public void setMaxCellIndex(int maxCellIndex) {
this._maxCellIndex = Math.max(this._maxCellIndex, maxCellIndex);
}
public int getMaxCellIndex() {
return _maxCellIndex;
}
public void setKeyCellIndex(int keyCellIndex) {
// TODO: check validity of new cell index, e.g., it's not in any group
this._keyCellIndex = keyCellIndex;
}
public int getKeyCellIndex() {
return _keyCellIndex;
}
public void update() { public void update() {
generateMaps(); generateMaps();
} }
@ -58,7 +75,7 @@ public class ColumnModel implements Serializable, Jsonizable {
} }
writer.endArray(); writer.endArray();
writer.key("keyCellIndex"); writer.value(keyCellIndex); writer.key("keyCellIndex"); writer.value(getKeyCellIndex());
writer.key("columnGroups"); writer.key("columnGroups");
writer.array(); writer.array();
for (ColumnGroup g : _rootColumnGroups) { for (ColumnGroup g : _rootColumnGroups) {
@ -114,8 +131,8 @@ public class ColumnModel implements Serializable, Jsonizable {
_cellIndexToColumn = new HashMap<Integer, Column>(); _cellIndexToColumn = new HashMap<Integer, Column>();
for (Column column : columns) { for (Column column : columns) {
_nameToColumn.put(column.headerLabel, column); _nameToColumn.put(column.getHeaderLabel(), column);
_cellIndexToColumn.put(column.cellIndex, column); _cellIndexToColumn.put(column.getCellIndex(), column);
} }
} }
} }

View File

@ -12,11 +12,11 @@ import com.metaweb.gridworks.process.ProcessManager;
public class Project implements Serializable { public class Project implements Serializable {
private static final long serialVersionUID = -5089046824819472570L; private static final long serialVersionUID = -5089046824819472570L;
public long id; final public long id;
public ColumnModel columnModel = new ColumnModel(); final public ColumnModel columnModel = new ColumnModel();
public List<Row> rows = new ArrayList<Row>(); final public List<Row> rows = new ArrayList<Row>();
public History history; final public History history;
transient public ProcessManager processManager; transient public ProcessManager processManager;

View File

@ -12,11 +12,19 @@ import com.metaweb.gridworks.expr.HasFields;
public class ReconCandidate implements Serializable, HasFields, Jsonizable { public class ReconCandidate implements Serializable, HasFields, Jsonizable {
private static final long serialVersionUID = -8013997214978715606L; private static final long serialVersionUID = -8013997214978715606L;
public String topicID; final public String topicID;
public String topicGUID; final public String topicGUID;
public String topicName; final public String topicName;
public String[] typeIDs; final public String[] typeIDs;
public double score; final public double score;
public ReconCandidate(String topicID, String topicGUID, String topicName, String[] typeIDs, double score) {
this.topicID = topicID;
this.topicGUID = topicGUID;
this.topicName = topicName;
this.typeIDs = typeIDs;
this.score = score;
}
@Override @Override
public Object getField(String name, Properties bindings) { public Object getField(String name, Properties bindings) {

View File

@ -16,7 +16,7 @@ public class Row implements Serializable, HasFields, Jsonizable {
public boolean flagged; public boolean flagged;
public boolean starred; public boolean starred;
public List<Cell> cells; final public List<Cell> cells;
transient public List<Integer> contextRows; transient public List<Integer> contextRows;
transient public List<Integer> contextCells; transient public List<Integer> contextCells;
@ -45,7 +45,7 @@ public class Row implements Serializable, HasFields, Jsonizable {
Project project = (Project) bindings.get("project"); Project project = (Project) bindings.get("project");
Column column = project.columnModel.getColumnByName(name); Column column = project.columnModel.getColumnByName(name);
if (column != null) { if (column != null) {
return cells.get(column.cellIndex); return cells.get(column.getCellIndex());
} }
return null; return null;
} }

View File

@ -165,9 +165,7 @@ public class ReconProcess extends LongRunningProcess implements Runnable {
for (ReconEntry entry : valueToEntries.get(value)) { for (ReconEntry entry : valueToEntries.get(value)) {
Cell oldCell = entry.cell; Cell oldCell = entry.cell;
Cell newCell = new Cell(); Cell newCell = new Cell(oldCell.value, recon);
newCell.value = oldCell.value;
newCell.recon = recon;
CellChange cellChange = new CellChange( CellChange cellChange = new CellChange(
entry.rowIndex, entry.rowIndex,
@ -198,19 +196,20 @@ public class ReconProcess extends LongRunningProcess implements Runnable {
continue; continue;
} }
ReconCandidate candidate = new ReconCandidate();
candidate.topicID = result.getString("id");
candidate.topicGUID = result.getString("guid");
candidate.topicName = result.getString("name");
candidate.score = result.getDouble("relevance:score");
JSONArray types = result.getJSONArray("type"); JSONArray types = result.getJSONArray("type");
candidate.typeIDs = new String[types.length()]; String[] typeIDs = new String[types.length()];
for (int j = 0; j < candidate.typeIDs.length; j++) { for (int j = 0; j < typeIDs.length; j++) {
candidate.typeIDs[j] = types.getJSONObject(j).getString("id"); typeIDs[j] = types.getJSONObject(j).getString("id");
} }
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("guid"),
result.getString("name"),
typeIDs,
result.getDouble("relevance:score")
);
// best match // best match
if (i == 0) { if (i == 0) {
recon.features.put("nameMatch", text.equalsIgnoreCase(candidate.topicName)); recon.features.put("nameMatch", text.equalsIgnoreCase(candidate.topicName));