s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Internal refactor for IO - HistoryEntry is now a concrete class, so can be instantiated (reverting Operations classes back to r972 which were changed as a result of HistoryEntry being abstract).

Browse Source 
HistoryEntry now deals with backend (filesystem etc.) through classes which implement HistoryEntryManager.  This HistoryEntryManager is held by ProjectManager, which allows for FileProjectManager to create FileHistoryEntryManager as appropriate.

git-svn-id: http://google-refine.googlecode.com/svn/trunk@982 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Iain Sproat 2010-06-16 12:35:37 +00:00
parent 280daad2f6
commit f92fc2d056
23 changed files with 447 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
main/src/com/metaweb/gridworks/ProjectManager.java
View File

@ -2,7 +2,6 @@ package com.metaweb.gridworks;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -11,9 +10,7 @@ import org.apache.tools.tar.TarOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.history.HistoryEntryManager;
import com.metaweb.gridworks.model.Project;
@ -44,6 +41,8 @@ public abstract class ProjectManager {
*/
transient protected Map<Long, Project> _projects;
protected HistoryEntryManager _historyEntryManager;
static public ProjectManager singleton;
public InterProjectModel getInterProjectModel() {
@ -126,6 +125,9 @@ public abstract class ProjectManager {
public abstract void deleteProject(long projectID) ;
public abstract HistoryEntry createHistoryEntry(long id, long projectID, String description, AbstractOperation operation, Date time);
public abstract HistoryEntry createHistoryEntry(long id, Project project, String description, AbstractOperation operation, Change change);
//public abstract HistoryEntry createHistoryEntry(long id, long projectID, String description, AbstractOperation operation, Date time);
//public abstract HistoryEntry createHistoryEntry(long id, Project project, String description, AbstractOperation operation, Change change);
public HistoryEntryManager getHistoryEntryManager(){
return this._historyEntryManager;
}
}

3
main/src/com/metaweb/gridworks/commands/cell/EditOneCellCommand.java
View File

@ -10,7 +10,6 @@ import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.history.HistoryEntry;
@ -122,7 +121,7 @@ public class EditOneCellCommand extends Command {
Change change = new CellChange(rowIndex, cellIndex, cell, newCell);
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID, _project, description, null, change);
}
}

3
main/src/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java
View File

@ -9,7 +9,6 @@ import javax.servlet.http.HttpServletResponse;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.history.Change;
@ -199,7 +198,7 @@ public class ReconJudgeOneCellCommand extends Command {
stats
);
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID, _project, description, null, change);
}
}

5
main/src/com/metaweb/gridworks/commands/row/AnnotateOneRowCommand.java
View File

@ -6,7 +6,6 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Project;
@ -83,7 +82,7 @@ public class AnnotateOneRowCommand extends Command {
}
protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception {
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
_project,
(starred ? "Star row " : "Unstar row ") + (rowIndex + 1),
@ -109,7 +108,7 @@ public class AnnotateOneRowCommand extends Command {
}
protected HistoryEntry createHistoryEntry(long historyEntryID) throws Exception {
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
_project,
(flagged ? "Flag row " : "Unflag row ") + (rowIndex + 1),

92
main/src/com/metaweb/gridworks/history/History.java
View File

@ -38,18 +38,18 @@ public class History implements Jsonizable {
reader.close();
}
}
static public Change readOneChange(LineNumberReader reader, Pool pool) throws Exception {
/* String version = */ reader.readLine();
String className = reader.readLine();
Class<? extends Change> klass = getChangeClass(className);
Method load = klass.getMethod("load", LineNumberReader.class, Pool.class);
return (Change) load.invoke(null, reader, pool);
}
static public void writeOneChange(OutputStream out, Change change, Pool pool) throws IOException {
Writer writer = new OutputStreamWriter(out);
try {
@ -58,47 +58,47 @@ public class History implements Jsonizable {
writer.flush();
}
}
static public void writeOneChange(Writer writer, Change change, Pool pool) throws IOException {
Properties options = new Properties();
options.setProperty("mode", "save");
options.put("pool", pool);
writeOneChange(writer, change, options);
}
static public void writeOneChange(Writer writer, Change change, Properties options) throws IOException {
writer.write(GridworksServlet.getVersion()); writer.write('\n');
writer.write(change.getClass().getName()); writer.write('\n');
change.save(writer, options);
}
@SuppressWarnings("unchecked")
static public Class<? extends Change> getChangeClass(String className) throws ClassNotFoundException {
return (Class<? extends Change>) Class.forName(className);
}
protected long _projectID;
protected List<HistoryEntry> _pastEntries; // done changes, can be undone
protected List<HistoryEntry> _futureEntries; // undone changes, can be redone
public History(Project project) {
_projectID = project.id;
_pastEntries = new ArrayList<HistoryEntry>();
_futureEntries = new ArrayList<HistoryEntry>();
}
synchronized public void addEntry(HistoryEntry entry) {
entry.apply(ProjectManager.singleton.getProject(_projectID));
_pastEntries.add(entry);
setModified();
// Any new change will clear all future entries.
List<HistoryEntry> futureEntries = _futureEntries;
_futureEntries = new ArrayList<HistoryEntry>();
for (HistoryEntry entry2 : futureEntries) {
try {
// remove residual data on disk
@ -108,11 +108,11 @@ public class History implements Jsonizable {
}
}
}
protected void setModified() {
ProjectManager.singleton.getProjectMetadata(_projectID).updateModified();
}
synchronized public List<HistoryEntry> getLastPastEntries(int count) {
if (count <= 0) {
return new LinkedList<HistoryEntry>(_pastEntries);
@ -120,7 +120,7 @@ public class History implements Jsonizable {
return _pastEntries.subList(Math.max(_pastEntries.size() - count, 0), _pastEntries.size());
}
}
synchronized public void undoRedo(long lastDoneEntryID) {
if (lastDoneEntryID == 0) {
// undo all the way back to the start of the project
@ -132,7 +132,7 @@ public class History implements Jsonizable {
return;
}
}
for (int i = 0; i < _futureEntries.size(); i++) {
if (_futureEntries.get(i).id == lastDoneEntryID) {
redo(i + 1);
@ -141,7 +141,7 @@ public class History implements Jsonizable {
}
}
}
synchronized public long getPrecedingEntryID(long entryID) {
if (entryID == 0) {
return -1;
@ -151,7 +151,7 @@ public class History implements Jsonizable {
return i == 0 ? 0 : _pastEntries.get(i - 1).id;
}
}
for (int i = 0; i < _futureEntries.size(); i++) {
if (_futureEntries.get(i).id == entryID) {
if (i > 0) {
@ -166,14 +166,14 @@ public class History implements Jsonizable {
}
return -1;
}
protected HistoryEntry getEntry(long entryID) {
for (int i = 0; i < _pastEntries.size(); i++) {
if (_pastEntries.get(i).id == entryID) {
return _pastEntries.get(i);
}
}
for (int i = 0; i < _futureEntries.size(); i++) {
if (_futureEntries.get(i).id == entryID) {
return _futureEntries.get(i);
@ -181,89 +181,89 @@ public class History implements Jsonizable {
}
return null;
}
protected void undo(int times) {
Project project = ProjectManager.singleton.getProject(_projectID);
while (times > 0 && _pastEntries.size() > 0) {
HistoryEntry entry = _pastEntries.get(_pastEntries.size() - 1);
entry.revert(project);
setModified();
times--;
_pastEntries.remove(_pastEntries.size() - 1);
_futureEntries.add(0, entry);
}
}
protected void redo(int times) {
Project project = ProjectManager.singleton.getProject(_projectID);
while (times > 0 && _futureEntries.size() > 0) {
HistoryEntry entry = _futureEntries.get(0);
entry.apply(project);
setModified();
times--;
_pastEntries.add(entry);
_futureEntries.remove(0);
}
}
synchronized public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("past"); writer.array();
for (HistoryEntry entry : _pastEntries) {
entry.write(writer, options);
}
writer.endArray();
writer.key("future"); writer.array();
for (HistoryEntry entry : _futureEntries) {
entry.write(writer, options);
}
writer.endArray();
writer.endObject();
}
synchronized public void save(Writer writer, Properties options) throws IOException {
writer.write("pastEntryCount="); writer.write(Integer.toString(_pastEntries.size())); writer.write('\n');
for (HistoryEntry entry : _pastEntries) {
entry.save(writer, options); writer.write('\n');
}
writer.write("futureEntryCount="); writer.write(Integer.toString(_futureEntries.size())); writer.write('\n');
for (HistoryEntry entry : _futureEntries) {
entry.save(writer, options); writer.write('\n');
}
writer.write("/e/\n");
}
synchronized public void load(Project project, LineNumberReader reader) throws Exception {
String line;
while ((line = reader.readLine()) != null && !"/e/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("pastEntryCount".equals(field)) {
int count = Integer.parseInt(value);
for (int i = 0; i < count; i++) {
_pastEntries.add(HistoryEntry.load(project, reader.readLine()));
}
} else if ("futureEntryCount".equals(field)) {
int count = Integer.parseInt(value);
for (int i = 0; i < count; i++) {
_futureEntries.add(HistoryEntry.load(project, reader.readLine()));
}

53
main/src/com/metaweb/gridworks/history/HistoryEntry.java
View File

@ -19,20 +19,31 @@ import com.metaweb.gridworks.util.ParsingUtilities;
* This is the metadata of a Change. It's small, so we can load it in order to
* obtain information about a change without actually loading the change.
*/
public abstract class HistoryEntry implements Jsonizable {
public class HistoryEntry implements Jsonizable {
final public long id;
final public long projectID;
final public String description;
final public Date time;
// the manager (deals with IO systems or databases etc.)
final public HistoryEntryManager _manager;
// the abstract operation, if any, that results in the change
final public AbstractOperation operation;
// the actual change, loaded on demand
transient protected Change _change;
private transient Change _change;
private final static String OPERATION = "operation";
public void setChange(Change _change) {
this._change = _change;
}
public Change getChange() {
return _change;
}
static public long allocateID() {
return Math.round(Math.random() * 1000000) + System.currentTimeMillis();
}
@ -44,7 +55,8 @@ public abstract class HistoryEntry implements Jsonizable {
this.operation = operation;
this.time = new Date();
_change = change;
this._manager = ProjectManager.singleton.getHistoryEntryManager();
setChange(change);
}
protected HistoryEntry(long id, long projectID, String description, AbstractOperation operation, Date time) {
@ -53,6 +65,7 @@ public abstract class HistoryEntry implements Jsonizable {
this.description = description;
this.operation = operation;
this.time = time;
this._manager = ProjectManager.singleton.getHistoryEntryManager();
}
public void write(JSONWriter writer, Properties options)
@ -68,23 +81,27 @@ public abstract class HistoryEntry implements Jsonizable {
writer.endObject();
}
public void save(Writer writer, Properties options){
_manager.save(this, writer, options);
}
public void apply(Project project) {
if (_change == null) {
loadChange();
if (getChange() == null) {
ProjectManager.singleton.getHistoryEntryManager().loadChange(this);
}
synchronized (project) {
_change.apply(project);
getChange().apply(project);
// When a change is applied, it can hang on to old data (in order to be able
// to revert later). Hence, we need to save the change out.
try {
saveChange();
_manager.saveChange(this);
} catch (Exception e) {
e.printStackTrace();
_change.revert(project);
getChange().revert(project);
throw new RuntimeException("Failed to apply change", e);
}
@ -92,20 +109,12 @@ public abstract class HistoryEntry implements Jsonizable {
}
public void revert(Project project) {
if (_change == null) {
loadChange();
if (getChange() == null) {
_manager.loadChange(this);
}
_change.revert(project);
getChange().revert(project);
}
public abstract void loadChange();
protected abstract void saveChange() throws Exception;
public abstract void save(Writer writer, Properties options);
protected abstract void delete();
static public HistoryEntry load(Project project, String s) throws Exception {
JSONObject obj = ParsingUtilities.evaluateJsonStringToObject(s);
@ -114,7 +123,7 @@ public abstract class HistoryEntry implements Jsonizable {
operation = OperationRegistry.reconstruct(project, obj.getJSONObject(OPERATION));
}
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
obj.getLong("id"),
project.id,
obj.getString("description"),
@ -123,6 +132,8 @@ public abstract class HistoryEntry implements Jsonizable {
);
}
public void delete(){
_manager.delete(this);
}
}

53
main/src/com/metaweb/gridworks/io/FileHistoryEntry.java
View File

@ -8,55 +8,46 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import java.io.Writer;
import java.util.Date;
import org.json.JSONException;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.history.History;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.history.HistoryEntryManager;
import com.metaweb.gridworks.util.Pool;
public class FileHistoryEntry extends HistoryEntry{
protected FileHistoryEntry(long id, long projectID, String description, AbstractOperation operation, Date time) {
super(id, projectID, description, operation, time);
}
protected FileHistoryEntry(long id, Project project, String description, AbstractOperation operation, Change change){
super(id, project, description, operation, change);
}
public class FileHistoryEntry implements HistoryEntryManager{
protected void delete() {
File file = getChangeFile();
public void delete(HistoryEntry historyEntry) {
File file = getChangeFile(historyEntry);
if (file.exists()) {
file.delete();
}
}
public void save(Writer writer, Properties options) {
public void save(HistoryEntry historyEntry, Writer writer, Properties options) {
JSONWriter jsonWriter = new JSONWriter(writer);
try {
write(jsonWriter, options);
historyEntry.write(jsonWriter, options);
} catch (JSONException e) {
e.printStackTrace();
}
}
public void loadChange() {
File changeFile = getChangeFile();
public void loadChange(HistoryEntry historyEntry) {
File changeFile = getChangeFile(historyEntry);
try {
loadChange(changeFile);
loadChange(historyEntry, changeFile);
} catch (Exception e) {
throw new RuntimeException("Failed to load change file " + changeFile.getAbsolutePath(), e);
}
}
protected void loadChange(File file) throws Exception {
protected void loadChange(HistoryEntry historyEntry, File file) throws Exception {
ZipFile zipFile = new ZipFile(file);
try {
Pool pool = new Pool();
@ -66,28 +57,28 @@ public class FileHistoryEntry extends HistoryEntry{
zipFile.getInputStream(poolEntry)));
} // else, it's a legacy project file
_change = History.readOneChange(
zipFile.getInputStream(zipFile.getEntry("change.txt")), pool);
historyEntry.setChange(History.readOneChange(
zipFile.getInputStream(zipFile.getEntry("change.txt")), pool));
} finally {
zipFile.close();
}
}
protected void saveChange() throws Exception {
File changeFile = getChangeFile();
public void saveChange(HistoryEntry historyEntry) throws Exception {
File changeFile = getChangeFile(historyEntry);
if (!(changeFile.exists())) {
saveChange(changeFile);
saveChange(historyEntry, changeFile);
}
}
protected void saveChange(File file) throws Exception {
protected void saveChange(HistoryEntry historyEntry, File file) throws Exception {
ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file));
try {
Pool pool = new Pool();
out.putNextEntry(new ZipEntry("change.txt"));
try {
History.writeOneChange(out, _change, pool);
History.writeOneChange(out, historyEntry.getChange(), pool);
} finally {
out.closeEntry();
}
@ -103,12 +94,14 @@ public class FileHistoryEntry extends HistoryEntry{
}
}
protected File getChangeFile() {
return new File(getHistoryDir(), id + ".change.zip");
protected File getChangeFile(HistoryEntry historyEntry) {
return new File(getHistoryDir(historyEntry), historyEntry.id + ".change.zip");
}
protected File getHistoryDir() {//FIXME relies on FileProjectManager
File dir = new File(((FileProjectManager)ProjectManager.singleton).getProjectDir(projectID), "history");
protected File getHistoryDir(HistoryEntry historyEntry) {
File dir = new File(((FileProjectManager)ProjectManager.singleton)
.getProjectDir(historyEntry.projectID),
"history");
dir.mkdirs();
return dir;

13
main/src/com/metaweb/gridworks/io/FileProjectManager.java
View File

@ -30,9 +30,6 @@ import org.slf4j.LoggerFactory;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.JSONUtilities;
@ -47,6 +44,7 @@ public class FileProjectManager extends ProjectManager{
logger.info("Using workspace directory: {}", dir.getAbsolutePath());
singleton = new FileProjectManager(dir);
}
}
private FileProjectManager(File dir) {
@ -57,6 +55,8 @@ public class FileProjectManager extends ProjectManager{
_expressions = new LinkedList<String>();
_projects = new HashMap<Long, Project>();
_historyEntryManager = new FileHistoryEntry();
load();
}
@ -445,11 +445,4 @@ public class FileProjectManager extends ProjectManager{
return found;
}
public HistoryEntry createHistoryEntry(long id, long projectID, String description, AbstractOperation operation, Date time){
return new FileHistoryEntry(id, projectID, description, operation, time);
}
public HistoryEntry createHistoryEntry(long id, Project project, String description, AbstractOperation operation, Change change){
return new FileHistoryEntry(id, project, description, operation, change);
}
}

19
main/src/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java
View File

@ -5,7 +5,6 @@ import java.util.List;
import org.json.JSONObject;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -19,7 +18,7 @@ import com.metaweb.gridworks.model.changes.MassCellChange;
abstract public class EngineDependentMassCellOperation extends EngineDependentOperation {
final protected String _columnName;
final protected boolean _updateRowContextDependencies;
protected EngineDependentMassCellOperation(
JSONObject engineConfig, String columnName, boolean updateRowContextDependencies) {
super(engineConfig);
@ -29,32 +28,32 @@ abstract public class EngineDependentMassCellOperation extends EngineDependentOp
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Engine engine = createEngine(project);
Column column = project.columnModel.getColumnByName(_columnName);
if (column == null) {
throw new Exception("No column named " + _columnName);
}
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
try {
filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID));
} catch (Exception e) {
e.printStackTrace();
}
String description = createDescription(column, cellChanges);
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID, project, description, this, createChange(project, column, cellChanges));
}
protected Change createChange(Project project, Column column, List<CellChange> cellChanges) {
return new MassCellChange(
cellChanges, column.getName(), _updateRowContextDependencies);
}
abstract protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges, long historyEntryID) throws Exception;
abstract protected String createDescription(Column column, List<CellChange> cellChanges);
}

33
main/src/com/metaweb/gridworks/operations/SaveProtographOperation.java
View File

@ -9,7 +9,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
@ -20,20 +19,20 @@ import com.metaweb.gridworks.util.Pool;
public class SaveProtographOperation extends AbstractOperation {
final protected Protograph _protograph;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new SaveProtographOperation(
Protograph.reconstruct(obj.getJSONObject("protograph"))
);
}
public SaveProtographOperation(Protograph protograph) {
_protograph = protograph;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Save protograph");
@ -48,20 +47,20 @@ public class SaveProtographOperation extends AbstractOperation {
@Override
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
String description = "Save schema-alignment protograph";
Change change = new ProtographChange(_protograph);
return ProjectManager.singleton.createHistoryEntry(historyEntryID, project, description, SaveProtographOperation.this, change);
return new HistoryEntry(historyEntryID, project, description, SaveProtographOperation.this, change);
}
static public class ProtographChange implements Change {
final protected Protograph _newProtograph;
protected Protograph _oldProtograph;
public ProtographChange(Protograph protograph) {
_newProtograph = protograph;
}
public void apply(Project project) {
synchronized (project) {
_oldProtograph = project.protograph;
@ -74,36 +73,36 @@ public class SaveProtographOperation extends AbstractOperation {
project.protograph = _oldProtograph;
}
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("newProtograph="); writeProtograph(_newProtograph, writer); writer.write('\n');
writer.write("oldProtograph="); writeProtograph(_oldProtograph, writer); writer.write('\n');
writer.write("/ec/\n"); // end of change marker
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
Protograph oldProtograph = null;
Protograph newProtograph = null;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("oldProtograph".equals(field) && value.length() > 0) {
oldProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
} else if ("newProtograph".equals(field) && value.length() > 0) {
newProtograph = Protograph.reconstruct(ParsingUtilities.evaluateJsonStringToObject(value));
}
}
ProtographChange change = new ProtographChange(newProtograph);
change._oldProtograph = oldProtograph;
return change;
}
static protected void writeProtograph(Protograph p, Writer writer) throws IOException {
if (p != null) {
JSONWriter jsonWriter = new JSONWriter(writer);
@ -114,5 +113,5 @@ public class SaveProtographOperation extends AbstractOperation {
}
}
}
}
}
}

37
main/src/com/metaweb/gridworks/operations/cell/MultiValuedCellJoinOperation.java
View File

@ -8,7 +8,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
@ -31,7 +30,7 @@ public class MultiValuedCellJoinOperation extends AbstractOperation {
obj.getString("separator")
);
}
public MultiValuedCellJoinOperation(
String columnName,
String keyColumnName,
@ -44,7 +43,7 @@ public class MultiValuedCellJoinOperation extends AbstractOperation {
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -53,7 +52,7 @@ public class MultiValuedCellJoinOperation extends AbstractOperation {
writer.key("separator"); writer.value(_separator);
writer.endObject();
}
protected String getBriefDescription(Project project) {
return "Join multi-valued cells in column " + _columnName;
}
@ -64,34 +63,34 @@ public class MultiValuedCellJoinOperation extends AbstractOperation {
throw new Exception("No column named " + _columnName);
}
int cellIndex = column.getCellIndex();
Column keyColumn = project.columnModel.getColumnByName(_keyColumnName);
if (keyColumn == null) {
throw new Exception("No key column named " + _keyColumnName);
}
int keyCellIndex = keyColumn.getCellIndex();
List<Row> newRows = new ArrayList<Row>();
int oldRowCount = project.rows.size();
for (int r = 0; r < oldRowCount; r++) {
Row oldRow = project.rows.get(r);
if (oldRow.isCellBlank(keyCellIndex)) {
newRows.add(oldRow.dup());
continue;
}
int r2 = r + 1;
while (r2 < oldRowCount && project.rows.get(r2).isCellBlank(keyCellIndex)) {
r2++;
}
if (r2 == r + 1) {
newRows.add(oldRow.dup());
continue;
}
StringBuffer sb = new StringBuffer();
for (int r3 = r; r3 < r2; r3++) {
Object value = project.rows.get(r3).getCellValue(cellIndex);
@ -102,7 +101,7 @@ public class MultiValuedCellJoinOperation extends AbstractOperation {
sb.append(value.toString());
}
}
for (int r3 = r; r3 < r2; r3++) {
Row newRow = project.rows.get(r3).dup();
if (r3 == r) {
@ -110,20 +109,20 @@ public class MultiValuedCellJoinOperation extends AbstractOperation {
} else {
newRow.setCell(cellIndex, null);
}
if (!newRow.isEmpty()) {
newRows.add(newRow);
}
}
r = r2 - 1; // r will be incremented by the for loop anyway
}
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
project,
getBriefDescription(null),
this,
project,
getBriefDescription(null),
this,
new MassRowChange(newRows)
);
}

47
main/src/com/metaweb/gridworks/operations/cell/MultiValuedCellSplitOperation.java
View File

@ -9,7 +9,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Cell;
@ -33,7 +32,7 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
obj.getString("mode")
);
}
public MultiValuedCellSplitOperation(
String columnName,
String keyColumnName,
@ -48,7 +47,7 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Split multi-valued cells in column " + _columnName);
@ -70,15 +69,15 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
throw new Exception("No column named " + _columnName);
}
int cellIndex = column.getCellIndex();
Column keyColumn = project.columnModel.getColumnByName(_keyColumnName);
if (keyColumn == null) {
throw new Exception("No key column named " + _keyColumnName);
}
int keyCellIndex = keyColumn.getCellIndex();
List<Row> newRows = new ArrayList<Row>();
int oldRowCount = project.rows.size();
for (int r = 0; r < oldRowCount; r++) {
Row oldRow = project.rows.get(r);
@ -86,7 +85,7 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
newRows.add(oldRow.dup());
continue;
}
Object value = oldRow.getCellValue(cellIndex);
String s = value instanceof String ? ((String) value) : value.toString();
String[] values = null;
@ -95,53 +94,53 @@ public class MultiValuedCellSplitOperation extends AbstractOperation {
} else {
values = StringUtils.splitByWholeSeparator(s, _separator);
}
if (values.length < 2) {
newRows.add(oldRow.dup());
continue;
}
// First value goes into the same row
{
Row firstNewRow = oldRow.dup();
firstNewRow.setCell(cellIndex, new Cell(values[0].trim(), null));
newRows.add(firstNewRow);
}
int r2 = r + 1;
for (int v = 1; v < values.length; v++) {
Cell newCell = new Cell(values[v].trim(), null);
if (r2 < project.rows.size()) {
Row oldRow2 = project.rows.get(r2);
if (oldRow2.isCellBlank(cellIndex) &&
if (oldRow2.isCellBlank(cellIndex) &&
oldRow2.isCellBlank(keyCellIndex)) {
Row newRow = oldRow2.dup();
newRow.setCell(cellIndex, newCell);
newRows.add(newRow);
r2++;
continue;
}
}
Row newRow = new Row(cellIndex + 1);
newRow.setCell(cellIndex, newCell);
newRows.add(newRow);
}
r = r2 - 1; // r will be incremented by the for loop anyway
}
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
project,
getBriefDescription(null),
this,
project,
getBriefDescription(null),
this,
new MassRowChange(newRows)
);
}

63
main/src/com/metaweb/gridworks/operations/column/ColumnAdditionOperation.java
View File

@ -9,7 +9,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -35,13 +34,13 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
final protected String _baseColumnName;
final protected String _expression;
final protected OnError _onError;
final protected String _newColumnName;
final protected int _columnInsertIndex;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ColumnAdditionOperation(
engineConfig,
obj.getString("baseColumnName"),
@ -51,28 +50,28 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
obj.getInt("columnInsertIndex")
);
}
public ColumnAdditionOperation(
JSONObject engineConfig,
String baseColumnName,
String expression,
OnError onError,
String newColumnName,
int columnInsertIndex
String newColumnName,
int columnInsertIndex
) {
super(engineConfig);
_baseColumnName = baseColumnName;
_expression = expression;
_onError = onError;
_newColumnName = newColumnName;
_columnInsertIndex = columnInsertIndex;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -86,52 +85,52 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
}
protected String getBriefDescription(Project project) {
return "Create column " + _newColumnName +
" at index " + _columnInsertIndex +
" based on column " + _baseColumnName +
return "Create column " + _newColumnName +
" at index " + _columnInsertIndex +
" based on column " + _baseColumnName +
" using expression " + _expression;
}
protected String createDescription(Column column, List<CellAtRow> cellsAtRows) {
return "Create new column " + _newColumnName +
" based on column " + column.getName() +
return "Create new column " + _newColumnName +
" based on column " + column.getName() +
" by filling " + cellsAtRows.size() +
" rows with " + _expression;
}
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Engine engine = createEngine(project);
Column column = project.columnModel.getColumnByName(_baseColumnName);
if (column == null) {
throw new Exception("No column named " + _baseColumnName);
}
List<CellAtRow> cellsAtRows = new ArrayList<CellAtRow>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, cellsAtRows));
String description = createDescription(column, cellsAtRows);
Change change = new ColumnAdditionChange(_newColumnName, _columnInsertIndex, cellsAtRows);
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID, project, description, this, change);
}
protected RowVisitor createRowVisitor(Project project, List<CellAtRow> cellsAtRows) throws Exception {
Column column = project.columnModel.getColumnByName(_baseColumnName);
Evaluable eval = MetaParser.parse(_expression);
Properties bindings = ExpressionUtils.createBindings(project);
return new RowVisitor() {
int cellIndex;
Properties bindings;
List<CellAtRow> cellsAtRows;
Evaluable eval;
public RowVisitor init(int cellIndex, Properties bindings, List<CellAtRow> cellsAtRows, Evaluable eval) {
this.cellIndex = cellIndex;
this.bindings = bindings;
@ -139,23 +138,23 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
this.eval = eval;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(cellIndex);
Cell newCell = null;
ExpressionUtils.bind(bindings, row, rowIndex, _baseColumnName, cell);
Object o = eval.evaluate(bindings);
if (o != null) {
if (o instanceof Cell) {
@ -171,17 +170,17 @@ public class ColumnAdditionOperation extends EngineDependentOperation {
v = cell != null ? cell.value : null;
}
}
if (v != null) {
newCell = new Cell(v, null);
}
}
}
if (newCell != null) {
cellsAtRows.add(new CellAtRow(rowIndex, newCell));
}
return false;
}
}.init(column.getCellIndex(), bindings, cellsAtRows, eval);

15
main/src/com/metaweb/gridworks/operations/column/ColumnRemovalOperation.java
View File

@ -6,7 +6,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
@ -23,16 +22,16 @@ public class ColumnRemovalOperation extends AbstractOperation {
obj.getString("columnName")
);
}
public ColumnRemovalOperation(
String columnName
) {
_columnName = columnName;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Remove column " + _columnName);
@ -50,11 +49,11 @@ public class ColumnRemovalOperation extends AbstractOperation {
if (column == null) {
throw new Exception("No column named " + _columnName);
}
String description = "Remove column " + column.getName();
Change change = new ColumnRemovalChange(project.columnModel.columns.indexOf(column));
return ProjectManager.singleton.createHistoryEntry(historyEntryID, project, description, ColumnRemovalOperation.this, change);
return new HistoryEntry(historyEntryID, project, description, ColumnRemovalOperation.this, change);
}
}

13
main/src/com/metaweb/gridworks/operations/column/ColumnRenameOperation.java
View File

@ -6,7 +6,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
@ -24,7 +23,7 @@ public class ColumnRenameOperation extends AbstractOperation {
obj.getString("newColumnName")
);
}
public ColumnRenameOperation(
String oldColumnName,
String newColumnName
@ -32,10 +31,10 @@ public class ColumnRenameOperation extends AbstractOperation {
_oldColumnName = oldColumnName;
_newColumnName = newColumnName;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Rename column " + _oldColumnName + " to " + _newColumnName);
@ -56,9 +55,9 @@ public class ColumnRenameOperation extends AbstractOperation {
if (project.columnModel.getColumnByName(_newColumnName) != null) {
throw new Exception("Another column already named " + _newColumnName);
}
Change change = new ColumnRenameChange(_oldColumnName, _newColumnName);
return ProjectManager.singleton.createHistoryEntry(historyEntryID, project, getBriefDescription(null), ColumnRenameOperation.this, change);
return new HistoryEntry(historyEntryID, project, getBriefDescription(null), ColumnRenameOperation.this, change);
}
}

87
main/src/com/metaweb/gridworks/operations/column/ColumnSplitOperation.java
View File

@ -11,7 +11,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -33,17 +32,17 @@ public class ColumnSplitOperation extends EngineDependentOperation {
final protected boolean _guessCellType;
final protected boolean _removeOriginalColumn;
final protected String _mode;
final protected String _separator;
final protected boolean _regex;
final protected int _maxColumns;
final protected int[] _fieldLengths;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
String mode = obj.getString("mode");
if ("separator".equals(mode)) {
return new ColumnSplitOperation(
engineConfig,
@ -64,7 +63,7 @@ public class ColumnSplitOperation extends EngineDependentOperation {
);
}
}
public ColumnSplitOperation(
JSONObject engineConfig,
String columnName,
@ -75,19 +74,19 @@ public class ColumnSplitOperation extends EngineDependentOperation {
int maxColumns
) {
super(engineConfig);
_columnName = columnName;
_guessCellType = guessCellType;
_removeOriginalColumn = removeOriginalColumn;
_mode = "separator";
_separator = separator;
_regex = regex;
_maxColumns = maxColumns;
_fieldLengths = null;
}
public ColumnSplitOperation(
JSONObject engineConfig,
String columnName,
@ -96,22 +95,22 @@ public class ColumnSplitOperation extends EngineDependentOperation {
int[] fieldLengths
) {
super(engineConfig);
_columnName = columnName;
_guessCellType = guessCellType;
_removeOriginalColumn = removeOriginalColumn;
_mode = "lengths";
_separator = null;
_regex = false;
_maxColumns = -1;
_fieldLengths = fieldLengths;
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -135,53 +134,53 @@ public class ColumnSplitOperation extends EngineDependentOperation {
}
protected String getBriefDescription(Project project) {
return "Split column " + _columnName +
return "Split column " + _columnName +
("separator".equals(_mode) ? " by separator" : " by field lengths");
}
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Engine engine = createEngine(project);
Column column = project.columnModel.getColumnByName(_columnName);
if (column == null) {
throw new Exception("No column named " + _columnName);
}
List<String> columnNames = new ArrayList<String>();
List<Integer> rowIndices = new ArrayList<Integer>(project.rows.size());
List<List<Serializable>> tuples = new ArrayList<List<Serializable>>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
RowVisitor rowVisitor;
if ("lengths".equals(_mode)) {
rowVisitor = new ColumnSplitRowVisitor(column.getCellIndex(), columnNames, rowIndices, tuples) {
protected java.util.List<Serializable> split(String s) {
List<Serializable> results = new ArrayList<Serializable>(_fieldLengths.length + 1);
int lastIndex = 0;
for (int i = 0; i < _fieldLengths.length; i++) {
int from = lastIndex;
int length = _fieldLengths[i];
int to = Math.min(from + length, s.length());
results.add(stringToValue(s.substring(from, to)));
lastIndex = to;
}
return results;
};
};
} else if (_regex) {
Pattern pattern = Pattern.compile(_separator);
rowVisitor = new ColumnSplitRowVisitor(column.getCellIndex(), columnNames, rowIndices, tuples) {
Pattern _pattern;
protected java.util.List<Serializable> split(String s) {
return stringArrayToValueList(_pattern.split(s, _maxColumns));
};
public RowVisitor init(Pattern pattern) {
_pattern = pattern;
return this;
@ -195,13 +194,13 @@ public class ColumnSplitOperation extends EngineDependentOperation {
};
};
}
filteredRows.accept(project, rowVisitor);
String description =
"Split " + rowIndices.size() +
" cell(s) in column " + _columnName +
" into several columns" +
String description =
"Split " + rowIndices.size() +
" cell(s) in column " + _columnName +
" into several columns" +
("separator".equals(_mode) ? " by separator" : " by field lengths");
Change change = new ColumnSplitChange(
@ -211,8 +210,8 @@ public class ColumnSplitOperation extends EngineDependentOperation {
tuples,
_removeOriginalColumn
);
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID, project, description, this, change);
}
@ -222,9 +221,9 @@ public class ColumnSplitOperation extends EngineDependentOperation {
List<String> columnNames;
List<Integer> rowIndices;
List<List<Serializable>> tuples;
int columnNameIndex = 1;
ColumnSplitRowVisitor(
int cellIndex,
List<String> columnNames,
@ -236,27 +235,27 @@ public class ColumnSplitOperation extends EngineDependentOperation {
this.rowIndices = rowIndices;
this.tuples = tuples;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
Object value = row.getCellValue(cellIndex);
if (ExpressionUtils.isNonBlankData(value)) {
String s = value instanceof String ? ((String) value) : value.toString();
List<Serializable> tuple = split(s);
rowIndices.add(rowIndex);
tuples.add(tuple);
for (int i = columnNames.size(); i < tuple.size(); i++) {
while (true) {
String newColumnName = _columnName + " " + columnNameIndex++;
@ -269,21 +268,21 @@ public class ColumnSplitOperation extends EngineDependentOperation {
}
return false;
}
protected List<Serializable> split(String s) {
throw new UnsupportedOperationException();
}
protected Serializable stringToValue(String s) {
return _guessCellType ? ImporterUtilities.parseCellValue(s) : s;
}
protected List<Serializable> stringArrayToValueList(String[] cells) {
List<Serializable> results = new ArrayList<Serializable>(cells.length);
for (String cell : cells) {
results.add(stringToValue(cell));
}
return results;
}
}

101
main/src/com/metaweb/gridworks/operations/column/ExtendDataOperation.java
View File

@ -13,7 +13,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -39,10 +38,10 @@ public class ExtendDataOperation extends EngineDependentOperation {
final protected String _baseColumnName;
final protected JSONObject _extension;
final protected int _columnInsertIndex;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ExtendDataOperation(
engineConfig,
obj.getString("baseColumnName"),
@ -50,15 +49,15 @@ public class ExtendDataOperation extends EngineDependentOperation {
obj.getInt("columnInsertIndex")
);
}
public ExtendDataOperation(
JSONObject engineConfig,
String baseColumnName,
JSONObject extension,
int columnInsertIndex
int columnInsertIndex
) {
super(engineConfig);
_baseColumnName = baseColumnName;
_extension = extension;
_columnInsertIndex = columnInsertIndex;
@ -66,7 +65,7 @@ public class ExtendDataOperation extends EngineDependentOperation {
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -78,24 +77,24 @@ public class ExtendDataOperation extends EngineDependentOperation {
}
protected String getBriefDescription(Project project) {
return "Extend data at index " + _columnInsertIndex +
return "Extend data at index " + _columnInsertIndex +
" based on column " + _baseColumnName;
}
protected String createDescription(Column column, List<CellAtRow> cellsAtRows) {
return "Extend data at index " + _columnInsertIndex +
" based on column " + column.getName() +
return "Extend data at index " + _columnInsertIndex +
" based on column " + column.getName() +
" by filling " + cellsAtRows.size();
}
public Process createProcess(Project project, Properties options) throws Exception {
return new ExtendDataProcess(
project,
project,
getEngineConfig(),
getBriefDescription(null)
);
}
public class ExtendDataProcess extends LongRunningProcess implements Runnable {
final protected Project _project;
final protected JSONObject _engineConfig;
@ -104,21 +103,21 @@ public class ExtendDataOperation extends EngineDependentOperation {
protected FreebaseDataExtensionJob _job;
public ExtendDataProcess(
Project project,
JSONObject engineConfig,
Project project,
JSONObject engineConfig,
String description
) throws JSONException {
super(description);
_project = project;
_engineConfig = engineConfig;
_historyEntryID = HistoryEntry.allocateID();
_job = new FreebaseDataExtensionJob(_extension);
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(hashCode());
writer.key("description"); writer.value(_description);
@ -127,111 +126,111 @@ public class ExtendDataOperation extends EngineDependentOperation {
writer.key("progress"); writer.value(_progress);
writer.endObject();
}
protected Runnable getRunnable() {
return this;
}
protected void populateRowsWithMatches(List<Integer> rowIndices) throws Exception {
Engine engine = new Engine(_project);
engine.initializeFromJSON(_engineConfig);
Column column = _project.columnModel.getColumnByName(_baseColumnName);
if (column == null) {
throw new Exception("No column named " + _baseColumnName);
}
_cellIndex = column.getCellIndex();
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, new RowVisitor() {
List<Integer> _rowIndices;
public RowVisitor init(List<Integer> rowIndices) {
_rowIndices = rowIndices;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex);
}
return false;
}
}.init(rowIndices));
}
protected int extendRows(
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int from,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int from,
int limit,
Map<String, ReconCandidate> reconCandidateMap
) {
Set<String> guids = new HashSet<String>();
int end;
for (end = from; end < limit && guids.size() < 10; end++) {
int index = rowIndices.get(end);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
guids.add(cell.recon.match.topicGUID);
}
Map<String, DataExtension> map = null;
try {
map = _job.extend(guids, reconCandidateMap);
} catch (Exception e) {
map = new HashMap<String, DataExtension>();
}
for (int i = from; i < end; i++) {
int index = rowIndices.get(i);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
String guid = cell.recon.match.topicGUID;
if (map.containsKey(guid)) {
dataExtensions.add(map.get(guid));
} else {
dataExtensions.add(null);
}
}
return end;
}
public void run() {
List<Integer> rowIndices = new ArrayList<Integer>();
List<DataExtension> dataExtensions = new ArrayList<DataExtension>();
try {
populateRowsWithMatches(rowIndices);
} catch (Exception e2) {
// TODO : Not sure what to do here?
e2.printStackTrace();
}
int start = 0;
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
while (start < rowIndices.size()) {
int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
start = end;
_progress = end * 100 / rowIndices.size();
try {
Thread.sleep(200);
@ -241,23 +240,23 @@ public class ExtendDataOperation extends EngineDependentOperation {
}
}
}
if (!_canceled) {
List<String> columnNames = new ArrayList<String>();
for (ColumnInfo info : _job.columns) {
columnNames.add(StringUtils.join(info.names, " - "));
}
List<FreebaseType> columnTypes = new ArrayList<FreebaseType>();
for (ColumnInfo info : _job.columns) {
columnTypes.add(info.expectedType);
}
HistoryEntry historyEntry = ProjectManager.singleton.createHistoryEntry(
HistoryEntry historyEntry = new HistoryEntry(
_historyEntryID,
_project,
_description,
ExtendDataOperation.this,
_project,
_description,
ExtendDataOperation.this,
new DataExtensionChange(
_baseColumnName,
_columnInsertIndex,
@ -267,7 +266,7 @@ public class ExtendDataOperation extends EngineDependentOperation {
dataExtensions,
_historyEntryID)
);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}

105
main/src/com/metaweb/gridworks/operations/recon/ReconOperation.java
View File

@ -10,7 +10,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -36,20 +35,20 @@ import com.metaweb.gridworks.process.Process;
public class ReconOperation extends EngineDependentOperation {
final protected String _columnName;
final protected ReconConfig _reconConfig;
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new ReconOperation(
engineConfig,
engineConfig,
obj.getString("columnName"),
ReconConfig.reconstruct(obj.getJSONObject("config"))
);
}
public ReconOperation(
JSONObject engineConfig,
String columnName,
JSONObject engineConfig,
String columnName,
ReconConfig reconConfig
) {
super(engineConfig);
@ -59,19 +58,19 @@ public class ReconOperation extends EngineDependentOperation {
public Process createProcess(Project project, Properties options) throws Exception {
return new ReconProcess(
project,
project,
getEngineConfig(),
getBriefDescription(null)
);
}
protected String getBriefDescription(Project project) {
return _reconConfig.getBriefDescription(project, _columnName);
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -84,7 +83,7 @@ public class ReconOperation extends EngineDependentOperation {
static protected class ReconEntry {
final public int rowIndex;
final public Cell cell;
public ReconEntry(int rowIndex, Cell cell) {
this.rowIndex = rowIndex;
this.cell = cell;
@ -93,22 +92,22 @@ public class ReconOperation extends EngineDependentOperation {
static protected class JobGroup {
final public ReconJob job;
final public List<ReconEntry> entries = new ArrayList<ReconEntry>();
public JobGroup(ReconJob job) {
this.job = job;
}
}
public class ReconProcess extends LongRunningProcess implements Runnable {
final protected Project _project;
final protected JSONObject _engineConfig;
final protected long _historyEntryID;
protected List<ReconEntry> _entries;
protected int _cellIndex;
public ReconProcess(
Project project,
JSONObject engineConfig,
Project project,
JSONObject engineConfig,
String description
) {
super(description);
@ -116,10 +115,10 @@ public class ReconOperation extends EngineDependentOperation {
_engineConfig = engineConfig;
_historyEntryID = HistoryEntry.allocateID();
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("id"); writer.value(hashCode());
writer.key("description"); writer.value(_description);
@ -160,35 +159,35 @@ public class ReconOperation extends EngineDependentOperation {
writer.endArray();
writer.endObject();
}
protected Runnable getRunnable() {
return this;
}
protected void populateEntries() throws Exception {
Engine engine = new Engine(_project);
engine.initializeFromJSON(_engineConfig);
Column column = _project.columnModel.getColumnByName(_columnName);
if (column == null) {
throw new Exception("No column named " + _columnName);
}
_entries = new ArrayList<ReconEntry>(_project.rows.size());
_cellIndex = column.getCellIndex();
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(_project, new RowVisitor() {
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
if (_cellIndex < row.cells.size()) {
Cell cell = row.cells.get(_cellIndex);
@ -200,7 +199,7 @@ public class ReconOperation extends EngineDependentOperation {
}
});
}
public void run() {
try {
populateEntries();
@ -208,18 +207,18 @@ public class ReconOperation extends EngineDependentOperation {
// TODO : Not sure what to do here?
e2.printStackTrace();
}
Map<Integer, JobGroup> jobKeyToGroup = new HashMap<Integer, JobGroup>();
for (ReconEntry entry : _entries) {
ReconJob job = _reconConfig.createJob(
_project,
entry.rowIndex,
_project.rows.get(entry.rowIndex),
_columnName,
_project,
entry.rowIndex,
_project.rows.get(entry.rowIndex),
_columnName,
entry.cell
);
int key = job.getKey();
JobGroup group = jobKeyToGroup.get(key);
if (group == null) {
@ -228,42 +227,42 @@ public class ReconOperation extends EngineDependentOperation {
}
group.entries.add(entry);
}
List<CellChange> cellChanges = new ArrayList<CellChange>(_entries.size());
List<JobGroup> groups = new ArrayList<JobGroup>(jobKeyToGroup.values());
int batchSize = _reconConfig.getBatchSize();
for (int i = 0; i < groups.size(); i += batchSize) {
int to = Math.min(i + batchSize, groups.size());
List<ReconJob> jobs = new ArrayList<ReconJob>(to - i);
for (int j = i; j < to; j++) {
jobs.add(groups.get(j).job);
}
List<Recon> recons = _reconConfig.batchRecon(jobs, _historyEntryID);
for (int j = i; j < to; j++) {
Recon recon = recons.get(j - i);
List<ReconEntry> entries = groups.get(j).entries;
if (recon != null) {
recon.judgmentBatchSize = entries.size();
}
for (ReconEntry entry : entries) {
Cell oldCell = entry.cell;
Cell newCell = new Cell(oldCell.value, recon);
CellChange cellChange = new CellChange(
entry.rowIndex,
_cellIndex,
oldCell,
entry.rowIndex,
_cellIndex,
oldCell,
newCell
);
cellChanges.add(cellChange);
}
}
_progress = i * 100 / groups.size();
try {
Thread.sleep(50);
@ -273,23 +272,23 @@ public class ReconOperation extends EngineDependentOperation {
}
}
}
if (!_canceled) {
Change reconChange = new ReconChange(
cellChanges,
_columnName,
cellChanges,
_columnName,
_reconConfig,
null
);
HistoryEntry historyEntry = ProjectManager.singleton.createHistoryEntry(
HistoryEntry historyEntry = new HistoryEntry(
_historyEntryID,
_project,
_description,
ReconOperation.this,
_project,
_description,
ReconOperation.this,
reconChange
);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}

25
main/src/com/metaweb/gridworks/operations/row/DenormalizeOperation.java
View File

@ -8,7 +8,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Cell;
@ -23,13 +22,13 @@ public class DenormalizeOperation extends AbstractOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
return new DenormalizeOperation();
}
public DenormalizeOperation() {
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value("Denormalize");
@ -43,37 +42,37 @@ public class DenormalizeOperation extends AbstractOperation {
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
List<Row> newRows = new ArrayList<Row>();
List<Row> oldRows = project.rows;
for (int r = 0; r < oldRows.size(); r++) {
Row oldRow = oldRows.get(r);
Row newRow = null;
RowDependency rd = project.recordModel.getRowDependency(r);
if (rd.cellDependencies != null) {
newRow = oldRow.dup();
for (int c = 0; c < rd.cellDependencies.length; c++) {
CellDependency cd = rd.cellDependencies[c];
if (cd != null) {
int contextRowIndex = cd.rowIndex;
int contextCellIndex = cd.cellIndex;
if (contextRowIndex >= 0 && contextRowIndex < oldRows.size()) {
Row contextRow = oldRows.get(contextRowIndex);
Cell contextCell = contextRow.getCell(contextCellIndex);
newRow.setCell(contextCellIndex, contextCell);
}
}
}
}
newRows.add(newRow != null ? newRow : oldRow);
}
return ProjectManager.singleton.createHistoryEntry(
historyEntryID,
return new HistoryEntry(
historyEntryID,
project,
getBriefDescription(project),
DenormalizeOperation.this,

33
main/src/com/metaweb/gridworks/operations/row/RowFlagOperation.java
View File

@ -8,7 +8,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -28,13 +27,13 @@ public class RowFlagOperation extends EngineDependentOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
boolean flagged = obj.getBoolean("flagged");
return new RowFlagOperation(
engineConfig,
engineConfig,
flagged
);
}
public RowFlagOperation(JSONObject engineConfig, boolean flagged) {
super(engineConfig);
_flagged = flagged;
@ -42,7 +41,7 @@ public class RowFlagOperation extends EngineDependentOperation {
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -57,17 +56,17 @@ public class RowFlagOperation extends EngineDependentOperation {
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Engine engine = createEngine(project);
List<Change> changes = new ArrayList<Change>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, changes));
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
project,
(_flagged ? "Flag" : "Unflag") + " " + changes.size() + " rows",
this,
project,
(_flagged ? "Flag" : "Unflag") + " " + changes.size() + " rows",
this,
new MassChange(changes, false)
);
}
@ -75,26 +74,26 @@ public class RowFlagOperation extends EngineDependentOperation {
protected RowVisitor createRowVisitor(Project project, List<Change> changes) throws Exception {
return new RowVisitor() {
List<Change> changes;
public RowVisitor init(List<Change> changes) {
this.changes = changes;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
if (row.flagged != _flagged) {
RowFlagChange change = new RowFlagChange(rowIndex, _flagged);
changes.add(change);
}
return false;

31
main/src/com/metaweb/gridworks/operations/row/RowRemovalOperation.java
View File

@ -8,7 +8,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -23,19 +22,19 @@ import com.metaweb.gridworks.operations.OperationRegistry;
public class RowRemovalOperation extends EngineDependentOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
return new RowRemovalOperation(
engineConfig
);
}
public RowRemovalOperation(JSONObject engineConfig) {
super(engineConfig);
}
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -49,17 +48,17 @@ public class RowRemovalOperation extends EngineDependentOperation {
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Engine engine = createEngine(project);
List<Integer> rowIndices = new ArrayList<Integer>();
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, rowIndices));
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
project,
"Remove " + rowIndices.size() + " rows",
this,
project,
"Remove " + rowIndices.size() + " rows",
this,
new RowRemovalChange(rowIndices)
);
}
@ -67,25 +66,25 @@ public class RowRemovalOperation extends EngineDependentOperation {
protected RowVisitor createRowVisitor(Project project, List<Integer> rowIndices) throws Exception {
return new RowVisitor() {
List<Integer> rowIndices;
public RowVisitor init(List<Integer> rowIndices) {
this.rowIndices = rowIndices;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
rowIndices.add(rowIndex);
return false;
}
}.init(rowIndices);

35
main/src/com/metaweb/gridworks/operations/row/RowReorderOperation.java
View File

@ -8,7 +8,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.RecordVisitor;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -28,13 +27,13 @@ public class RowReorderOperation extends AbstractOperation {
String mode = obj.getString("mode");
JSONObject sorting = obj.has("sorting") && !obj.isNull("sorting") ?
obj.getJSONObject("sorting") : null;
return new RowReorderOperation(Engine.stringToMode(mode), sorting);
}
final protected Mode _mode;
final protected JSONObject _sorting;
public RowReorderOperation(Mode mode, JSONObject sorting) {
_mode = mode;
_sorting = sorting;
@ -42,7 +41,7 @@ public class RowReorderOperation extends AbstractOperation {
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -58,50 +57,50 @@ public class RowReorderOperation extends AbstractOperation {
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Engine engine = new Engine(project);
engine.setMode(_mode);
List<Integer> rowIndices = new ArrayList<Integer>();
if (_mode == Mode.RowBased) {
RowVisitor visitor = new IndexingVisitor(rowIndices);
if (_sorting != null) {
SortingRowVisitor srv = new SortingRowVisitor(visitor);
srv.initializeFromJSON(project, _sorting);
if (srv.hasCriteria()) {
visitor = srv;
}
}
engine.getAllRows().accept(project, visitor);
} else {
RecordVisitor visitor = new IndexingVisitor(rowIndices);
if (_sorting != null) {
SortingRecordVisitor srv = new SortingRecordVisitor(visitor);
srv.initializeFromJSON(project, _sorting);
if (srv.hasCriteria()) {
visitor = srv;
}
}
engine.getAllRecords().accept(project, visitor);
}
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
project,
"Reorder rows",
this,
project,
"Reorder rows",
this,
new RowReorderChange(rowIndices)
);
}
static protected class IndexingVisitor implements RowVisitor, RecordVisitor {
List<Integer> _indices;
IndexingVisitor(List<Integer> indices) {
_indices = indices;
}
@Override
public void start(Project project) {
}

33
main/src/com/metaweb/gridworks/operations/row/RowStarOperation.java
View File

@ -8,7 +8,6 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
@ -28,13 +27,13 @@ public class RowStarOperation extends EngineDependentOperation {
static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
JSONObject engineConfig = obj.getJSONObject("engineConfig");
boolean starred = obj.getBoolean("starred");
return new RowStarOperation(
engineConfig,
engineConfig,
starred
);
}
public RowStarOperation(JSONObject engineConfig, boolean starred) {
super(engineConfig);
_starred = starred;
@ -42,7 +41,7 @@ public class RowStarOperation extends EngineDependentOperation {
public void write(JSONWriter writer, Properties options)
throws JSONException {
writer.object();
writer.key("op"); writer.value(OperationRegistry.s_opClassToName.get(this.getClass()));
writer.key("description"); writer.value(getBriefDescription(null));
@ -57,17 +56,17 @@ public class RowStarOperation extends EngineDependentOperation {
protected HistoryEntry createHistoryEntry(Project project, long historyEntryID) throws Exception {
Engine engine = createEngine(project);
List<Change> changes = new ArrayList<Change>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, changes));
return ProjectManager.singleton.createHistoryEntry(
return new HistoryEntry(
historyEntryID,
project,
(_starred ? "Star" : "Unstar") + " " + changes.size() + " rows",
this,
project,
(_starred ? "Star" : "Unstar") + " " + changes.size() + " rows",
this,
new MassChange(changes, false)
);
}
@ -75,26 +74,26 @@ public class RowStarOperation extends EngineDependentOperation {
protected RowVisitor createRowVisitor(Project project, List<Change> changes) throws Exception {
return new RowVisitor() {
List<Change> changes;
public RowVisitor init(List<Change> changes) {
this.changes = changes;
return this;
}
@Override
public void start(Project project) {
// nothing to do
}
@Override
public void end(Project project) {
// nothing to do
}
public boolean visit(Project project, int rowIndex, Row row) {
if (row.starred != _starred) {
RowStarChange change = new RowStarChange(rowIndex, _starred);
changes.add(change);
}
return false;