s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactoring to support abstract operations.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@40 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-02-04 20:51:44 +00:00
parent 1804291631
commit bacb71ab6d
18 changed files with 856 additions and 670 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/com/metaweb/gridworks/commands/Command.java
View File

@ -170,6 +170,14 @@ public abstract class Command {
return a; return a;
} }
protected JSONObject getEngineConfig(HttpServletRequest request) throws Exception {
String json = request.getParameter("engine");
if (json != null) {
return jsonStringToObject(json);
}
return null;
}
protected Engine getEngine(HttpServletRequest request, Project project) throws Exception { protected Engine getEngine(HttpServletRequest request, Project project) throws Exception {
Engine engine = new Engine(project); Engine engine = new Engine(project);
String json = request.getParameter("engine"); String json = request.getParameter("engine");

38
src/main/java/com/metaweb/gridworks/commands/EngineDependentCommand.java Normal file
View File

@ -0,0 +1,38 @@
package com.metaweb.gridworks.commands;
import java.io.IOException;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.process.Process;
abstract public class EngineDependentCommand extends Command {
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
Project project = getProject(request);
AbstractOperation op = createOperation(request, getEngineConfig(request));
Process process = op.createProcess(project, new Properties());
boolean done = project.processManager.queueProcess(process);
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
} catch (Exception e) {
respondException(response, e);
}
}
abstract protected AbstractOperation createOperation(
HttpServletRequest request, JSONObject engineConfig) throws Exception;
}

99
src/main/java/com/metaweb/gridworks/commands/edit/DoTextTransformCommand.java
View File

@ -1,99 +1,22 @@
package com.metaweb.gridworks.commands.edit; package com.metaweb.gridworks.commands.edit;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.json.JSONObject;
import com.metaweb.gridworks.browsing.Engine; import com.metaweb.gridworks.commands.EngineDependentCommand;
import com.metaweb.gridworks.browsing.FilteredRows; import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.browsing.RowVisitor; import com.metaweb.gridworks.model.operations.TextTransformOperation;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.expr.Parser;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.changes.CellChange;
import com.metaweb.gridworks.model.changes.MassCellChange;
import com.metaweb.gridworks.process.QuickHistoryEntryProcess;
public class DoTextTransformCommand extends Command {
public class DoTextTransformCommand extends EngineDependentCommand {
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) protected AbstractOperation createOperation(HttpServletRequest request,
throws ServletException, IOException { JSONObject engineConfig) throws Exception {
try { int cellIndex = Integer.parseInt(request.getParameter("cell"));
Project project = getProject(request); String expression = request.getParameter("expression");
Engine engine = getEngine(request, project);
int cellIndex = Integer.parseInt(request.getParameter("cell")); return new TextTransformOperation(engineConfig, cellIndex, expression);
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
if (column == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No such column\" }");
return;
}
String columnName = column.getHeaderLabel();
String expression = request.getParameter("expression");
Evaluable eval = new Parser(expression).getExpression();
Properties bindings = ExpressionUtils.createBindings(project);
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, new RowVisitor() {
int cellIndex;
Properties bindings;
List<CellChange> cellChanges;
Evaluable eval;
public RowVisitor init(int cellIndex, Properties bindings, List<CellChange> cellChanges, Evaluable eval) {
this.cellIndex = cellIndex;
this.bindings = bindings;
this.cellChanges = cellChanges;
this.eval = eval;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell.value != null) {
ExpressionUtils.bind(bindings, row, cell);
Cell newCell = new Cell(eval.evaluate(bindings), cell.recon);
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
}
return false;
}
}.init(cellIndex, bindings, cellChanges, eval));
MassCellChange massCellChange = new MassCellChange(cellChanges, cellIndex);
HistoryEntry historyEntry = new HistoryEntry(
project, "Text transform on " + columnName + ": " + expression, massCellChange);
boolean done = project.processManager.queueProcess(
new QuickHistoryEntryProcess(project, historyEntry));
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
} catch (Exception e) {
respondException(response, e);
}
} }
} }

87
src/main/java/com/metaweb/gridworks/commands/recon/ApproveNewReconcileCommand.java
View File

@ -1,88 +1,21 @@
package com.metaweb.gridworks.commands.recon; package com.metaweb.gridworks.commands.recon;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.metaweb.gridworks.browsing.Engine; import org.json.JSONObject;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.Recon.Judgment;
import com.metaweb.gridworks.model.changes.CellChange;
import com.metaweb.gridworks.model.changes.MassCellChange;
import com.metaweb.gridworks.process.QuickHistoryEntryProcess;
public class ApproveNewReconcileCommand extends Command { import com.metaweb.gridworks.commands.EngineDependentCommand;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.operations.ApproveNewReconOperation;
public class ApproveNewReconcileCommand extends EngineDependentCommand {
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) protected AbstractOperation createOperation(HttpServletRequest request,
throws ServletException, IOException { JSONObject engineConfig) throws Exception {
try { int cellIndex = Integer.parseInt(request.getParameter("cell"));
Project project = getProject(request);
Engine engine = getEngine(request, project);
int cellIndex = Integer.parseInt(request.getParameter("cell")); return new ApproveNewReconOperation(engineConfig, cellIndex);
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
if (column == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No such column\" }");
return;
}
String columnName = column.getHeaderLabel();
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges) {
this.cellIndex = cellIndex;
this.cellChanges = cellChanges;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
Cell newCell = new Cell(
cell.value,
cell.recon != null ? cell.recon.dup() : new Recon()
);
newCell.recon.match = null;
newCell.recon.judgment = Judgment.New;
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
return false;
}
}.init(cellIndex, cellChanges));
MassCellChange massCellChange = new MassCellChange(cellChanges, cellIndex);
HistoryEntry historyEntry = new HistoryEntry(
project, "Approve new topics for " + columnName, massCellChange);
boolean done = project.processManager.queueProcess(
new QuickHistoryEntryProcess(project, historyEntry));
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
} catch (Exception e) {
respondException(response, e);
}
} }
} }

87
src/main/java/com/metaweb/gridworks/commands/recon/ApproveReconcileCommand.java
View File

@ -1,88 +1,21 @@
package com.metaweb.gridworks.commands.recon; package com.metaweb.gridworks.commands.recon;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.metaweb.gridworks.browsing.Engine; import org.json.JSONObject;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.Recon.Judgment;
import com.metaweb.gridworks.model.changes.CellChange;
import com.metaweb.gridworks.model.changes.MassCellChange;
import com.metaweb.gridworks.process.QuickHistoryEntryProcess;
public class ApproveReconcileCommand extends Command { import com.metaweb.gridworks.commands.EngineDependentCommand;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.operations.ApproveReconOperation;
public class ApproveReconcileCommand extends EngineDependentCommand {
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) protected AbstractOperation createOperation(HttpServletRequest request,
throws ServletException, IOException { JSONObject engineConfig) throws Exception {
try { int cellIndex = Integer.parseInt(request.getParameter("cell"));
Project project = getProject(request);
Engine engine = getEngine(request, project);
int cellIndex = Integer.parseInt(request.getParameter("cell")); return new ApproveReconOperation(engineConfig, cellIndex);
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
if (column == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No such column\" }");
return;
}
String columnName = column.getHeaderLabel();
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges) {
this.cellIndex = cellIndex;
this.cellChanges = cellChanges;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell.recon != null && cell.recon.candidates.size() > 0) {
Cell newCell = new Cell(
cell.value,
cell.recon.dup()
);
newCell.recon.match = newCell.recon.candidates.get(0);
newCell.recon.judgment = Judgment.Approve;
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
}
return false;
}
}.init(cellIndex, cellChanges));
MassCellChange massCellChange = new MassCellChange(cellChanges, cellIndex);
HistoryEntry historyEntry = new HistoryEntry(
project, "Approve best recon candidates for " + columnName, massCellChange);
boolean done = project.processManager.queueProcess(
new QuickHistoryEntryProcess(project, historyEntry));
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
} catch (Exception e) {
respondException(response, e);
}
} }
} }

79
src/main/java/com/metaweb/gridworks/commands/recon/DiscardReconcileCommand.java
View File

@ -1,81 +1,20 @@
package com.metaweb.gridworks.commands.recon; package com.metaweb.gridworks.commands.recon;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.metaweb.gridworks.browsing.Engine; import org.json.JSONObject;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.changes.CellChange;
import com.metaweb.gridworks.model.changes.MassCellChange;
import com.metaweb.gridworks.process.QuickHistoryEntryProcess;
public class DiscardReconcileCommand extends Command { import com.metaweb.gridworks.commands.EngineDependentCommand;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.operations.DiscardReconOperation;
public class DiscardReconcileCommand extends EngineDependentCommand {
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) protected AbstractOperation createOperation(HttpServletRequest request,
throws ServletException, IOException { JSONObject engineConfig) throws Exception {
try { int cellIndex = Integer.parseInt(request.getParameter("cell"));
Project project = getProject(request);
Engine engine = getEngine(request, project);
int cellIndex = Integer.parseInt(request.getParameter("cell")); return new DiscardReconOperation(engineConfig, cellIndex);
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
if (column == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No such column\" }");
return;
}
String columnName = column.getHeaderLabel();
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges) {
this.cellIndex = cellIndex;
this.cellChanges = cellChanges;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
Cell newCell = new Cell(cell.value, null);
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
return false;
}
}.init(cellIndex, cellChanges));
MassCellChange massCellChange = new MassCellChange(cellChanges, cellIndex);
HistoryEntry historyEntry = new HistoryEntry(
project, "Discard recon results for " + columnName, massCellChange);
boolean done = project.processManager.queueProcess(
new QuickHistoryEntryProcess(project, historyEntry));
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
} catch (Exception e) {
respondException(response, e);
}
} }
} }

83
src/main/java/com/metaweb/gridworks/commands/recon/ReconcileCommand.java
View File

@ -1,83 +1,22 @@
package com.metaweb.gridworks.commands.recon; package com.metaweb.gridworks.commands.recon;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.metaweb.gridworks.browsing.Engine; import org.json.JSONObject;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.process.ReconProcess;
import com.metaweb.gridworks.process.ReconProcess.ReconEntry;
public class ReconcileCommand extends Command { import com.metaweb.gridworks.commands.EngineDependentCommand;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.operations.ReconOperation;
public class ReconcileCommand extends EngineDependentCommand {
@Override @Override
public void doPost(HttpServletRequest request, HttpServletResponse response) protected AbstractOperation createOperation(HttpServletRequest request,
throws ServletException, IOException { JSONObject engineConfig) throws Exception {
try { int cellIndex = Integer.parseInt(request.getParameter("cell"));
Project project = getProject(request); String typeID = request.getParameter("type");
Engine engine = getEngine(request, project);
int cellIndex = Integer.parseInt(request.getParameter("cell")); return new ReconOperation(engineConfig, cellIndex, typeID);
Column column = project.columnModel.getColumnByCellIndex(cellIndex);
if (column == null) {
respond(response, "{ \"code\" : \"error\", \"message\" : \"No such column\" }");
return;
}
String columnName = column.getHeaderLabel();
String typeID = request.getParameter("type");
List<ReconEntry> entries = new ArrayList<ReconEntry>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, new RowVisitor() {
int cellIndex;
List<ReconEntry> entries;
public RowVisitor init(int cellIndex, List<ReconEntry> entries) {
this.cellIndex = cellIndex;
this.entries = entries;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell.value != null) {
entries.add(new ReconEntry(rowIndex, cell));
}
}
return false;
}
}.init(cellIndex, entries));
ReconProcess process = new ReconProcess(
project,
"Reconcile " + columnName + " to type " + typeID,
cellIndex,
entries,
typeID
);
boolean done = project.processManager.queueProcess(process);
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
} catch (Exception e) {
respondException(response, e);
}
} }
} }

13
src/main/java/com/metaweb/gridworks/history/HistoryEntry.java
View File

@ -16,22 +16,25 @@ import org.json.JSONWriter;
import com.metaweb.gridworks.Jsonizable; import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.ProjectManager; import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Project; import com.metaweb.gridworks.model.Project;
public class HistoryEntry implements Serializable, Jsonizable { public class HistoryEntry implements Serializable, Jsonizable {
private static final long serialVersionUID = 532766467813930262L; private static final long serialVersionUID = 532766467813930262L;
public long id; final public long id;
public long projectID; final public long projectID;
public String description; final public String description;
public Date time; final public AbstractOperation operation;
final public Date time;
transient protected Change _change; transient protected Change _change;
public HistoryEntry(Project project, String description, Change change) { public HistoryEntry(Project project, String description, AbstractOperation operation, Change change) {
this.id = Math.round(Math.random() * 1000000) + System.currentTimeMillis(); this.id = Math.round(Math.random() * 1000000) + System.currentTimeMillis();
this.projectID = project.id; this.projectID = project.id;
this.description = description; this.description = description;
this.operation = operation;
this.time = new Date(); this.time = new Date();
_change = change; _change = change;

15
src/main/java/com/metaweb/gridworks/model/AbstractOperation.java Normal file
View File

@ -0,0 +1,15 @@
package com.metaweb.gridworks.model;
import java.io.Serializable;
import java.util.Properties;
import com.metaweb.gridworks.Jsonizable;
import com.metaweb.gridworks.process.Process;
/*
* An abstract operation can be applied to different but similar
* projects.
*/
public interface AbstractOperation extends Serializable, Jsonizable {
public Process createProcess(Project project, Properties options) throws Exception;
}

73
src/main/java/com/metaweb/gridworks/model/operations/ApproveNewReconOperation.java Normal file
View File

@ -0,0 +1,73 @@
package com.metaweb.gridworks.model.operations;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.Recon.Judgment;
import com.metaweb.gridworks.model.changes.CellChange;
public class ApproveNewReconOperation extends EngineDependentMassCellOperation {
private static final long serialVersionUID = -5205694623711144436L;
public ApproveNewReconOperation(JSONObject engineConfig, int cellIndex) {
super(engineConfig, cellIndex);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
// TODO Auto-generated method stub
}
@Override
protected String createDescription(Column column,
List<CellChange> cellChanges) {
return "Approve new topics for " + cellChanges.size() +
" cells in column " + column.getHeaderLabel();
}
@Override
protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges) throws Exception {
// TODO Auto-generated method stub
return new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges) {
this.cellIndex = cellIndex;
this.cellChanges = cellChanges;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
Cell newCell = new Cell(
cell.value,
cell.recon != null ? cell.recon.dup() : new Recon()
);
newCell.recon.match = null;
newCell.recon.judgment = Judgment.New;
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
return false;
}
}.init(_cellIndex, cellChanges);
}
}

73
src/main/java/com/metaweb/gridworks/model/operations/ApproveReconOperation.java Normal file
View File

@ -0,0 +1,73 @@
package com.metaweb.gridworks.model.operations;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.Recon.Judgment;
import com.metaweb.gridworks.model.changes.CellChange;
public class ApproveReconOperation extends EngineDependentMassCellOperation {
private static final long serialVersionUID = 5393888241057341155L;
public ApproveReconOperation(JSONObject engineConfig, int cellIndex) {
super(engineConfig, cellIndex);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
// TODO Auto-generated method stub
}
@Override
protected String createDescription(Column column,
List<CellChange> cellChanges) {
return "Approve best candidates for " + cellChanges.size() +
" cells in column " + column.getHeaderLabel();
}
@Override
protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges) throws Exception {
// TODO Auto-generated method stub
return new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges) {
this.cellIndex = cellIndex;
this.cellChanges = cellChanges;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell.recon != null && cell.recon.candidates.size() > 0) {
Cell newCell = new Cell(
cell.value,
cell.recon.dup()
);
newCell.recon.match = newCell.recon.candidates.get(0);
newCell.recon.judgment = Judgment.Approve;
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
}
return false;
}
}.init(_cellIndex, cellChanges);
}
}

66
src/main/java/com/metaweb/gridworks/model/operations/DiscardReconOperation.java Normal file
View File

@ -0,0 +1,66 @@
package com.metaweb.gridworks.model.operations;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.changes.CellChange;
public class DiscardReconOperation extends EngineDependentMassCellOperation {
private static final long serialVersionUID = 6799029731665369179L;
public DiscardReconOperation(JSONObject engineConfig, int cellIndex) {
super(engineConfig, cellIndex);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
// TODO Auto-generated method stub
}
@Override
protected String createDescription(Column column,
List<CellChange> cellChanges) {
return "Discard recon results for " + cellChanges.size() +
" cells in column " + column.getHeaderLabel();
}
@Override
protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges) throws Exception {
// TODO Auto-generated method stub
return new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges) {
this.cellIndex = cellIndex;
this.cellChanges = cellChanges;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
Cell newCell = new Cell(cell.value, null);
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
return false;
}
}.init(_cellIndex, cellChanges);
}
}

55
src/main/java/com/metaweb/gridworks/model/operations/EngineDependentMassCellOperation.java Normal file
View File

@ -0,0 +1,55 @@
package com.metaweb.gridworks.model.operations;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.json.JSONObject;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.changes.CellChange;
import com.metaweb.gridworks.model.changes.MassCellChange;
import com.metaweb.gridworks.process.Process;
import com.metaweb.gridworks.process.QuickHistoryEntryProcess;
abstract public class EngineDependentMassCellOperation extends EngineDependentOperation {
private static final long serialVersionUID = -8962461328087299452L;
final protected int _cellIndex;
protected EngineDependentMassCellOperation(JSONObject engineConfig, int cellIndex) {
super(engineConfig);
_cellIndex = cellIndex;
}
@Override
public Process createProcess(Project project, Properties options) throws Exception {
Engine engine = createEngine(project);
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
if (column == null) {
throw new Exception("No column corresponding to cell index " + _cellIndex);
}
List<CellChange> cellChanges = new ArrayList<CellChange>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, createRowVisitor(project, cellChanges));
String description = createDescription(column, cellChanges);
MassCellChange massCellChange = new MassCellChange(cellChanges, _cellIndex);
HistoryEntry historyEntry = new HistoryEntry(
project, description, this, massCellChange);
return new QuickHistoryEntryProcess(project, historyEntry);
}
abstract protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges) throws Exception;
abstract protected String createDescription(Column column, List<CellChange> cellChanges);
}

37
src/main/java/com/metaweb/gridworks/model/operations/EngineDependentOperation.java Normal file
View File

@ -0,0 +1,37 @@
package com.metaweb.gridworks.model.operations;
import org.json.JSONException;
import org.json.JSONObject;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.model.AbstractOperation;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.ParsingUtilities;
abstract public class EngineDependentOperation implements AbstractOperation {
final private String _engineConfigString;
transient protected JSONObject _engineConfig;
protected EngineDependentOperation(JSONObject engineConfig) {
_engineConfig = engineConfig;
_engineConfigString = engineConfig.toString();
}
protected Engine createEngine(Project project) throws Exception {
Engine engine = new Engine(project);
engine.initializeFromJSON(getEngineConfig());
return engine;
}
protected JSONObject getEngineConfig() {
if (_engineConfig == null) {
try {
_engineConfig = ParsingUtilities.evaluateJsonStringToObject(_engineConfigString);
} catch (JSONException e) {
// ignore
}
}
return _engineConfig;
}
}

344
src/main/java/com/metaweb/gridworks/model/operations/ReconOperation.java Normal file
View File

@ -0,0 +1,344 @@
package com.metaweb.gridworks.model.operations;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.Engine;
import com.metaweb.gridworks.browsing.FilteredRows;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.ReconCandidate;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.changes.CellChange;
import com.metaweb.gridworks.model.changes.MassCellChange;
import com.metaweb.gridworks.process.LongRunningProcess;
import com.metaweb.gridworks.process.Process;
import com.metaweb.gridworks.util.ParsingUtilities;
public class ReconOperation extends EngineDependentOperation {
private static final long serialVersionUID = 838795186905314865L;
final protected int _cellIndex;
final protected String _typeID;
public ReconOperation(JSONObject engineConfig, int cellIndex, String typeID) {
super(engineConfig);
_cellIndex = cellIndex;
_typeID = typeID;
}
@Override
public Process createProcess(Project project, Properties options) throws Exception {
Engine engine = createEngine(project);
Column column = project.columnModel.getColumnByCellIndex(_cellIndex);
if (column == null) {
throw new Exception("No column corresponding to cell index " + _cellIndex);
}
List<ReconEntry> entries = new ArrayList<ReconEntry>(project.rows.size());
FilteredRows filteredRows = engine.getAllFilteredRows();
filteredRows.accept(project, new RowVisitor() {
int cellIndex;
List<ReconEntry> entries;
public RowVisitor init(int cellIndex, List<ReconEntry> entries) {
this.cellIndex = cellIndex;
this.entries = entries;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell.value != null) {
entries.add(new ReconEntry(rowIndex, cell));
}
}
return false;
}
}.init(_cellIndex, entries));
String description =
"Reconcile " + entries.size() +
" cells in column " + column.getHeaderLabel() +
" to type " + _typeID;
return new ReconProcess(project, description, entries);
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
// TODO Auto-generated method stub
}
static protected class ReconEntry {
final public int rowIndex;
final public Cell cell;
public ReconEntry(int rowIndex, Cell cell) {
this.rowIndex = rowIndex;
this.cell = cell;
}
}
public class ReconProcess extends LongRunningProcess implements Runnable {
final protected Project _project;
final protected List<ReconEntry> _entries;
public ReconProcess(Project project, String description, List<ReconEntry> entries) {
super(description);
_project = project;
_entries = entries;
}
@Override
protected Runnable getRunnable() {
return this;
}
@Override
public void run() {
Map<String, List<ReconEntry>> valueToEntries = new HashMap<String, List<ReconEntry>>();
for (ReconEntry entry : _entries) {
Object value = entry.cell.value;
if (value != null && value instanceof String) {
List<ReconEntry> entries2;
if (valueToEntries.containsKey(value)) {
entries2 = valueToEntries.get(value);
} else {
entries2 = new LinkedList<ReconEntry>();
valueToEntries.put((String) value, entries2);
}
entries2.add(entry);
}
}
List<CellChange> cellChanges = new ArrayList<CellChange>(_entries.size());
List<String> values = new ArrayList<String>(valueToEntries.keySet());
for (int i = 0; i < values.size(); i += 10) {
try {
recon(valueToEntries, values, i, Math.min(i + 10, values.size()), cellChanges);
} catch (JSONException e1) {
e1.printStackTrace();
}
_progress = i * 100 / values.size();
try {
Thread.sleep(50);
} catch (InterruptedException e) {
if (_canceled) {
break;
}
}
}
MassCellChange massCellChange = new MassCellChange(cellChanges, _cellIndex);
HistoryEntry historyEntry = new HistoryEntry(
_project,
_description,
ReconOperation.this,
massCellChange
);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}
protected void recon(
Map<String, List<ReconEntry>> valueToEntries,
List<String> values,
int from,
int to,
List<CellChange> cellChanges
) throws JSONException {
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
for (int i = 0; from + i < to; i++) {
jsonWriter.key("q" + i + ":search");
jsonWriter.object();
jsonWriter.key("query"); jsonWriter.value(values.get(from + i));
jsonWriter.key("limit"); jsonWriter.value(5);
jsonWriter.key("type"); jsonWriter.value(_typeID);
jsonWriter.key("type_strict"); jsonWriter.value("should");
jsonWriter.key("indent"); jsonWriter.value(1);
jsonWriter.key("type_exclude"); jsonWriter.value("/common/image");
jsonWriter.key("domain_exclude"); jsonWriter.value("/freebase");
jsonWriter.endObject();
}
jsonWriter.endObject();
StringBuffer sb = new StringBuffer();
sb.append("http://api.freebase.com/api/service/search?indent=1&queries=");
sb.append(ParsingUtilities.encode(stringWriter.toString()));
try {
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
for (int i = 0; from + i < to; i++) {
String value = values.get(from + i);
String key = "q" + i + ":search";
if (!o.has(key)) {
continue;
}
JSONObject o2 = o.getJSONObject(key);
if (!(o2.has("result"))) {
continue;
}
JSONArray results = o2.getJSONArray("result");
Recon recon = createRecon(value, results);
for (ReconEntry entry : valueToEntries.get(value)) {
Cell oldCell = entry.cell;
Cell newCell = new Cell(oldCell.value, recon);
CellChange cellChange = new CellChange(
entry.rowIndex,
_cellIndex,
oldCell,
newCell
);
cellChanges.add(cellChange);
}
}
} finally {
is.close();
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
protected Recon createRecon(String text, JSONArray results) throws JSONException {
Recon recon = new Recon();
int length = results.length();
for (int i = 0; i < length && recon.candidates.size() < 3; i++) {
JSONObject result = results.getJSONObject(i);
if (!result.has("name")) {
continue;
}
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getJSONObject(j).getString("id");
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("guid"),
result.getString("name"),
typeIDs,
result.getDouble("relevance:score")
);
// best match
if (i == 0) {
recon.features.put("nameMatch", text.equalsIgnoreCase(candidate.topicName));
recon.features.put("nameLevenshtein", StringUtils.getLevenshteinDistance(text, candidate.topicName));
recon.features.put("nameWordDistance", wordDistance(text, candidate.topicName));
recon.features.put("typeMatch", false);
for (String typeID : candidate.typeIDs) {
if (_typeID.equals(typeID)) {
recon.features.put("typeMatch", true);
break;
}
}
}
recon.candidates.add(candidate);
}
return recon;
}
}
static protected double wordDistance(String s1, String s2) {
Set<String> words1 = breakWords(s1);
Set<String> words2 = breakWords(s2);
return words1.size() >= words2.size() ? wordDistance(words1, words2) : wordDistance(words2, words1);
}
static protected double wordDistance(Set<String> longWords, Set<String> shortWords) {
double common = 0;
for (String word : shortWords) {
if (longWords.contains(word)) {
common++;
}
}
return common / longWords.size();
}
static protected Set<String> s_stopWords;
static {
s_stopWords = new HashSet<String>();
s_stopWords.add("the");
s_stopWords.add("a");
s_stopWords.add("and");
s_stopWords.add("of");
s_stopWords.add("on");
s_stopWords.add("in");
s_stopWords.add("at");
s_stopWords.add("by");
}
static protected Set<String> breakWords(String s) {
String[] words = s.toLowerCase().split("\\s+");
Set<String> set = new HashSet<String>(words.length);
for (String word : words) {
if (!s_stopWords.contains(word)) {
set.add(word);
}
}
return set;
}
}

81
src/main/java/com/metaweb/gridworks/model/operations/TextTransformOperation.java Normal file
View File

@ -0,0 +1,81 @@
package com.metaweb.gridworks.model.operations;
import java.util.List;
import java.util.Properties;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.browsing.RowVisitor;
import com.metaweb.gridworks.expr.Evaluable;
import com.metaweb.gridworks.expr.ExpressionUtils;
import com.metaweb.gridworks.expr.Parser;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.model.changes.CellChange;
public class TextTransformOperation extends EngineDependentMassCellOperation {
private static final long serialVersionUID = -7698202759999537298L;
final protected String _expression;
public TextTransformOperation(JSONObject engineConfig, int cellIndex, String expression) {
super(engineConfig, cellIndex);
_expression = expression;
}
@Override
public void write(JSONWriter writer, Properties options)
throws JSONException {
// TODO Auto-generated method stub
}
@Override
protected String createDescription(Column column,
List<CellChange> cellChanges) {
return "Text transform on " + cellChanges.size() +
" cells in column " + column.getHeaderLabel() + ": " + _expression;
}
@Override
protected RowVisitor createRowVisitor(Project project, List<CellChange> cellChanges) throws Exception {
Evaluable eval = new Parser(_expression).getExpression();
Properties bindings = ExpressionUtils.createBindings(project);
return new RowVisitor() {
int cellIndex;
Properties bindings;
List<CellChange> cellChanges;
Evaluable eval;
public RowVisitor init(int cellIndex, Properties bindings, List<CellChange> cellChanges, Evaluable eval) {
this.cellIndex = cellIndex;
this.bindings = bindings;
this.cellChanges = cellChanges;
this.eval = eval;
return this;
}
@Override
public boolean visit(Project project, int rowIndex, Row row) {
if (cellIndex < row.cells.size()) {
Cell cell = row.cells.get(cellIndex);
if (cell.value != null) {
ExpressionUtils.bind(bindings, row, cell);
Cell newCell = new Cell(eval.evaluate(bindings), cell.recon);
CellChange cellChange = new CellChange(rowIndex, cellIndex, cell, newCell);
cellChanges.add(cellChange);
}
}
return false;
}
}.init(_cellIndex, bindings, cellChanges, eval);
}
}

274
src/main/java/com/metaweb/gridworks/process/ReconProcess.java
View File

@ -1,274 +0,0 @@
package com.metaweb.gridworks.process;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONWriter;
import com.metaweb.gridworks.history.HistoryEntry;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.ReconCandidate;
import com.metaweb.gridworks.model.changes.CellChange;
import com.metaweb.gridworks.model.changes.MassCellChange;
import com.metaweb.gridworks.util.ParsingUtilities;
public class ReconProcess extends LongRunningProcess implements Runnable {
static public class ReconEntry {
final public int rowIndex;
final public Cell cell;
public ReconEntry(int rowIndex, Cell cell) {
this.rowIndex = rowIndex;
this.cell = cell;
}
}
final protected Project _project;
final protected int _cellIndex;
final protected List<ReconEntry> _entries;
final protected String _typeID;
public ReconProcess(Project project, String description, int cellIndex, List<ReconEntry> entries, String typeID) {
super(description);
_project = project;
_cellIndex = cellIndex;
_entries = entries;
_typeID = typeID;
}
@Override
protected Runnable getRunnable() {
return this;
}
@Override
public void run() {
Map<String, List<ReconEntry>> valueToEntries = new HashMap<String, List<ReconEntry>>();
for (ReconEntry entry : _entries) {
Object value = entry.cell.value;
if (value != null && value instanceof String) {
List<ReconEntry> entries2;
if (valueToEntries.containsKey(value)) {
entries2 = valueToEntries.get(value);
} else {
entries2 = new LinkedList<ReconEntry>();
valueToEntries.put((String) value, entries2);
}
entries2.add(entry);
}
}
List<CellChange> cellChanges = new ArrayList<CellChange>(_entries.size());
List<String> values = new ArrayList<String>(valueToEntries.keySet());
for (int i = 0; i < values.size(); i += 10) {
try {
recon(valueToEntries, values, i, Math.min(i + 10, values.size()), cellChanges);
} catch (JSONException e1) {
e1.printStackTrace();
}
_progress = i * 100 / values.size();
try {
Thread.sleep(50);
} catch (InterruptedException e) {
if (_canceled) {
break;
}
}
}
MassCellChange massCellChange = new MassCellChange(cellChanges, _cellIndex);
HistoryEntry historyEntry = new HistoryEntry(_project, _description, massCellChange);
_project.history.addEntry(historyEntry);
_project.processManager.onDoneProcess(this);
}
protected void recon(
Map<String, List<ReconEntry>> valueToEntries,
List<String> values,
int from,
int to,
List<CellChange> cellChanges
) throws JSONException {
StringWriter stringWriter = new StringWriter();
JSONWriter jsonWriter = new JSONWriter(stringWriter);
jsonWriter.object();
for (int i = 0; from + i < to; i++) {
jsonWriter.key("q" + i + ":search");
jsonWriter.object();
jsonWriter.key("query"); jsonWriter.value(values.get(from + i));
jsonWriter.key("limit"); jsonWriter.value(5);
jsonWriter.key("type"); jsonWriter.value(_typeID);
jsonWriter.key("type_strict"); jsonWriter.value("should");
jsonWriter.key("indent"); jsonWriter.value(1);
jsonWriter.key("type_exclude"); jsonWriter.value("/common/image");
jsonWriter.key("domain_exclude"); jsonWriter.value("/freebase");
jsonWriter.endObject();
}
jsonWriter.endObject();
StringBuffer sb = new StringBuffer();
sb.append("http://api.freebase.com/api/service/search?indent=1&queries=");
sb.append(ParsingUtilities.encode(stringWriter.toString()));
try {
URL url = new URL(sb.toString());
URLConnection connection = url.openConnection();
connection.setConnectTimeout(5000);
connection.connect();
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
JSONObject o = ParsingUtilities.evaluateJsonStringToObject(s);
for (int i = 0; from + i < to; i++) {
String value = values.get(from + i);
String key = "q" + i + ":search";
if (!o.has(key)) {
continue;
}
JSONObject o2 = o.getJSONObject(key);
if (!(o2.has("result"))) {
continue;
}
JSONArray results = o2.getJSONArray("result");
Recon recon = createRecon(value, results);
for (ReconEntry entry : valueToEntries.get(value)) {
Cell oldCell = entry.cell;
Cell newCell = new Cell(oldCell.value, recon);
CellChange cellChange = new CellChange(
entry.rowIndex,
_cellIndex,
oldCell,
newCell
);
cellChanges.add(cellChange);
}
}
} finally {
is.close();
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
protected Recon createRecon(String text, JSONArray results) throws JSONException {
Recon recon = new Recon();
int length = results.length();
for (int i = 0; i < length && recon.candidates.size() < 3; i++) {
JSONObject result = results.getJSONObject(i);
if (!result.has("name")) {
continue;
}
JSONArray types = result.getJSONArray("type");
String[] typeIDs = new String[types.length()];
for (int j = 0; j < typeIDs.length; j++) {
typeIDs[j] = types.getJSONObject(j).getString("id");
}
ReconCandidate candidate = new ReconCandidate(
result.getString("id"),
result.getString("guid"),
result.getString("name"),
typeIDs,
result.getDouble("relevance:score")
);
// best match
if (i == 0) {
recon.features.put("nameMatch", text.equalsIgnoreCase(candidate.topicName));
recon.features.put("nameLevenshtein", StringUtils.getLevenshteinDistance(text, candidate.topicName));
recon.features.put("nameWordDistance", wordDistance(text, candidate.topicName));
recon.features.put("typeMatch", false);
for (String typeID : candidate.typeIDs) {
if (_typeID.equals(typeID)) {
recon.features.put("typeMatch", true);
break;
}
}
}
recon.candidates.add(candidate);
}
return recon;
}
protected double wordDistance(String s1, String s2) {
Set<String> words1 = breakWords(s1);
Set<String> words2 = breakWords(s2);
return words1.size() >= words2.size() ? wordDistance(words1, words2) : wordDistance(words2, words1);
}
protected double wordDistance(Set<String> longWords, Set<String> shortWords) {
double common = 0;
for (String word : shortWords) {
if (longWords.contains(word)) {
common++;
}
}
return common / longWords.size();
}
static protected Set<String> s_stopWords;
static {
s_stopWords = new HashSet<String>();
s_stopWords.add("the");
s_stopWords.add("a");
s_stopWords.add("and");
s_stopWords.add("of");
s_stopWords.add("on");
s_stopWords.add("in");
s_stopWords.add("at");
s_stopWords.add("by");
}
protected Set<String> breakWords(String s) {
String[] words = s.toLowerCase().split("\\s+");
Set<String> set = new HashSet<String>(words.length);
for (String word : words) {
if (!s_stopWords.contains(word)) {
set.add(word);
}
}
return set;
}
}