s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

detab + dedos for java files (no functional changes)

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@594 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-05-04 23:24:48 +00:00
parent d35930fb66
commit 92ecc0c0f5
198 changed files with 17612 additions and 17612 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
src/main/java/com/metaweb/gridworks/ProjectManager.java
View File

@ -34,7 +34,7 @@ public class ProjectManager {
protected File _workspaceDir; protected File _workspaceDir;
protected Map<Long, ProjectMetadata> _projectsMetadata; protected Map<Long, ProjectMetadata> _projectsMetadata;
protected List<String> _expressions; protected List<String> _expressions;
final static Logger logger = LoggerFactory.getLogger("project_manager"); final static Logger logger = LoggerFactory.getLogger("project_manager");
@ -70,48 +70,48 @@ public class ProjectManager {
String os = Configurations.get("os.name").toLowerCase(); String os = Configurations.get("os.name").toLowerCase();
if (os.contains("windows")) { if (os.contains("windows")) {
try { try {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA // NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java // see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs, // so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet. // it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks"); DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(localDataPath.getPath()); File data = new File(localDataPath.getPath());
data.mkdirs(); data.mkdirs();
return data; return data;
} catch (Error e) { } catch (Error e) {
/* /*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll * The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up * we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path. * environment variables and try our best to find a user-specific path.
*/ */
logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables"); logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
File parentDir = null; File parentDir = null;
{ {
String appData = System.getenv("APPDATA"); String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) { if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming // e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData); parentDir = new File(appData);
} else { } else {
String userProfile = System.getenv("USERPROFILE"); String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) { if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid] // e.g., C:\Users\[userid]
parentDir = new File(userProfile); parentDir = new File(userProfile);
} }
} }
}
if (parentDir == null) {
parentDir = new File(".");
} }
if (parentDir == null) {
parentDir = new File(".");
}
File data = new File(parentDir, "Gridworks"); File data = new File(parentDir, "Gridworks");
data.mkdirs(); data.mkdirs();
return data; return data;
} }
} else if (os.contains("mac os x")) { } else if (os.contains("mac os x")) {
// on macosx, use "~/Library/Application Support" // on macosx, use "~/Library/Application Support"
String home = System.getProperty("user.home"); String home = System.getProperty("user.home");
@ -262,16 +262,16 @@ public class ProjectManager {
public void addLatestExpression(String s) { public void addLatestExpression(String s) {
synchronized (this) { synchronized (this) {
_expressions.remove(s); _expressions.remove(s);
_expressions.add(0, s); _expressions.add(0, s);
while (_expressions.size() > s_expressionHistoryMax) { while (_expressions.size() > s_expressionHistoryMax) {
_expressions.remove(_expressions.size() - 1); _expressions.remove(_expressions.size() - 1);
} }
} }
} }
public List<String> getExpressions() { public List<String> getExpressions() {
return _expressions; return _expressions;
} }
public void save(boolean allModified) { public void save(boolean allModified) {
@ -285,8 +285,8 @@ public class ProjectManager {
*/ */
protected void saveWorkspace() { protected void saveWorkspace() {
synchronized (this) { synchronized (this) {
File tempFile = new File(_workspaceDir, "workspace.temp.json"); File tempFile = new File(_workspaceDir, "workspace.temp.json");
try { try {
saveToFile(tempFile); saveToFile(tempFile);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();

20
src/main/java/com/metaweb/gridworks/ProjectMetadata.java
View File

@ -170,9 +170,9 @@ public class ProjectMetadata implements Jsonizable {
} }
public void setEncodingConfidence(String confidence) { public void setEncodingConfidence(String confidence) {
if (confidence != null) { if (confidence != null) {
this.setEncodingConfidence(Integer.parseInt(confidence)); this.setEncodingConfidence(Integer.parseInt(confidence));
} }
} }
public int getEncodingConfidence() { public int getEncodingConfidence() {
@ -196,16 +196,16 @@ public class ProjectMetadata implements Jsonizable {
} }
public void addLatestExpression(String s) { public void addLatestExpression(String s) {
_expressions.remove(s); _expressions.remove(s);
_expressions.add(0, s); _expressions.add(0, s);
while (_expressions.size() > s_expressionHistoryMax) { while (_expressions.size() > s_expressionHistoryMax) {
_expressions.remove(_expressions.size() - 1); _expressions.remove(_expressions.size() - 1);
} }
ProjectManager.singleton.addLatestExpression(s); ProjectManager.singleton.addLatestExpression(s);
} }
public List<String> getExpressions() { public List<String> getExpressions() {
return _expressions; return _expressions;
} }
} }

34
src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java
View File

@ -14,7 +14,7 @@ import com.metaweb.gridworks.model.Row;
public class ConjunctiveFilteredRows implements FilteredRows { public class ConjunctiveFilteredRows implements FilteredRows {
final protected List<RowFilter> _rowFilters = new LinkedList<RowFilter>(); final protected List<RowFilter> _rowFilters = new LinkedList<RowFilter>();
final protected boolean _includeContextual; final protected boolean _includeContextual;
final protected boolean _includeDependent; final protected boolean _includeDependent;
public ConjunctiveFilteredRows(boolean includeContextual, boolean includeDependent) { public ConjunctiveFilteredRows(boolean includeContextual, boolean includeDependent) {
_includeContextual = includeContextual; _includeContextual = includeContextual;
@ -27,34 +27,34 @@ public class ConjunctiveFilteredRows implements FilteredRows {
public void accept(Project project, RowVisitor visitor) { public void accept(Project project, RowVisitor visitor) {
int lastVisitedRowRowIndex = -1; int lastVisitedRowRowIndex = -1;
int lastRecordRowAcceptedRowIndex = -1; int lastRecordRowAcceptedRowIndex = -1;
int c = project.rows.size(); int c = project.rows.size();
for (int rowIndex = 0; rowIndex < c; rowIndex++) { for (int rowIndex = 0; rowIndex < c; rowIndex++) {
Row row = project.rows.get(rowIndex); Row row = project.rows.get(rowIndex);
if (matchRow(project, rowIndex, row)) { if (matchRow(project, rowIndex, row)) {
if (row.recordIndex >= 0) { if (row.recordIndex >= 0) {
lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself
} }
visitRow(project, visitor, rowIndex, row, lastVisitedRowRowIndex); visitRow(project, visitor, rowIndex, row, lastVisitedRowRowIndex);
lastVisitedRowRowIndex = rowIndex; lastVisitedRowRowIndex = rowIndex;
} else if ( } else if (
// this row doesn't match by itself but ... // this row doesn't match by itself but ...
// we want to include dependent rows // we want to include dependent rows
_includeDependent && _includeDependent &&
// and this row is a dependent row since it's not a record row // and this row is a dependent row since it's not a record row
row.recordIndex < 0 && row.recordIndex < 0 &&
row.contextRows != null && row.contextRows != null &&
row.contextRows.size() > 0 && row.contextRows.size() > 0 &&
row.contextRows.get(0) == lastRecordRowAcceptedRowIndex row.contextRows.get(0) == lastRecordRowAcceptedRowIndex
) { ) {
// this row depends on the last previously matched record row, // this row depends on the last previously matched record row,
// so we visit it as well as a dependent row // so we visit it as well as a dependent row
visitor.visit(project, rowIndex, row, false, true); visitor.visit(project, rowIndex, row, false, true);
lastVisitedRowRowIndex = rowIndex; lastVisitedRowRowIndex = rowIndex;
@ -63,7 +63,7 @@ public class ConjunctiveFilteredRows implements FilteredRows {
} }
protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, int lastVisitedRow) { protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, int lastVisitedRow) {
if (_includeContextual && // we need to include any context row and if (_includeContextual && // we need to include any context row and
row.contextRows != null && // this row itself isn't a context row and row.contextRows != null && // this row itself isn't a context row and
lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row
// that we haven't visited yet // that we haven't visited yet
@ -80,7 +80,7 @@ public class ConjunctiveFilteredRows implements FilteredRows {
lastVisitedRow = contextRowIndex; lastVisitedRow = contextRowIndex;
} }
} }
} }
visitor.visit(project, rowIndex, row, false, false); visitor.visit(project, rowIndex, row, false, false);
} }

60
src/main/java/com/metaweb/gridworks/browsing/Engine.java
View File

@ -24,7 +24,7 @@ import com.metaweb.gridworks.model.Project;
public class Engine implements Jsonizable { public class Engine implements Jsonizable {
protected Project _project; protected Project _project;
protected List<Facet> _facets = new LinkedList<Facet>(); protected List<Facet> _facets = new LinkedList<Facet>();
protected boolean _includeDependent; protected boolean _includeDependent;
public final static String INCLUDE_DEPENDENT = "includeDependent"; public final static String INCLUDE_DEPENDENT = "includeDependent";
@ -50,38 +50,38 @@ public class Engine implements Jsonizable {
} }
public void initializeFromJSON(JSONObject o) throws Exception { public void initializeFromJSON(JSONObject o) throws Exception {
if (o == null) { if (o == null) {
return; return;
} }
if (o.has("facets") && !o.isNull("facets")) { if (o.has("facets") && !o.isNull("facets")) {
JSONArray a = o.getJSONArray("facets"); JSONArray a = o.getJSONArray("facets");
int length = a.length(); int length = a.length();
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
JSONObject fo = a.getJSONObject(i); JSONObject fo = a.getJSONObject(i);
String type = fo.has("type") ? fo.getString("type") : "list"; String type = fo.has("type") ? fo.getString("type") : "list";
Facet facet = null; Facet facet = null;
if ("list".equals(type)) { if ("list".equals(type)) {
facet = new ListFacet(); facet = new ListFacet();
} else if ("range".equals(type)) { } else if ("range".equals(type)) {
facet = new RangeFacet(); facet = new RangeFacet();
} else if ("scatterplot".equals(type)) { } else if ("scatterplot".equals(type)) {
facet = new ScatterplotFacet(); facet = new ScatterplotFacet();
} else if ("text".equals(type)) { } else if ("text".equals(type)) {
facet = new TextSearchFacet(); facet = new TextSearchFacet();
} }
if (facet != null) { if (facet != null) {
facet.initializeFromJSON(_project, fo); facet.initializeFromJSON(_project, fo);
_facets.add(facet); _facets.add(facet);
} }
} }
} }
if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) { if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) {
_includeDependent = o.getBoolean(INCLUDE_DEPENDENT); _includeDependent = o.getBoolean(INCLUDE_DEPENDENT);
} }
} }
@ -98,11 +98,11 @@ public class Engine implements Jsonizable {
writer.object(); writer.object();
writer.key("facets"); writer.key("facets");
writer.array(); writer.array();
for (Facet facet : _facets) { for (Facet facet : _facets) {
facet.write(writer, options); facet.write(writer, options);
} }
writer.endArray(); writer.endArray();
writer.key(INCLUDE_DEPENDENT); writer.value(_includeDependent); writer.key(INCLUDE_DEPENDENT); writer.value(_includeDependent);
writer.endObject(); writer.endObject();
} }

14
src/main/java/com/metaweb/gridworks/browsing/RowVisitor.java
View File

@ -11,12 +11,12 @@ import com.metaweb.gridworks.model.Row;
*/ */
public interface RowVisitor { public interface RowVisitor {
public boolean visit( public boolean visit(
Project project, Project project,
int rowIndex, // zero-based row index int rowIndex, // zero-based row index
Row row, Row row,
boolean contextual, // true if this row is included because it's the context row boolean contextual, // true if this row is included because it's the context row
// of a matched row, that is, a matched row depends on it // of a matched row, that is, a matched row depends on it
boolean dependent // true if this row is included because it depends on a matched row, boolean dependent // true if this row is included because it depends on a matched row,
// that is, it depends on a matched row // that is, it depends on a matched row
); );
} }

2
src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java
View File

@ -110,7 +110,7 @@ public class ExpressionNumericRowBinner implements RowVisitor {
int bin = (int) Math.floor((d - _index.getMin()) / _index.getStep()); int bin = (int) Math.floor((d - _index.getMin()) / _index.getStep());
if (bin >= 0 && bin < bins.length) { // as a precaution if (bin >= 0 && bin < bins.length) { // as a precaution
bins[bin]++; bins[bin]++;
} }
} else { } else {
rowHasError = true; rowHasError = true;

2
src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java
View File

@ -107,7 +107,7 @@ public class ListFacet implements Facet {
_errorMessage = "No column named " + _columnName; _errorMessage = "No column named " + _columnName;
} }
} else { } else {
_cellIndex = -1; _cellIndex = -1;
} }
try { try {

4
src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java
View File

@ -156,8 +156,8 @@ public class NumericBinIndex {
} }
if (_max <= originalMax) { if (_max <= originalMax) {
_max += _step; _max += _step;
binCount++; binCount++;
} }
_bins = new int[(int) Math.round(binCount)]; _bins = new int[(int) Math.round(binCount)];

12
src/main/java/com/metaweb/gridworks/commands/Command.java
View File

@ -119,11 +119,11 @@ public abstract class Command {
} }
static protected void performProcessAndRespond( static protected void performProcessAndRespond(
HttpServletRequest request, HttpServletRequest request,
HttpServletResponse response, HttpServletResponse response,
Project project, Project project,
Process process Process process
) throws Exception { ) throws Exception {
response.setCharacterEncoding("UTF-8"); response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json"); response.setHeader("Content-Type", "application/json");
@ -141,7 +141,7 @@ public abstract class Command {
w.flush(); w.flush();
w.close(); w.close();
} else { } else {
respond(response, "{ \"code\" : \"pending\" }"); respond(response, "{ \"code\" : \"pending\" }");
} }
} }

6
src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java
View File

@ -31,7 +31,7 @@ public class ExportProjectCommand extends Command {
OutputStream os = response.getOutputStream(); OutputStream os = response.getOutputStream();
try { try {
gzipTarToOutputStream( gzipTarToOutputStream(
ProjectManager.singleton.getProjectDir(project.id), ProjectManager.singleton.getProjectDir(project.id),
os os
); );
@ -44,9 +44,9 @@ public class ExportProjectCommand extends Command {
} }
protected void gzipTarToOutputStream(File dir, OutputStream os) throws IOException { protected void gzipTarToOutputStream(File dir, OutputStream os) throws IOException {
GZIPOutputStream gos = new GZIPOutputStream(os); GZIPOutputStream gos = new GZIPOutputStream(os);
try { try {
tarToOutputStream(dir, gos); tarToOutputStream(dir, gos);
} finally { } finally {
gos.close(); gos.close();
} }

6
src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java
View File

@ -138,10 +138,10 @@ public class ImportProjectCommand extends Command {
destDir.mkdirs(); destDir.mkdirs();
if (gziped) { if (gziped) {
GZIPInputStream gis = new GZIPInputStream(inputStream); GZIPInputStream gis = new GZIPInputStream(inputStream);
untar(destDir, gis); untar(destDir, gis);
} else { } else {
untar(destDir, inputStream); untar(destDir, inputStream);
} }
} }

12
src/main/java/com/metaweb/gridworks/commands/edit/TextTransformCommand.java
View File

@ -27,12 +27,12 @@ public class TextTransformCommand extends EngineDependentCommand {
} }
return new TextTransformOperation( return new TextTransformOperation(
engineConfig, engineConfig,
columnName, columnName,
expression, expression,
TextTransformOperation.stringToOnError(onError), TextTransformOperation.stringToOnError(onError),
repeat, repeat,
repeatCount repeatCount
); );
} }
} }

12
src/main/java/com/metaweb/gridworks/commands/edit/UndoRedoCommand.java
View File

@ -21,19 +21,19 @@ public class UndoRedoCommand extends Command {
long lastDoneID = -1; long lastDoneID = -1;
String lastDoneIDString = request.getParameter("lastDoneID"); String lastDoneIDString = request.getParameter("lastDoneID");
if (lastDoneIDString != null) { if (lastDoneIDString != null) {
lastDoneID = Long.parseLong(lastDoneIDString); lastDoneID = Long.parseLong(lastDoneIDString);
} else { } else {
String undoIDString = request.getParameter("undoID"); String undoIDString = request.getParameter("undoID");
if (undoIDString != null) { if (undoIDString != null) {
long undoID = Long.parseLong(undoIDString); long undoID = Long.parseLong(undoIDString);
lastDoneID = project.history.getPrecedingEntryID(undoID); lastDoneID = project.history.getPrecedingEntryID(undoID);
} }
} }
boolean done = lastDoneID == -1 || boolean done = lastDoneID == -1 ||
project.processManager.queueProcess( project.processManager.queueProcess(
new HistoryProcess(project, lastDoneID)); new HistoryProcess(project, lastDoneID));
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }"); respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
} }

34
src/main/java/com/metaweb/gridworks/commands/info/GetExpressionHistoryCommand.java
View File

@ -36,23 +36,23 @@ public class GetExpressionHistoryCommand extends Command {
JSONWriter writer = new JSONWriter(response.getWriter()); JSONWriter writer = new JSONWriter(response.getWriter());
writer.object(); writer.object();
writer.key("expressions"); writer.key("expressions");
writer.array(); writer.array();
for (String s : localExpressions) { for (String s : localExpressions) {
writer.object(); writer.object();
writer.key("code"); writer.value(s); writer.key("code"); writer.value(s);
writer.key("global"); writer.value(false); writer.key("global"); writer.value(false);
writer.endObject(); writer.endObject();
done.add(s); done.add(s);
} }
for (String s : globalExpressions) { for (String s : globalExpressions) {
if (!done.contains(s)) { if (!done.contains(s)) {
writer.object(); writer.object();
writer.key("code"); writer.value(s); writer.key("code"); writer.value(s);
writer.key("global"); writer.value(true); writer.key("global"); writer.value(true);
writer.endObject(); writer.endObject();
} }
} }
writer.endArray(); writer.endArray();
writer.endObject(); writer.endObject();
} catch (Exception e) { } catch (Exception e) {
respondException(response, e); respondException(response, e);

8
src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java
View File

@ -154,10 +154,10 @@ public class ReconJudgeOneCellCommand extends Command {
newCell.recon.match = this.match; newCell.recon.match = this.match;
for (int m = 0; m < newCell.recon.candidates.size(); m++) { for (int m = 0; m < newCell.recon.candidates.size(); m++) {
if (newCell.recon.candidates.get(m).topicGUID.equals(this.match.topicGUID)) { if (newCell.recon.candidates.get(m).topicGUID.equals(this.match.topicGUID)) {
newCell.recon.matchRank = m; newCell.recon.matchRank = m;
break; break;
} }
} }
description = "Match " + this.match.topicName + description = "Match " + this.match.topicName +

22
src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java
View File

@ -82,19 +82,19 @@ public class PreviewExtendDataCommand extends Command {
writer.key("columns"); writer.key("columns");
writer.array(); writer.array();
for (ColumnInfo info : job.columns) { for (ColumnInfo info : job.columns) {
writer.object(); writer.object();
writer.key("names"); writer.key("names");
writer.array(); writer.array();
for (String name : info.names) { for (String name : info.names) {
writer.value(name); writer.value(name);
} }
writer.endArray(); writer.endArray();
writer.key("path"); writer.key("path");
writer.array(); writer.array();
for (String id : info.path) { for (String id : info.path) {
writer.value(id); writer.value(id);
} }
writer.endArray(); writer.endArray();
writer.endObject(); writer.endObject();
} }
writer.endArray(); writer.endArray();

2
src/main/java/com/metaweb/gridworks/expr/JythonEvaluable.java
View File

@ -15,7 +15,7 @@ import org.python.core.PyString;
import org.python.util.PythonInterpreter; import org.python.util.PythonInterpreter;
public class JythonEvaluable implements Evaluable { public class JythonEvaluable implements Evaluable {
private static final String s_functionName = "___temp___"; private static final String s_functionName = "___temp___";
private static PythonInterpreter _engine; private static PythonInterpreter _engine;
static { static {

6
src/main/java/com/metaweb/gridworks/expr/functions/Get.java
View File

@ -51,11 +51,11 @@ public class Get implements Function {
if (end > start) { if (end > start) {
if (v.getClass().isArray()) { if (v.getClass().isArray()) {
Object[] a2 = new Object[end - start]; Object[] a2 = new Object[end - start];
System.arraycopy((Object[]) v, start, a2, 0, end - start); System.arraycopy((Object[]) v, start, a2, 0, end - start);
return a2; return a2;
} else { } else {
return ExpressionUtils.toObjectList(v).subList(start, end); return ExpressionUtils.toObjectList(v).subList(start, end);
} }

2
src/main/java/com/metaweb/gridworks/expr/functions/Length.java
View File

@ -21,7 +21,7 @@ public class Length implements Function {
Object[] a = (Object[]) v; Object[] a = (Object[]) v;
return a.length; return a.length;
} else if (v instanceof Collection<?>) { } else if (v instanceof Collection<?>) {
return ((Collection<?>) v).size(); return ((Collection<?>) v).size();
} else { } else {
String s = (v instanceof String ? (String) v : v.toString()); String s = (v instanceof String ? (String) v : v.toString());
return s.length(); return s.length();

32
src/main/java/com/metaweb/gridworks/expr/functions/arrays/Join.java
View File

@ -25,23 +25,23 @@ public class Join implements Function {
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();
if (v.getClass().isArray()) { if (v.getClass().isArray()) {
for (Object o : (Object[]) v) { for (Object o : (Object[]) v) {
if (o != null) { if (o != null) {
if (sb.length() > 0) { if (sb.length() > 0) {
sb.append(separator); sb.append(separator);
} }
sb.append(o.toString()); sb.append(o.toString());
} }
} }
} else { } else {
for (Object o : ExpressionUtils.toObjectList(v)) { for (Object o : ExpressionUtils.toObjectList(v)) {
if (o != null) { if (o != null) {
if (sb.length() > 0) { if (sb.length() > 0) {
sb.append(separator); sb.append(separator);
} }
sb.append(o.toString()); sb.append(o.toString());
} }
} }
} }
return sb.toString(); return sb.toString();

4
src/main/java/com/metaweb/gridworks/expr/functions/arrays/Sort.java
View File

@ -14,8 +14,8 @@ import com.metaweb.gridworks.gel.Function;
public class Sort implements Function { public class Sort implements Function {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public Object call(Properties bindings, Object[] args) { public Object call(Properties bindings, Object[] args) {
if (args.length == 1) { if (args.length == 1) {
Object v = args[0]; Object v = args[0];

36
src/main/java/com/metaweb/gridworks/gel/Scanner.java
View File

@ -146,18 +146,18 @@ public class Scanner {
} else if (c == '\\') { } else if (c == '\\') {
_index++; // skip escaping marker _index++; // skip escaping marker
if (_index < _limit) { if (_index < _limit) {
char c2 = _text.charAt(_index); char c2 = _text.charAt(_index);
if (c2 == 't') { if (c2 == 't') {
sb.append('\t'); sb.append('\t');
} else if (c2 == 'n') { } else if (c2 == 'n') {
sb.append('\n'); sb.append('\n');
} else if (c2 == 'r') { } else if (c2 == 'r') {
sb.append('\r'); sb.append('\r');
} else if (c2 == '\\') { } else if (c2 == '\\') {
sb.append('\\'); sb.append('\\');
} else { } else {
sb.append(c2); sb.append(c2);
} }
} }
} else { } else {
sb.append(c); sb.append(c);
@ -170,12 +170,12 @@ public class Scanner {
} else if (Character.isLetter(c) || c == '_') { // identifier } else if (Character.isLetter(c) || c == '_') { // identifier
while (_index < _limit) { while (_index < _limit) {
char c1 = _text.charAt(_index); char c1 = _text.charAt(_index);
if (c1 == '_' || Character.isLetterOrDigit(c1)) { if (c1 == '_' || Character.isLetterOrDigit(c1)) {
_index++; _index++;
} else { } else {
break; break;
} }
} }
return new Token( return new Token(

16
src/main/java/com/metaweb/gridworks/history/History.java
View File

@ -140,7 +140,7 @@ public class History implements Jsonizable {
public long getPrecedingEntryID(long entryID) { public long getPrecedingEntryID(long entryID) {
if (entryID == 0) { if (entryID == 0) {
return -1; return -1;
} else { } else {
for (int i = 0; i < _pastEntries.size(); i++) { for (int i = 0; i < _pastEntries.size(); i++) {
if (_pastEntries.get(i).id == entryID) { if (_pastEntries.get(i).id == entryID) {
@ -150,13 +150,13 @@ public class History implements Jsonizable {
for (int i = 0; i < _futureEntries.size(); i++) { for (int i = 0; i < _futureEntries.size(); i++) {
if (_futureEntries.get(i).id == entryID) { if (_futureEntries.get(i).id == entryID) {
if (i > 0) { if (i > 0) {
return _futureEntries.get(i - 1).id; return _futureEntries.get(i - 1).id;
} else if (_pastEntries.size() > 0) { } else if (_pastEntries.size() > 0) {
return _pastEntries.get(_pastEntries.size() - 1).id; return _pastEntries.get(_pastEntries.size() - 1).id;
} else { } else {
return 0; return 0;
} }
} }
} }
} }

32
src/main/java/com/metaweb/gridworks/importers/ExcelImporter.java
View File

@ -114,14 +114,14 @@ public class ExcelImporter implements Importer {
*/ */
Map<String, Integer> nameToIndex = new HashMap<String, Integer>(); Map<String, Integer> nameToIndex = new HashMap<String, Integer>();
for (int c = 0; c < nonBlankIndices.size(); c++) { for (int c = 0; c < nonBlankIndices.size(); c++) {
String cell = nonBlankHeaderStrings.get(c); String cell = nonBlankHeaderStrings.get(c);
if (nameToIndex.containsKey(cell)) { if (nameToIndex.containsKey(cell)) {
int index = nameToIndex.get(cell); int index = nameToIndex.get(cell);
nameToIndex.put(cell, index + 1); nameToIndex.put(cell, index + 1);
cell = cell.contains(" ") ? (cell + " " + index) : (cell + index); cell = cell.contains(" ") ? (cell + " " + index) : (cell + index);
} else { } else {
nameToIndex.put(cell, 2); nameToIndex.put(cell, 2);
} }
Column column = new Column(c, cell); Column column = new Column(c, cell);
@ -209,19 +209,19 @@ public class ExcelImporter implements Importer {
} }
if (reconMap.containsKey(id)) { if (reconMap.containsKey(id)) {
recon = reconMap.get(id); recon = reconMap.get(id);
recon.judgmentBatchSize++; recon.judgmentBatchSize++;
} else { } else {
recon = new Recon(0); recon = new Recon(0);
recon.service = "import"; recon.service = "import";
recon.match = new ReconCandidate(id, "", value.toString(), new String[0], 100); recon.match = new ReconCandidate(id, "", value.toString(), new String[0], 100);
recon.matchRank = 0; recon.matchRank = 0;
recon.judgment = Judgment.Matched; recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto"; recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1; recon.judgmentBatchSize = 1;
recon.addCandidate(recon.match); recon.addCandidate(recon.match);
reconMap.put(id, recon); reconMap.put(id, recon);
} }
} }

4
src/main/java/com/metaweb/gridworks/model/ColumnModel.java
View File

@ -27,7 +27,7 @@ public class ColumnModel implements Jsonizable {
transient protected Map<String, Column> _nameToColumn; transient protected Map<String, Column> _nameToColumn;
transient protected Map<Integer, Column> _cellIndexToColumn; transient protected Map<Integer, Column> _cellIndexToColumn;
transient protected List<ColumnGroup> _rootColumnGroups; transient protected List<ColumnGroup> _rootColumnGroups;
transient protected List<String> _columnNames; transient protected List<String> _columnNames;
transient boolean _hasDependentRows; transient boolean _hasDependentRows;
public ColumnModel() { public ColumnModel() {
@ -96,7 +96,7 @@ public class ColumnModel implements Jsonizable {
} }
public List<String> getColumnNames() { public List<String> getColumnNames() {
return _columnNames; return _columnNames;
} }
public void write(JSONWriter writer, Properties options) public void write(JSONWriter writer, Properties options)

4
src/main/java/com/metaweb/gridworks/model/Project.java
View File

@ -59,7 +59,7 @@ public class Project {
} }
public ProjectMetadata getMetadata() { public ProjectMetadata getMetadata() {
return ProjectManager.singleton.getProjectMetadata(id); return ProjectManager.singleton.getProjectMetadata(id);
} }
public void save() { public void save() {
@ -232,7 +232,7 @@ public class Project {
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
line = reader.readLine(); line = reader.readLine();
if (line != null) { if (line != null) {
Row row = Row.load(line, pool); Row row = Row.load(line, pool);
project.rows.add(row); project.rows.add(row);
maxCellCount = Math.max(maxCellCount, row.cells.size()); maxCellCount = Math.max(maxCellCount, row.cells.size());
} }

22
src/main/java/com/metaweb/gridworks/model/Recon.java
View File

@ -60,16 +60,16 @@ public class Recon implements HasFields, Jsonizable {
final public long id; final public long id;
public Object[] features = new Object[Feature_max]; public Object[] features = new Object[Feature_max];
public String service = "unknown"; public String service = "unknown";
public List<ReconCandidate> candidates; public List<ReconCandidate> candidates;
public Judgment judgment = Judgment.None; public Judgment judgment = Judgment.None;
public String judgmentAction = "unknown"; public String judgmentAction = "unknown";
public long judgmentHistoryEntry; public long judgmentHistoryEntry;
public int judgmentBatchSize = 0; public int judgmentBatchSize = 0;
public ReconCandidate match = null; public ReconCandidate match = null;
public int matchRank = -1; public int matchRank = -1;
public Recon(long judgmentHistoryEntry) { public Recon(long judgmentHistoryEntry) {
id = System.currentTimeMillis() * 1000000 + Math.round(Math.random() * 1000000); id = System.currentTimeMillis() * 1000000 + Math.round(Math.random() * 1000000);
@ -189,7 +189,7 @@ public class Recon implements HasFields, Jsonizable {
public void write(JSONWriter writer, Properties options) public void write(JSONWriter writer, Properties options)
throws JSONException { throws JSONException {
boolean saveMode = "save".equals(options.getProperty("mode")); boolean saveMode = "save".equals(options.getProperty("mode"));
writer.object(); writer.object();
writer.key("id"); writer.value(id); writer.key("id"); writer.value(id);
@ -225,7 +225,7 @@ public class Recon implements HasFields, Jsonizable {
writer.key("judgmentBatchSize"); writer.value(judgmentBatchSize); writer.key("judgmentBatchSize"); writer.value(judgmentBatchSize);
if (match != null) { if (match != null) {
writer.key("matchRank"); writer.value(matchRank); writer.key("matchRank"); writer.value(matchRank);
} }
} }
@ -314,13 +314,13 @@ public class Recon implements HasFields, Jsonizable {
} }
} }
} else if ("service".equals(fieldName)) { } else if ("service".equals(fieldName)) {
recon.service = jp.getText(); recon.service = jp.getText();
} else if ("judgmentAction".equals(fieldName)) { } else if ("judgmentAction".equals(fieldName)) {
recon.judgmentAction = jp.getText(); recon.judgmentAction = jp.getText();
} else if ("judgmentBatchSize".equals(fieldName)) { } else if ("judgmentBatchSize".equals(fieldName)) {
recon.judgmentBatchSize = jp.getIntValue(); recon.judgmentBatchSize = jp.getIntValue();
} else if ("matchRank".equals(fieldName)) { } else if ("matchRank".equals(fieldName)) {
recon.matchRank = jp.getIntValue(); recon.matchRank = jp.getIntValue();
} }
} }
} }

10
src/main/java/com/metaweb/gridworks/model/ReconCandidate.java
View File

@ -57,11 +57,11 @@ public class ReconCandidate implements HasFields, Jsonizable {
writer.key("score"); writer.value(score); writer.key("score"); writer.value(score);
/* if (!options.containsKey("reconCandidateOmitTypes")) */ { /* if (!options.containsKey("reconCandidateOmitTypes")) */ {
writer.key("types"); writer.array(); writer.key("types"); writer.array();
for (String typeID : typeIDs) { for (String typeID : typeIDs) {
writer.value(typeID); writer.value(typeID);
} }
writer.endArray(); writer.endArray();
} }
writer.endObject(); writer.endObject();

112
src/main/java/com/metaweb/gridworks/model/changes/ColumnSplitChange.java
View File

@ -19,10 +19,10 @@ import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.util.Pool; import com.metaweb.gridworks.util.Pool;
public class ColumnSplitChange implements Change { public class ColumnSplitChange implements Change {
final protected String _columnName; final protected String _columnName;
final protected List<String> _columnNames; final protected List<String> _columnNames;
final protected List<Integer> _rowIndices; final protected List<Integer> _rowIndices;
final protected List<List<Serializable>> _tuples; final protected List<List<Serializable>> _tuples;
final protected boolean _removeOriginalColumn; final protected boolean _removeOriginalColumn;
@ -30,20 +30,20 @@ public class ColumnSplitChange implements Change {
protected Column _column; protected Column _column;
protected int _columnIndex; protected int _columnIndex;
protected int _firstNewCellIndex = -1; protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows; protected List<Row> _oldRows;
protected List<Row> _newRows; protected List<Row> _newRows;
public ColumnSplitChange( public ColumnSplitChange(
String columnName, String columnName,
List<String> columnNames, List<String> columnNames,
List<Integer> rowIndices, List<Integer> rowIndices,
List<List<Serializable>> tuples, List<List<Serializable>> tuples,
boolean removeOriginalColumn boolean removeOriginalColumn
) { ) {
_columnName = columnName; _columnName = columnName;
_columnNames = columnNames; _columnNames = columnNames;
_rowIndices = rowIndices; _rowIndices = rowIndices;
_tuples = tuples; _tuples = tuples;
@ -60,10 +60,10 @@ public class ColumnSplitChange implements Change {
Column column, Column column,
int columnIndex, int columnIndex,
int firstNewCellIndex, int firstNewCellIndex,
List<Row> oldRows, List<Row> oldRows,
List<Row> newRows List<Row> newRows
) { ) {
_columnName = columnName; _columnName = columnName;
_columnNames = columnNames; _columnNames = columnNames;
@ -83,13 +83,13 @@ public class ColumnSplitChange implements Change {
public void apply(Project project) { public void apply(Project project) {
synchronized (project) { synchronized (project) {
if (_firstNewCellIndex < 0) { if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex(); _firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) { for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex(); project.columnModel.allocateNewCellIndex();
} }
_column = project.columnModel.getColumnByName(_columnName); _column = project.columnModel.getColumnByName(_columnName);
_columnIndex = project.columnModel.getColumnIndexByName(_columnName); _columnIndex = project.columnModel.getColumnIndexByName(_columnName);
_oldRows = new ArrayList<Row>(_rowIndices.size()); _oldRows = new ArrayList<Row>(_rowIndices.size());
_newRows = new ArrayList<Row>(_rowIndices.size()); _newRows = new ArrayList<Row>(_rowIndices.size());
@ -100,22 +100,22 @@ public class ColumnSplitChange implements Change {
int r = _rowIndices.get(i); int r = _rowIndices.get(i);
List<Serializable> tuple = _tuples.get(i); List<Serializable> tuple = _tuples.get(i);
Row oldRow = project.rows.get(r); Row oldRow = project.rows.get(r);
Row newRow = oldRow.dup(); Row newRow = oldRow.dup();
_oldRows.add(oldRow); _oldRows.add(oldRow);
_newRows.add(newRow); _newRows.add(newRow);
for (int c = 0; c < tuple.size(); c++) { for (int c = 0; c < tuple.size(); c++) {
Serializable value = tuple.get(c); Serializable value = tuple.get(c);
if (value != null) { if (value != null) {
newRow.setCell(_firstNewCellIndex + c, new Cell(value, null)); newRow.setCell(_firstNewCellIndex + c, new Cell(value, null));
} }
} }
if (_removeOriginalColumn) { if (_removeOriginalColumn) {
newRow.setCell(cellIndex, null); newRow.setCell(cellIndex, null);
} }
} }
} }
@ -127,12 +127,12 @@ public class ColumnSplitChange implements Change {
} }
for (int i = 0; i < _columnNames.size(); i++) { for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i); String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i; int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name); Column column = new Column(cellIndex, name);
project.columnModel.columns.add(_columnIndex + 1 + i, column); project.columnModel.columns.add(_columnIndex + 1 + i, column);
} }
if (_removeOriginalColumn) { if (_removeOriginalColumn) {
@ -158,7 +158,7 @@ public class ColumnSplitChange implements Change {
} }
for (int i = 0; i < _columnNames.size(); i++) { for (int i = 0; i < _columnNames.size(); i++) {
project.columnModel.columns.remove(_columnIndex + 1); project.columnModel.columns.remove(_columnIndex + 1);
} }
project.columnModel.update(); project.columnModel.update();
@ -167,29 +167,29 @@ public class ColumnSplitChange implements Change {
} }
public void save(Writer writer, Properties options) throws IOException { public void save(Writer writer, Properties options) throws IOException {
writer.write("columnName="); writer.write(_columnName); writer.write('\n'); writer.write("columnName="); writer.write(_columnName); writer.write('\n');
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n'); writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String name : _columnNames) { for (String name : _columnNames) {
writer.write(name); writer.write('\n'); writer.write(name); writer.write('\n');
} }
writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n'); writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
for (Integer rowIndex : _rowIndices) { for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n'); writer.write(rowIndex.toString()); writer.write('\n');
} }
writer.write("tupleCount="); writer.write(Integer.toString(_tuples.size())); writer.write('\n'); writer.write("tupleCount="); writer.write(Integer.toString(_tuples.size())); writer.write('\n');
for (List<Serializable> tuple : _tuples) { for (List<Serializable> tuple : _tuples) {
writer.write(Integer.toString(tuple.size())); writer.write('\n'); writer.write(Integer.toString(tuple.size())); writer.write('\n');
for (Serializable value : tuple) { for (Serializable value : tuple) {
if (value == null) { if (value == null) {
writer.write("null"); writer.write("null");
} else if (value instanceof String) { } else if (value instanceof String) {
writer.write(JSONObject.quote((String) value)); writer.write(JSONObject.quote((String) value));
} else { } else {
writer.write(value.toString()); writer.write(value.toString());
} }
writer.write('\n'); writer.write('\n');
} }
} }
writer.write("removeOriginalColumn="); writer.write(Boolean.toString(_removeOriginalColumn)); writer.write('\n'); writer.write("removeOriginalColumn="); writer.write(Boolean.toString(_removeOriginalColumn)); writer.write('\n');
@ -197,7 +197,7 @@ public class ColumnSplitChange implements Change {
writer.write("column="); _column.save(writer); writer.write('\n'); writer.write("column="); _column.save(writer); writer.write('\n');
writer.write("columnIndex="); writer.write(Integer.toString(_columnIndex)); writer.write('\n'); writer.write("columnIndex="); writer.write(Integer.toString(_columnIndex)); writer.write('\n');
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n'); writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
for (Row row : _newRows) { for (Row row : _newRows) {
@ -233,7 +233,7 @@ public class ColumnSplitChange implements Change {
String value = line.substring(equal + 1); String value = line.substring(equal + 1);
if ("columnName".equals(field)) { if ("columnName".equals(field)) {
columnName = value; columnName = value;
} else if ("columnNameCount".equals(field)) { } else if ("columnNameCount".equals(field)) {
int count = Integer.parseInt(value); int count = Integer.parseInt(value);
@ -267,7 +267,7 @@ public class ColumnSplitChange implements Change {
List<Serializable> tuple = new ArrayList<Serializable>(valueCount); List<Serializable> tuple = new ArrayList<Serializable>(valueCount);
for (int r = 0; r < valueCount; r++) { for (int r = 0; r < valueCount; r++) {
line = reader.readLine(); line = reader.readLine();
JSONTokener t = new JSONTokener(line); JSONTokener t = new JSONTokener(line);
Object o = t.nextValue(); Object o = t.nextValue();

252
src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java
View File

@ -30,34 +30,34 @@ import com.metaweb.gridworks.util.Pool;
import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension; import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension;
public class DataExtensionChange implements Change { public class DataExtensionChange implements Change {
final protected String _baseColumnName; final protected String _baseColumnName;
final protected int _columnInsertIndex; final protected int _columnInsertIndex;
final protected List<String> _columnNames; final protected List<String> _columnNames;
final protected List<FreebaseType> _columnTypes; final protected List<FreebaseType> _columnTypes;
final protected List<Integer> _rowIndices; final protected List<Integer> _rowIndices;
final protected List<DataExtension> _dataExtensions; final protected List<DataExtension> _dataExtensions;
protected long _historyEntryID; protected long _historyEntryID;
protected int _firstNewCellIndex = -1; protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows; protected List<Row> _oldRows;
protected List<Row> _newRows; protected List<Row> _newRows;
public DataExtensionChange( public DataExtensionChange(
String baseColumnName, String baseColumnName,
int columnInsertIndex, int columnInsertIndex,
List<String> columnNames, List<String> columnNames,
List<FreebaseType> columnTypes, List<FreebaseType> columnTypes,
List<Integer> rowIndices, List<Integer> rowIndices,
List<DataExtension> dataExtensions, List<DataExtension> dataExtensions,
long historyEntryID long historyEntryID
) { ) {
_baseColumnName = baseColumnName; _baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex; _columnInsertIndex = columnInsertIndex;
_columnNames = columnNames; _columnNames = columnNames;
_columnTypes = columnTypes; _columnTypes = columnTypes;
_rowIndices = rowIndices; _rowIndices = rowIndices;
_dataExtensions = dataExtensions; _dataExtensions = dataExtensions;
@ -66,22 +66,22 @@ public class DataExtensionChange implements Change {
} }
protected DataExtensionChange( protected DataExtensionChange(
String baseColumnName, String baseColumnName,
int columnInsertIndex, int columnInsertIndex,
List<String> columnNames, List<String> columnNames,
List<FreebaseType> columnTypes, List<FreebaseType> columnTypes,
List<Integer> rowIndices, List<Integer> rowIndices,
List<DataExtension> dataExtensions, List<DataExtension> dataExtensions,
int firstNewCellIndex, int firstNewCellIndex,
List<Row> oldRows, List<Row> oldRows,
List<Row> newRows List<Row> newRows
) { ) {
_baseColumnName = baseColumnName; _baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex; _columnInsertIndex = columnInsertIndex;
_columnNames = columnNames; _columnNames = columnNames;
_columnTypes = columnTypes; _columnTypes = columnTypes;
_rowIndices = rowIndices; _rowIndices = rowIndices;
@ -95,10 +95,10 @@ public class DataExtensionChange implements Change {
public void apply(Project project) { public void apply(Project project) {
synchronized (project) { synchronized (project) {
if (_firstNewCellIndex < 0) { if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex(); _firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) { for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex(); project.columnModel.allocateNewCellIndex();
} }
_oldRows = new ArrayList<Row>(project.rows); _oldRows = new ArrayList<Row>(project.rows);
@ -116,18 +116,18 @@ public class DataExtensionChange implements Change {
Map<String, Recon> reconMap = new HashMap<String, Recon>(); Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (int r = 0; r < _oldRows.size(); r++) { for (int r = 0; r < _oldRows.size(); r++) {
Row oldRow = _oldRows.get(r); Row oldRow = _oldRows.get(r);
if (r < rowIndex) { if (r < rowIndex) {
_newRows.add(oldRow.dup()); _newRows.add(oldRow.dup());
continue; continue;
} }
if (dataExtension == null || dataExtension.data.length == 0) { if (dataExtension == null || dataExtension.data.length == 0) {
_newRows.add(oldRow); _newRows.add(oldRow);
} else { } else {
Row firstNewRow = oldRow.dup(); Row firstNewRow = oldRow.dup();
extendRow(firstNewRow, dataExtension, 0, reconMap); extendRow(firstNewRow, dataExtension, 0, reconMap);
_newRows.add(firstNewRow); _newRows.add(firstNewRow);
int r2 = r + 1; int r2 = r + 1;
for (int subR = 1; subR < dataExtension.data.length; subR++) { for (int subR = 1; subR < dataExtension.data.length; subR++) {
@ -153,7 +153,7 @@ public class DataExtensionChange implements Change {
} }
r = r2 - 1; // r will be incremented by the for loop anyway r = r2 - 1; // r will be incremented by the for loop anyway
} }
rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size(); rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null; dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
@ -165,14 +165,14 @@ public class DataExtensionChange implements Change {
project.rows.addAll(_newRows); project.rows.addAll(_newRows);
for (int i = 0; i < _columnNames.size(); i++) { for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i); String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i; int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name); Column column = new Column(cellIndex, name);
column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i))); column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
column.setReconStats(ReconStats.create(project, cellIndex)); column.setReconStats(ReconStats.create(project, cellIndex));
project.columnModel.columns.add(_columnInsertIndex + i, column); project.columnModel.columns.add(_columnInsertIndex + i, column);
} }
project.columnModel.update(); project.columnModel.update();
@ -181,40 +181,40 @@ public class DataExtensionChange implements Change {
} }
protected void extendRow( protected void extendRow(
Row row, Row row,
DataExtension dataExtension, DataExtension dataExtension,
int extensionRowIndex, int extensionRowIndex,
Map<String, Recon> reconMap Map<String, Recon> reconMap
) { ) {
Object[] values = dataExtension.data[extensionRowIndex]; Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) { for (int c = 0; c < values.length; c++) {
Object value = values[c]; Object value = values[c];
Cell cell = null; Cell cell = null;
if (value instanceof ReconCandidate) { if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value; ReconCandidate rc = (ReconCandidate) value;
Recon recon; Recon recon;
if (reconMap.containsKey(rc.topicGUID)) { if (reconMap.containsKey(rc.topicGUID)) {
recon = reconMap.get(rc.topicGUID); recon = reconMap.get(rc.topicGUID);
} else { } else {
recon = new Recon(_historyEntryID); recon = new Recon(_historyEntryID);
recon.addCandidate(rc); recon.addCandidate(rc);
recon.service = "mql"; recon.service = "mql";
recon.match = rc; recon.match = rc;
recon.matchRank = 0; recon.matchRank = 0;
recon.judgment = Judgment.Matched; recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto"; recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1; recon.judgmentBatchSize = 1;
reconMap.put(rc.topicGUID, recon); reconMap.put(rc.topicGUID, recon);
} }
cell = new Cell(rc.topicName, recon); cell = new Cell(rc.topicName, recon);
} else { } else {
cell = new Cell((Serializable) value, null); cell = new Cell((Serializable) value, null);
} }
row.setCell(_firstNewCellIndex + c, cell); row.setCell(_firstNewCellIndex + c, cell);
} }
} }
public void revert(Project project) { public void revert(Project project) {
@ -223,7 +223,7 @@ public class DataExtensionChange implements Change {
project.rows.addAll(_oldRows); project.rows.addAll(_oldRows);
for (int i = 0; i < _columnNames.size(); i++) { for (int i = 0; i < _columnNames.size(); i++) {
project.columnModel.columns.remove(_columnInsertIndex); project.columnModel.columns.remove(_columnInsertIndex);
} }
project.columnModel.update(); project.columnModel.update();
@ -232,11 +232,11 @@ public class DataExtensionChange implements Change {
} }
public void save(Writer writer, Properties options) throws IOException { public void save(Writer writer, Properties options) throws IOException {
writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n'); writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n'); writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n'); writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String name : _columnNames) { for (String name : _columnNames) {
writer.write(name); writer.write('\n'); writer.write(name); writer.write('\n');
} }
writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n'); writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
for (FreebaseType type : _columnTypes) { for (FreebaseType type : _columnTypes) {
@ -251,34 +251,34 @@ public class DataExtensionChange implements Change {
} }
writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n'); writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
for (Integer rowIndex : _rowIndices) { for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n'); writer.write(rowIndex.toString()); writer.write('\n');
} }
writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n'); writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
for (DataExtension dataExtension : _dataExtensions) { for (DataExtension dataExtension : _dataExtensions) {
writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n'); writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
for (Object[] values : dataExtension.data) { for (Object[] values : dataExtension.data) {
for (Object value : values) { for (Object value : values) {
if (value == null) { if (value == null) {
writer.write("null"); writer.write("null");
} else if (value instanceof ReconCandidate) { } else if (value instanceof ReconCandidate) {
try { try {
JSONWriter jsonWriter = new JSONWriter(writer); JSONWriter jsonWriter = new JSONWriter(writer);
((ReconCandidate) value).write(jsonWriter, options); ((ReconCandidate) value).write(jsonWriter, options);
} catch (JSONException e) { } catch (JSONException e) {
// ??? // ???
} }
} else if (value instanceof String) { } else if (value instanceof String) {
writer.write(JSONObject.quote((String) value)); writer.write(JSONObject.quote((String) value));
} else { } else {
writer.write(value.toString()); writer.write(value.toString());
} }
writer.write('\n'); writer.write('\n');
} }
} }
} }
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n'); writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n'); writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
for (Row row : _newRows) { for (Row row : _newRows) {
@ -294,14 +294,14 @@ public class DataExtensionChange implements Change {
} }
static public Change load(LineNumberReader reader, Pool pool) throws Exception { static public Change load(LineNumberReader reader, Pool pool) throws Exception {
String baseColumnName = null; String baseColumnName = null;
int columnInsertIndex = -1; int columnInsertIndex = -1;
List<String> columnNames = null; List<String> columnNames = null;
List<FreebaseType> columnTypes = null; List<FreebaseType> columnTypes = null;
List<Integer> rowIndices = null; List<Integer> rowIndices = null;
List<DataExtension> dataExtensions = null; List<DataExtension> dataExtensions = null;
List<Row> oldRows = null; List<Row> oldRows = null;
List<Row> newRows = null; List<Row> newRows = null;
@ -315,11 +315,11 @@ public class DataExtensionChange implements Change {
String value = line.substring(equal + 1); String value = line.substring(equal + 1);
if ("baseColumnName".equals(field)) { if ("baseColumnName".equals(field)) {
baseColumnName = value; baseColumnName = value;
} else if ("columnInsertIndex".equals(field)) { } else if ("columnInsertIndex".equals(field)) {
columnInsertIndex = Integer.parseInt(value); columnInsertIndex = Integer.parseInt(value);
} else if ("firstNewCellIndex".equals(field)) { } else if ("firstNewCellIndex".equals(field)) {
firstNewCellIndex = Integer.parseInt(value); firstNewCellIndex = Integer.parseInt(value);
} else if ("rowIndexCount".equals(field)) { } else if ("rowIndexCount".equals(field)) {
int count = Integer.parseInt(value); int count = Integer.parseInt(value);
@ -361,14 +361,14 @@ public class DataExtensionChange implements Change {
Object[][] data = new Object[rowCount][]; Object[][] data = new Object[rowCount][];
for (int r = 0; r < rowCount; r++) { for (int r = 0; r < rowCount; r++) {
Object[] row = new Object[columnNames.size()]; Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) { for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine(); line = reader.readLine();
row[c] = ReconCandidate.loadStreaming(line); row[c] = ReconCandidate.loadStreaming(line);
} }
data[r] = row; data[r] = row;
} }
dataExtensions.add(new DataExtension(data)); dataExtensions.add(new DataExtension(data));
@ -398,15 +398,15 @@ public class DataExtensionChange implements Change {
} }
DataExtensionChange change = new DataExtensionChange( DataExtensionChange change = new DataExtensionChange(
baseColumnName, baseColumnName,
columnInsertIndex, columnInsertIndex,
columnNames, columnNames,
columnTypes, columnTypes,
rowIndices, rowIndices,
dataExtensions, dataExtensions,
firstNewCellIndex, firstNewCellIndex,
oldRows, oldRows,
newRows newRows
); );

16
src/main/java/com/metaweb/gridworks/model/recon/HeuristicReconConfig.java
View File

@ -300,7 +300,7 @@ public class HeuristicReconConfig extends ReconConfig {
} }
if (count > 0) { if (count > 0) {
ReconCandidate candidate = recon.candidates.get(0); ReconCandidate candidate = recon.candidates.get(0);
recon.setFeature(Recon.Feature_nameMatch, text.equalsIgnoreCase(candidate.topicName)); recon.setFeature(Recon.Feature_nameMatch, text.equalsIgnoreCase(candidate.topicName));
recon.setFeature(Recon.Feature_nameLevenshtein, StringUtils.getLevenshteinDistance(text, candidate.topicName)); recon.setFeature(Recon.Feature_nameLevenshtein, StringUtils.getLevenshteinDistance(text, candidate.topicName));
@ -311,14 +311,14 @@ public class HeuristicReconConfig extends ReconConfig {
if (this.typeID.equals(typeID)) { if (this.typeID.equals(typeID)) {
recon.setFeature(Recon.Feature_typeMatch, true); recon.setFeature(Recon.Feature_typeMatch, true);
if (autoMatch && candidate.score >= 100) { if (autoMatch && candidate.score >= 100) {
if (count == 1 || if (count == 1 ||
candidate.score / recon.candidates.get(1).score >= 1.5) { candidate.score / recon.candidates.get(1).score >= 1.5) {
recon.match = candidate; recon.match = candidate;
recon.matchRank = 0; recon.matchRank = 0;
recon.judgment = Judgment.Matched; recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto"; recon.judgmentAction = "auto";
} }
} }
break; break;
} }

4
src/main/java/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java
View File

@ -38,9 +38,9 @@ abstract public class EngineDependentMassCellOperation extends EngineDependentOp
FilteredRows filteredRows = engine.getAllFilteredRows(false); FilteredRows filteredRows = engine.getAllFilteredRows(false);
try { try {
filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID)); filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID));
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
String description = createDescription(column, cellChanges); String description = createDescription(column, cellChanges);

2
src/main/java/com/metaweb/gridworks/operations/EngineDependentOperation.java
View File

@ -16,7 +16,7 @@ abstract public class EngineDependentOperation extends AbstractOperation {
protected EngineDependentOperation(JSONObject engineConfig) { protected EngineDependentOperation(JSONObject engineConfig) {
_engineConfig = engineConfig; _engineConfig = engineConfig;
_engineConfigString = engineConfig == null || engineConfig.length() == 0 _engineConfigString = engineConfig == null || engineConfig.length() == 0
? null : engineConfig.toString(); ? null : engineConfig.toString();
} }
protected Engine createEngine(Project project) throws Exception { protected Engine createEngine(Project project) throws Exception {

98
src/main/java/com/metaweb/gridworks/operations/ExtendDataOperation.java
View File

@ -51,7 +51,7 @@ public class ExtendDataOperation extends EngineDependentOperation {
public ExtendDataOperation( public ExtendDataOperation(
JSONObject engineConfig, JSONObject engineConfig,
String baseColumnName, String baseColumnName,
JSONObject extension, JSONObject extension,
int columnInsertIndex int columnInsertIndex
) { ) {
super(engineConfig); super(engineConfig);
@ -142,17 +142,17 @@ public class ExtendDataOperation extends EngineDependentOperation {
FilteredRows filteredRows = engine.getAllFilteredRows(false); FilteredRows filteredRows = engine.getAllFilteredRows(false);
filteredRows.accept(_project, new RowVisitor() { filteredRows.accept(_project, new RowVisitor() {
List<Integer> _rowIndices; List<Integer> _rowIndices;
public RowVisitor init(List<Integer> rowIndices) { public RowVisitor init(List<Integer> rowIndices) {
_rowIndices = rowIndices; _rowIndices = rowIndices;
return this; return this;
} }
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
if (!includeContextual) { if (!includeContextual) {
Cell cell = row.getCell(_cellIndex); Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) { if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex); _rowIndices.add(rowIndex);
} }
} }
return false; return false;
@ -161,49 +161,49 @@ public class ExtendDataOperation extends EngineDependentOperation {
} }
protected int extendRows( protected int extendRows(
List<Integer> rowIndices, List<Integer> rowIndices,
List<DataExtension> dataExtensions, List<DataExtension> dataExtensions,
int from, int from,
int limit, int limit,
Map<String, ReconCandidate> reconCandidateMap Map<String, ReconCandidate> reconCandidateMap
) { ) {
Set<String> guids = new HashSet<String>(); Set<String> guids = new HashSet<String>();
int end; int end;
for (end = from; end < limit && guids.size() < 10; end++) { for (end = from; end < limit && guids.size() < 10; end++) {
int index = rowIndices.get(end); int index = rowIndices.get(end);
Row row = _project.rows.get(index); Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex); Cell cell = row.getCell(_cellIndex);
guids.add(cell.recon.match.topicGUID); guids.add(cell.recon.match.topicGUID);
} }
Map<String, DataExtension> map = null; Map<String, DataExtension> map = null;
try { try {
map = _job.extend(guids, reconCandidateMap); map = _job.extend(guids, reconCandidateMap);
} catch (Exception e) { } catch (Exception e) {
map = new HashMap<String, DataExtension>(); map = new HashMap<String, DataExtension>();
} }
for (int i = from; i < end; i++) { for (int i = from; i < end; i++) {
int index = rowIndices.get(i); int index = rowIndices.get(i);
Row row = _project.rows.get(index); Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex); Cell cell = row.getCell(_cellIndex);
String guid = cell.recon.match.topicGUID; String guid = cell.recon.match.topicGUID;
if (map.containsKey(guid)) { if (map.containsKey(guid)) {
dataExtensions.add(map.get(guid)); dataExtensions.add(map.get(guid));
} else { } else {
dataExtensions.add(null); dataExtensions.add(null);
} }
} }
return end; return end;
} }
public void run() { public void run() {
List<Integer> rowIndices = new ArrayList<Integer>(); List<Integer> rowIndices = new ArrayList<Integer>();
List<DataExtension> dataExtensions = new ArrayList<DataExtension>(); List<DataExtension> dataExtensions = new ArrayList<DataExtension>();
try { try {
populateRowsWithMatches(rowIndices); populateRowsWithMatches(rowIndices);
@ -216,8 +216,8 @@ public class ExtendDataOperation extends EngineDependentOperation {
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>(); Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
while (start < rowIndices.size()) { while (start < rowIndices.size()) {
int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap); int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
start = end; start = end;
_progress = end * 100 / rowIndices.size(); _progress = end * 100 / rowIndices.size();
try { try {
@ -230,10 +230,10 @@ public class ExtendDataOperation extends EngineDependentOperation {
} }
if (!_canceled) { if (!_canceled) {
List<String> columnNames = new ArrayList<String>(); List<String> columnNames = new ArrayList<String>();
for (ColumnInfo info : _job.columns) { for (ColumnInfo info : _job.columns) {
columnNames.add(StringUtils.join(info.names, " - ")); columnNames.add(StringUtils.join(info.names, " - "));
} }
List<FreebaseType> columnTypes = new ArrayList<FreebaseType>(); List<FreebaseType> columnTypes = new ArrayList<FreebaseType>();
for (ColumnInfo info : _job.columns) { for (ColumnInfo info : _job.columns) {
@ -246,13 +246,13 @@ public class ExtendDataOperation extends EngineDependentOperation {
_description, _description,
ExtendDataOperation.this, ExtendDataOperation.this,
new DataExtensionChange( new DataExtensionChange(
_baseColumnName, _baseColumnName,
_columnInsertIndex, _columnInsertIndex,
columnNames, columnNames,
columnTypes, columnTypes,
rowIndices, rowIndices,
dataExtensions, dataExtensions,
_historyEntryID) _historyEntryID)
); );
_project.history.addEntry(historyEntry); _project.history.addEntry(historyEntry);

6
src/main/java/com/metaweb/gridworks/operations/OnError.java
View File

@ -4,7 +4,7 @@
package com.metaweb.gridworks.operations; package com.metaweb.gridworks.operations;
public enum OnError { public enum OnError {
KeepOriginal, KeepOriginal,
SetToBlank, SetToBlank,
StoreError StoreError
} }

18
src/main/java/com/metaweb/gridworks/operations/ReconDiscardJudgmentsOperation.java
View File

@ -77,20 +77,20 @@ public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOpera
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
Cell cell = row.getCell(cellIndex); Cell cell = row.getCell(cellIndex);
if (cell != null && cell.recon != null) { if (cell != null && cell.recon != null) {
Recon newRecon; Recon newRecon;
if (dupReconMap.containsKey(cell.recon.id)) { if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id); newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++; newRecon.judgmentBatchSize++;
} else { } else {
newRecon = cell.recon.dup(historyEntryID); newRecon = cell.recon.dup(historyEntryID);
newRecon.match = null; newRecon.match = null;
newRecon.matchRank = -1; newRecon.matchRank = -1;
newRecon.judgment = Judgment.None; newRecon.judgment = Judgment.None;
newRecon.judgmentAction = "mass"; newRecon.judgmentAction = "mass";
newRecon.judgmentBatchSize = 1; newRecon.judgmentBatchSize = 1;
dupReconMap.put(cell.recon.id, newRecon); dupReconMap.put(cell.recon.id, newRecon);
} }
Cell newCell = new Cell(cell.value, newRecon); Cell newCell = new Cell(cell.value, newRecon);

48
src/main/java/com/metaweb/gridworks/operations/ReconJudgeSimilarCellsOperation.java
View File

@ -151,8 +151,8 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
return new RowVisitor() { return new RowVisitor() {
int _cellIndex; int _cellIndex;
List<CellChange> _cellChanges; List<CellChange> _cellChanges;
Recon _sharedNewRecon = null; Recon _sharedNewRecon = null;
Map<Long, Recon> _dupReconMap = new HashMap<Long, Recon>(); Map<Long, Recon> _dupReconMap = new HashMap<Long, Recon>();
long _historyEntryID; long _historyEntryID;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) { public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) {
@ -170,22 +170,22 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
Recon recon = null; Recon recon = null;
if (_judgment == Judgment.New && _shareNewTopics) { if (_judgment == Judgment.New && _shareNewTopics) {
if (_sharedNewRecon == null) { if (_sharedNewRecon == null) {
_sharedNewRecon = new Recon(_historyEntryID); _sharedNewRecon = new Recon(_historyEntryID);
_sharedNewRecon.judgment = Judgment.New; _sharedNewRecon.judgment = Judgment.New;
_sharedNewRecon.judgmentBatchSize = 0; _sharedNewRecon.judgmentBatchSize = 0;
_sharedNewRecon.judgmentAction = "similar"; _sharedNewRecon.judgmentAction = "similar";
} }
_sharedNewRecon.judgmentBatchSize++; _sharedNewRecon.judgmentBatchSize++;
recon = _sharedNewRecon; recon = _sharedNewRecon;
} else { } else {
if (_dupReconMap.containsKey(cell.recon.id)) { if (_dupReconMap.containsKey(cell.recon.id)) {
recon = _dupReconMap.get(cell.recon.id); recon = _dupReconMap.get(cell.recon.id);
recon.judgmentBatchSize++; recon.judgmentBatchSize++;
} else { } else {
recon = cell.recon.dup(_historyEntryID); recon = cell.recon.dup(_historyEntryID);
recon.judgmentBatchSize = 1; recon.judgmentBatchSize = 1;
recon.matchRank = -1; recon.matchRank = -1;
recon.judgmentAction = "similar"; recon.judgmentAction = "similar";
@ -194,12 +194,12 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
recon.match = _match; recon.match = _match;
if (recon.candidates != null) { if (recon.candidates != null) {
for (int m = 0; m < recon.candidates.size(); m++) { for (int m = 0; m < recon.candidates.size(); m++) {
if (recon.candidates.get(m).topicGUID.equals(_match.topicGUID)) { if (recon.candidates.get(m).topicGUID.equals(_match.topicGUID)) {
recon.matchRank = m; recon.matchRank = m;
break; break;
} }
} }
} }
} else if (_judgment == Judgment.New) { } else if (_judgment == Judgment.New) {
recon.judgment = Recon.Judgment.New; recon.judgment = Recon.Judgment.New;
@ -209,8 +209,8 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
recon.match = null; recon.match = null;
} }
_dupReconMap.put(cell.recon.id, recon); _dupReconMap.put(cell.recon.id, recon);
} }
} }
Cell newCell = new Cell(cell.value, recon); Cell newCell = new Cell(cell.value, recon);

34
src/main/java/com/metaweb/gridworks/operations/ReconMarkNewTopicsOperation.java
View File

@ -72,10 +72,10 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio
Column column = project.columnModel.getColumnByName(_columnName); Column column = project.columnModel.getColumnByName(_columnName);
return new RowVisitor() { return new RowVisitor() {
int cellIndex; int cellIndex;
List<CellChange> cellChanges; List<CellChange> cellChanges;
Map<String, Recon> sharedRecons = new HashMap<String, Recon>(); Map<String, Recon> sharedRecons = new HashMap<String, Recon>();
Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>(); Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>();
long historyEntryID; long historyEntryID;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) { public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) {
@ -98,25 +98,25 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio
recon = new Recon(historyEntryID); recon = new Recon(historyEntryID);
recon.judgment = Judgment.New; recon.judgment = Judgment.New;
recon.judgmentBatchSize = 1; recon.judgmentBatchSize = 1;
recon.judgmentAction = "mass"; recon.judgmentAction = "mass";
sharedRecons.put(s, recon); sharedRecons.put(s, recon);
} }
} else { } else {
long reconID = cell.recon == null ? 0 : cell.recon.id; long reconID = cell.recon == null ? 0 : cell.recon.id;
if (dupReconMap.containsKey(reconID)) { if (dupReconMap.containsKey(reconID)) {
recon = dupReconMap.get(reconID); recon = dupReconMap.get(reconID);
recon.judgmentBatchSize++; recon.judgmentBatchSize++;
} else { } else {
recon = cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID); recon = cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID);
recon.match = null; recon.match = null;
recon.matchRank = -1; recon.matchRank = -1;
recon.judgment = Judgment.New; recon.judgment = Judgment.New;
recon.judgmentBatchSize = 1; recon.judgmentBatchSize = 1;
recon.judgmentAction = "mass"; recon.judgmentAction = "mass";
dupReconMap.put(reconID, recon); dupReconMap.put(reconID, recon);
} }
} }
Cell newCell = new Cell(cell.value, recon); Cell newCell = new Cell(cell.value, recon);

24
src/main/java/com/metaweb/gridworks/operations/ReconMatchBestCandidatesOperation.java
View File

@ -63,9 +63,9 @@ public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOp
Column column = project.columnModel.getColumnByName(_columnName); Column column = project.columnModel.getColumnByName(_columnName);
return new RowVisitor() { return new RowVisitor() {
int cellIndex; int cellIndex;
List<CellChange> cellChanges; List<CellChange> cellChanges;
Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>(); Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>();
long historyEntryID; long historyEntryID;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) { public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) {
@ -81,20 +81,20 @@ public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOp
if (cell != null && cell.recon != null) { if (cell != null && cell.recon != null) {
ReconCandidate candidate = cell.recon.getBestCandidate(); ReconCandidate candidate = cell.recon.getBestCandidate();
if (candidate != null) { if (candidate != null) {
Recon newRecon; Recon newRecon;
if (dupReconMap.containsKey(cell.recon.id)) { if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id); newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++; newRecon.judgmentBatchSize++;
} else { } else {
newRecon = cell.recon.dup(historyEntryID); newRecon = cell.recon.dup(historyEntryID);
newRecon.judgmentBatchSize = 1; newRecon.judgmentBatchSize = 1;
newRecon.match = candidate; newRecon.match = candidate;
newRecon.matchRank = 0; newRecon.matchRank = 0;
newRecon.judgment = Judgment.Matched; newRecon.judgment = Judgment.Matched;
newRecon.judgmentAction = "mass"; newRecon.judgmentAction = "mass";
dupReconMap.put(cell.recon.id, newRecon); dupReconMap.put(cell.recon.id, newRecon);
} }
Cell newCell = new Cell( Cell newCell = new Cell(
cell.value, cell.value,
newRecon newRecon

22
src/main/java/com/metaweb/gridworks/operations/ReconMatchSpecificTopicOperation.java
View File

@ -109,24 +109,24 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
} }
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) { public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
Cell cell = row.getCell(cellIndex); Cell cell = row.getCell(cellIndex);
if (cell != null) { if (cell != null) {
long reconID = cell.recon != null ? cell.recon.id : 0; long reconID = cell.recon != null ? cell.recon.id : 0;
Recon newRecon; Recon newRecon;
if (dupReconMap.containsKey(reconID)) { if (dupReconMap.containsKey(reconID)) {
newRecon = dupReconMap.get(reconID); newRecon = dupReconMap.get(reconID);
newRecon.judgmentBatchSize++; newRecon.judgmentBatchSize++;
} else { } else {
newRecon = cell.recon != null ? cell.recon.dup(historyEntryID) : new Recon(historyEntryID); newRecon = cell.recon != null ? cell.recon.dup(historyEntryID) : new Recon(historyEntryID);
newRecon.match = match; newRecon.match = match;
newRecon.matchRank = -1; newRecon.matchRank = -1;
newRecon.judgment = Judgment.Matched; newRecon.judgment = Judgment.Matched;
newRecon.judgmentAction = "mass"; newRecon.judgmentAction = "mass";
newRecon.judgmentBatchSize = 1; newRecon.judgmentBatchSize = 1;
dupReconMap.put(reconID, newRecon); dupReconMap.put(reconID, newRecon);
} }
Cell newCell = new Cell( Cell newCell = new Cell(
cell.value, cell.value,

24
src/main/java/com/metaweb/gridworks/operations/ReconOperation.java
View File

@ -142,17 +142,17 @@ public class ReconOperation extends EngineDependentOperation {
writer.endObject(); writer.endObject();
if (_reconConfig instanceof HeuristicReconConfig) { if (_reconConfig instanceof HeuristicReconConfig) {
writer.object(); writer.object();
writer.key("action"); writer.value("createFacet"); writer.key("action"); writer.value("createFacet");
writer.key("facetType"); writer.value("range"); writer.key("facetType"); writer.value("range");
writer.key("facetConfig"); writer.key("facetConfig");
writer.object(); writer.object();
writer.key("name"); writer.value(_columnName + ": best candidate's score"); writer.key("name"); writer.value(_columnName + ": best candidate's score");
writer.key("columnName"); writer.value(_columnName); writer.key("columnName"); writer.value(_columnName);
writer.key("expression"); writer.value("cell.recon.best.score"); writer.key("expression"); writer.value("cell.recon.best.score");
writer.key("mode"); writer.value("range"); writer.key("mode"); writer.value("range");
writer.endObject(); writer.endObject();
writer.endObject(); writer.endObject();
} }
writer.endArray(); writer.endArray();
writer.endObject(); writer.endObject();
@ -234,7 +234,7 @@ public class ReconOperation extends EngineDependentOperation {
List<ReconEntry> entries = groups.get(j).entries; List<ReconEntry> entries = groups.get(j).entries;
if (recon != null) { if (recon != null) {
recon.judgmentBatchSize = entries.size(); recon.judgmentBatchSize = entries.size();
} }
for (ReconEntry entry : entries) { for (ReconEntry entry : entries) {

38
src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java
View File

@ -40,22 +40,22 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
} }
static public OnError stringToOnError(String s) { static public OnError stringToOnError(String s) {
if ("set-to-blank".equalsIgnoreCase(s)) { if ("set-to-blank".equalsIgnoreCase(s)) {
return OnError.SetToBlank; return OnError.SetToBlank;
} else if ("store-error".equalsIgnoreCase(s)) { } else if ("store-error".equalsIgnoreCase(s)) {
return OnError.StoreError; return OnError.StoreError;
} else { } else {
return OnError.KeepOriginal; return OnError.KeepOriginal;
} }
} }
static public String onErrorToString(OnError onError) { static public String onErrorToString(OnError onError) {
if (onError == OnError.SetToBlank) { if (onError == OnError.SetToBlank) {
return "set-to-blank"; return "set-to-blank";
} else if (onError == OnError.StoreError) { } else if (onError == OnError.StoreError) {
return "store-error"; return "store-error";
} else { } else {
return "keep-original"; return "keep-original";
} }
} }
public TextTransformOperation( public TextTransformOperation(
@ -136,11 +136,11 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
} else { } else {
Serializable newValue = ExpressionUtils.wrapStorable(o); Serializable newValue = ExpressionUtils.wrapStorable(o);
if (ExpressionUtils.isError(newValue)) { if (ExpressionUtils.isError(newValue)) {
if (_onError == OnError.KeepOriginal) { if (_onError == OnError.KeepOriginal) {
return false; return false;
} else if (_onError == OnError.SetToBlank) { } else if (_onError == OnError.SetToBlank) {
newValue = null; newValue = null;
} }
} }
if (!ExpressionUtils.sameValue(oldValue, newValue)) { if (!ExpressionUtils.sameValue(oldValue, newValue)) {

64
src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java
View File

@ -35,19 +35,19 @@ public class FreebaseDataExtensionJob {
} }
static public class ColumnInfo { static public class ColumnInfo {
final public List<String> names; final public List<String> names;
final public List<String> path; final public List<String> path;
final public FreebaseType expectedType; final public FreebaseType expectedType;
protected ColumnInfo(List<String> names, List<String> path, FreebaseType expectedType) { protected ColumnInfo(List<String> names, List<String> path, FreebaseType expectedType) {
this.names = names; this.names = names;
this.path = path; this.path = path;
this.expectedType = expectedType; this.expectedType = expectedType;
} }
} }
final public JSONObject extension; final public JSONObject extension;
final public int columnCount; final public int columnCount;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>(); final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
public FreebaseDataExtensionJob(JSONObject obj) throws JSONException { public FreebaseDataExtensionJob(JSONObject obj) throws JSONException {
@ -57,9 +57,9 @@ public class FreebaseDataExtensionJob {
} }
public Map<String, FreebaseDataExtensionJob.DataExtension> extend( public Map<String, FreebaseDataExtensionJob.DataExtension> extend(
Set<String> guids, Set<String> guids,
Map<String, ReconCandidate> reconCandidateMap Map<String, ReconCandidate> reconCandidateMap
) throws Exception { ) throws Exception {
StringWriter writer = new StringWriter(); StringWriter writer = new StringWriter();
formulateQuery(guids, extension, writer); formulateQuery(guids, extension, writer);
@ -92,9 +92,9 @@ public class FreebaseDataExtensionJob {
} }
protected FreebaseDataExtensionJob.DataExtension collectResult( protected FreebaseDataExtensionJob.DataExtension collectResult(
JSONObject obj, JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap Map<String, ReconCandidate> reconCandidateMap
) throws JSONException { ) throws JSONException {
List<Object[]> rows = new ArrayList<Object[]>(); List<Object[]> rows = new ArrayList<Object[]>();
collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap); collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap);
@ -125,12 +125,12 @@ public class FreebaseDataExtensionJob {
JSONObject obj, JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap Map<String, ReconCandidate> reconCandidateMap
) throws JSONException { ) throws JSONException {
String guid = obj.getString("guid"); String guid = obj.getString("guid");
ReconCandidate rc; ReconCandidate rc;
if (reconCandidateMap.containsKey(guid)) { if (reconCandidateMap.containsKey(guid)) {
rc = reconCandidateMap.get(guid); rc = reconCandidateMap.get(guid);
} else { } else {
rc = new ReconCandidate( rc = new ReconCandidate(
obj.getString("id"), obj.getString("id"),
obj.getString("guid"), obj.getString("guid"),
obj.getString("name"), obj.getString("name"),
@ -138,8 +138,8 @@ public class FreebaseDataExtensionJob {
100 100
); );
reconCandidateMap.put(guid, rc); reconCandidateMap.put(guid, rc);
} }
storeCell(rows, row, col, rc, reconCandidateMap); storeCell(rows, row, col, rc, reconCandidateMap);
} }
@ -372,17 +372,17 @@ public class FreebaseDataExtensionJob {
} }
static protected int countColumns(JSONObject obj, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException { static protected int countColumns(JSONObject obj, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
String name = obj.getString("name"); String name = obj.getString("name");
List<String> names2 = null; List<String> names2 = null;
List<String> path2 = null; List<String> path2 = null;
if (columns != null) { if (columns != null) {
names2 = new ArrayList<String>(names); names2 = new ArrayList<String>(names);
names2.add(name); names2.add(name);
path2 = new ArrayList<String>(path); path2 = new ArrayList<String>(path);
path2.add(obj.getString("id")); path2.add(obj.getString("id"));
} }
if (obj.has("properties") && !obj.isNull("properties")) { if (obj.has("properties") && !obj.isNull("properties")) {
boolean included = (obj.has("included") && obj.getBoolean("included")); boolean included = (obj.has("included") && obj.getBoolean("included"));
@ -394,7 +394,7 @@ public class FreebaseDataExtensionJob {
} }
return (included ? 1 : 0) + return (included ? 1 : 0) +
countColumns(obj.getJSONArray("properties"), columns, names2, path2); countColumns(obj.getJSONArray("properties"), columns, names2, path2);
} else { } else {
if (columns != null) { if (columns != null) {
JSONObject expected = obj.getJSONObject("expected"); JSONObject expected = obj.getJSONObject("expected");