s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

detab + dedos for java files (no functional changes)

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@594 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Stefano Mazzocchi 2010-05-04 23:24:48 +00:00
parent d35930fb66
commit 92ecc0c0f5
198 changed files with 17612 additions and 17612 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
src/main/java/com/metaweb/gridworks/ProjectManager.java
View File

@ -34,7 +34,7 @@ public class ProjectManager {
protected File _workspaceDir;
protected Map<Long, ProjectMetadata> _projectsMetadata;
protected List<String> _expressions;
protected List<String> _expressions;
final static Logger logger = LoggerFactory.getLogger("project_manager");
@ -70,48 +70,48 @@ public class ProjectManager {
String os = Configurations.get("os.name").toLowerCase();
if (os.contains("windows")) {
try {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet.
try {
// NOTE(SM): finding the "local data app" in windows from java is actually a PITA
// see http://stackoverflow.com/questions/1198911/how-to-get-local-application-data-folder-in-java
// so we're using a library that uses JNI to ask directly the win32 APIs,
// it's not elegant but it's the safest bet.
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(localDataPath.getPath());
data.mkdirs();
return data;
} catch (Error e) {
/*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path.
*/
DataPath localDataPath = JDataPathSystem.getLocalSystem().getLocalDataPath("Gridworks");
File data = new File(localDataPath.getPath());
data.mkdirs();
return data;
} catch (Error e) {
/*
* The above trick can fail, particularly on a 64-bit OS as the jdatapath.dll
* we include is compiled for 32-bit. In this case, we just have to dig up
* environment variables and try our best to find a user-specific path.
*/
logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
logger.warn("Failed to use jdatapath to detect user data path: resorting to environment variables");
File parentDir = null;
{
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
String appData = System.getenv("APPDATA");
if (appData != null && appData.length() > 0) {
// e.g., C:\Users\[userid]\AppData\Roaming
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
parentDir = new File(appData);
} else {
String userProfile = System.getenv("USERPROFILE");
if (userProfile != null && userProfile.length() > 0) {
// e.g., C:\Users\[userid]
parentDir = new File(userProfile);
}
}
}
if (parentDir == null) {
parentDir = new File(".");
}
if (parentDir == null) {
parentDir = new File(".");
}
File data = new File(parentDir, "Gridworks");
data.mkdirs();
File data = new File(parentDir, "Gridworks");
data.mkdirs();
return data;
}
return data;
}
} else if (os.contains("mac os x")) {
// on macosx, use "~/Library/Application Support"
String home = System.getProperty("user.home");
@ -262,16 +262,16 @@ public class ProjectManager {
public void addLatestExpression(String s) {
synchronized (this) {
_expressions.remove(s);
_expressions.add(0, s);
while (_expressions.size() > s_expressionHistoryMax) {
_expressions.remove(_expressions.size() - 1);
}
_expressions.remove(s);
_expressions.add(0, s);
while (_expressions.size() > s_expressionHistoryMax) {
_expressions.remove(_expressions.size() - 1);
}
}
}
public List<String> getExpressions() {
return _expressions;
return _expressions;
}
public void save(boolean allModified) {
@ -285,8 +285,8 @@ public class ProjectManager {
*/
protected void saveWorkspace() {
synchronized (this) {
File tempFile = new File(_workspaceDir, "workspace.temp.json");
try {
File tempFile = new File(_workspaceDir, "workspace.temp.json");
try {
saveToFile(tempFile);
} catch (Exception e) {
e.printStackTrace();

20
src/main/java/com/metaweb/gridworks/ProjectMetadata.java
View File

@ -170,9 +170,9 @@ public class ProjectMetadata implements Jsonizable {
}
public void setEncodingConfidence(String confidence) {
if (confidence != null) {
this.setEncodingConfidence(Integer.parseInt(confidence));
}
if (confidence != null) {
this.setEncodingConfidence(Integer.parseInt(confidence));
}
}
public int getEncodingConfidence() {
@ -196,16 +196,16 @@ public class ProjectMetadata implements Jsonizable {
}
public void addLatestExpression(String s) {
_expressions.remove(s);
_expressions.add(0, s);
while (_expressions.size() > s_expressionHistoryMax) {
_expressions.remove(_expressions.size() - 1);
}
_expressions.remove(s);
_expressions.add(0, s);
while (_expressions.size() > s_expressionHistoryMax) {
_expressions.remove(_expressions.size() - 1);
}
ProjectManager.singleton.addLatestExpression(s);
ProjectManager.singleton.addLatestExpression(s);
}
public List<String> getExpressions() {
return _expressions;
return _expressions;
}
}

34
src/main/java/com/metaweb/gridworks/browsing/ConjunctiveFilteredRows.java
View File

@ -14,7 +14,7 @@ import com.metaweb.gridworks.model.Row;
public class ConjunctiveFilteredRows implements FilteredRows {
final protected List<RowFilter> _rowFilters = new LinkedList<RowFilter>();
final protected boolean _includeContextual;
final protected boolean _includeDependent;
final protected boolean _includeDependent;
public ConjunctiveFilteredRows(boolean includeContextual, boolean includeDependent) {
_includeContextual = includeContextual;
@ -27,34 +27,34 @@ public class ConjunctiveFilteredRows implements FilteredRows {
public void accept(Project project, RowVisitor visitor) {
int lastVisitedRowRowIndex = -1;
int lastRecordRowAcceptedRowIndex = -1;
int lastRecordRowAcceptedRowIndex = -1;
int c = project.rows.size();
int c = project.rows.size();
for (int rowIndex = 0; rowIndex < c; rowIndex++) {
Row row = project.rows.get(rowIndex);
if (matchRow(project, rowIndex, row)) {
if (row.recordIndex >= 0) {
lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself
}
if (row.recordIndex >= 0) {
lastRecordRowAcceptedRowIndex = rowIndex; // this is a record row itself
}
visitRow(project, visitor, rowIndex, row, lastVisitedRowRowIndex);
visitRow(project, visitor, rowIndex, row, lastVisitedRowRowIndex);
lastVisitedRowRowIndex = rowIndex;
} else if (
// this row doesn't match by itself but ...
// we want to include dependent rows
_includeDependent &&
// and this row is a dependent row since it's not a record row
row.recordIndex < 0 &&
row.contextRows != null &&
row.contextRows.size() > 0 &&
_includeDependent &&
// and this row is a dependent row since it's not a record row
row.recordIndex < 0 &&
row.contextRows != null &&
row.contextRows.size() > 0 &&
row.contextRows.get(0) == lastRecordRowAcceptedRowIndex
row.contextRows.get(0) == lastRecordRowAcceptedRowIndex
) {
// this row depends on the last previously matched record row,
// so we visit it as well as a dependent row
// this row depends on the last previously matched record row,
// so we visit it as well as a dependent row
visitor.visit(project, rowIndex, row, false, true);
lastVisitedRowRowIndex = rowIndex;
@ -63,7 +63,7 @@ public class ConjunctiveFilteredRows implements FilteredRows {
}
protected void visitRow(Project project, RowVisitor visitor, int rowIndex, Row row, int lastVisitedRow) {
if (_includeContextual && // we need to include any context row and
if (_includeContextual && // we need to include any context row and
row.contextRows != null && // this row itself isn't a context row and
lastVisitedRow < rowIndex - 1 // there is definitely some rows before this row
// that we haven't visited yet
@ -80,7 +80,7 @@ public class ConjunctiveFilteredRows implements FilteredRows {
lastVisitedRow = contextRowIndex;
}
}
}
}
visitor.visit(project, rowIndex, row, false, false);
}

60
src/main/java/com/metaweb/gridworks/browsing/Engine.java
View File

@ -24,7 +24,7 @@ import com.metaweb.gridworks.model.Project;
public class Engine implements Jsonizable {
protected Project _project;
protected List<Facet> _facets = new LinkedList<Facet>();
protected boolean _includeDependent;
protected boolean _includeDependent;
public final static String INCLUDE_DEPENDENT = "includeDependent";
@ -50,38 +50,38 @@ public class Engine implements Jsonizable {
}
public void initializeFromJSON(JSONObject o) throws Exception {
if (o == null) {
return;
}
if (o == null) {
return;
}
if (o.has("facets") && !o.isNull("facets")) {
JSONArray a = o.getJSONArray("facets");
int length = a.length();
if (o.has("facets") && !o.isNull("facets")) {
JSONArray a = o.getJSONArray("facets");
int length = a.length();
for (int i = 0; i < length; i++) {
JSONObject fo = a.getJSONObject(i);
String type = fo.has("type") ? fo.getString("type") : "list";
for (int i = 0; i < length; i++) {
JSONObject fo = a.getJSONObject(i);
String type = fo.has("type") ? fo.getString("type") : "list";
Facet facet = null;
if ("list".equals(type)) {
facet = new ListFacet();
} else if ("range".equals(type)) {
facet = new RangeFacet();
Facet facet = null;
if ("list".equals(type)) {
facet = new ListFacet();
} else if ("range".equals(type)) {
facet = new RangeFacet();
} else if ("scatterplot".equals(type)) {
facet = new ScatterplotFacet();
} else if ("text".equals(type)) {
facet = new TextSearchFacet();
}
} else if ("text".equals(type)) {
facet = new TextSearchFacet();
}
if (facet != null) {
facet.initializeFromJSON(_project, fo);
_facets.add(facet);
}
}
}
if (facet != null) {
facet.initializeFromJSON(_project, fo);
_facets.add(facet);
}
}
}
if (o.has(INCLUDE_DEPENDENT) && !o.isNull(INCLUDE_DEPENDENT)) {
_includeDependent = o.getBoolean(INCLUDE_DEPENDENT);
_includeDependent = o.getBoolean(INCLUDE_DEPENDENT);
}
}
@ -98,11 +98,11 @@ public class Engine implements Jsonizable {
writer.object();
writer.key("facets");
writer.array();
for (Facet facet : _facets) {
facet.write(writer, options);
}
writer.endArray();
writer.array();
for (Facet facet : _facets) {
facet.write(writer, options);
}
writer.endArray();
writer.key(INCLUDE_DEPENDENT); writer.value(_includeDependent);
writer.endObject();
}

14
src/main/java/com/metaweb/gridworks/browsing/RowVisitor.java
View File

@ -11,12 +11,12 @@ import com.metaweb.gridworks.model.Row;
*/
public interface RowVisitor {
public boolean visit(
Project project,
int rowIndex, // zero-based row index
Row row,
boolean contextual, // true if this row is included because it's the context row
// of a matched row, that is, a matched row depends on it
boolean dependent // true if this row is included because it depends on a matched row,
// that is, it depends on a matched row
Project project,
int rowIndex, // zero-based row index
Row row,
boolean contextual, // true if this row is included because it's the context row
// of a matched row, that is, a matched row depends on it
boolean dependent // true if this row is included because it depends on a matched row,
// that is, it depends on a matched row
);
}

2
src/main/java/com/metaweb/gridworks/browsing/facets/ExpressionNumericRowBinner.java
View File

@ -110,7 +110,7 @@ public class ExpressionNumericRowBinner implements RowVisitor {
int bin = (int) Math.floor((d - _index.getMin()) / _index.getStep());
if (bin >= 0 && bin < bins.length) { // as a precaution
bins[bin]++;
bins[bin]++;
}
} else {
rowHasError = true;

2
src/main/java/com/metaweb/gridworks/browsing/facets/ListFacet.java
View File

@ -107,7 +107,7 @@ public class ListFacet implements Facet {
_errorMessage = "No column named " + _columnName;
}
} else {
_cellIndex = -1;
_cellIndex = -1;
}
try {

4
src/main/java/com/metaweb/gridworks/browsing/facets/NumericBinIndex.java
View File

@ -156,8 +156,8 @@ public class NumericBinIndex {
}
if (_max <= originalMax) {
_max += _step;
binCount++;
_max += _step;
binCount++;
}
_bins = new int[(int) Math.round(binCount)];

12
src/main/java/com/metaweb/gridworks/commands/Command.java
View File

@ -119,11 +119,11 @@ public abstract class Command {
}
static protected void performProcessAndRespond(
HttpServletRequest request,
HttpServletResponse response,
Project project,
Process process
) throws Exception {
HttpServletRequest request,
HttpServletResponse response,
Project project,
Process process
) throws Exception {
response.setCharacterEncoding("UTF-8");
response.setHeader("Content-Type", "application/json");
@ -141,7 +141,7 @@ public abstract class Command {
w.flush();
w.close();
} else {
respond(response, "{ \"code\" : \"pending\" }");
respond(response, "{ \"code\" : \"pending\" }");
}
}

6
src/main/java/com/metaweb/gridworks/commands/edit/ExportProjectCommand.java
View File

@ -31,7 +31,7 @@ public class ExportProjectCommand extends Command {
OutputStream os = response.getOutputStream();
try {
gzipTarToOutputStream(
gzipTarToOutputStream(
ProjectManager.singleton.getProjectDir(project.id),
os
);
@ -44,9 +44,9 @@ public class ExportProjectCommand extends Command {
}
protected void gzipTarToOutputStream(File dir, OutputStream os) throws IOException {
GZIPOutputStream gos = new GZIPOutputStream(os);
GZIPOutputStream gos = new GZIPOutputStream(os);
try {
tarToOutputStream(dir, gos);
tarToOutputStream(dir, gos);
} finally {
gos.close();
}

6
src/main/java/com/metaweb/gridworks/commands/edit/ImportProjectCommand.java
View File

@ -138,10 +138,10 @@ public class ImportProjectCommand extends Command {
destDir.mkdirs();
if (gziped) {
GZIPInputStream gis = new GZIPInputStream(inputStream);
untar(destDir, gis);
GZIPInputStream gis = new GZIPInputStream(inputStream);
untar(destDir, gis);
} else {
untar(destDir, inputStream);
untar(destDir, inputStream);
}
}

12
src/main/java/com/metaweb/gridworks/commands/edit/TextTransformCommand.java
View File

@ -27,12 +27,12 @@ public class TextTransformCommand extends EngineDependentCommand {
}
return new TextTransformOperation(
engineConfig,
columnName,
expression,
TextTransformOperation.stringToOnError(onError),
repeat,
repeatCount
engineConfig,
columnName,
expression,
TextTransformOperation.stringToOnError(onError),
repeat,
repeatCount
);
}
}

12
src/main/java/com/metaweb/gridworks/commands/edit/UndoRedoCommand.java
View File

@ -21,19 +21,19 @@ public class UndoRedoCommand extends Command {
long lastDoneID = -1;
String lastDoneIDString = request.getParameter("lastDoneID");
if (lastDoneIDString != null) {
lastDoneID = Long.parseLong(lastDoneIDString);
lastDoneID = Long.parseLong(lastDoneIDString);
} else {
String undoIDString = request.getParameter("undoID");
String undoIDString = request.getParameter("undoID");
if (undoIDString != null) {
long undoID = Long.parseLong(undoIDString);
long undoID = Long.parseLong(undoIDString);
lastDoneID = project.history.getPrecedingEntryID(undoID);
lastDoneID = project.history.getPrecedingEntryID(undoID);
}
}
boolean done = lastDoneID == -1 ||
project.processManager.queueProcess(
new HistoryProcess(project, lastDoneID));
project.processManager.queueProcess(
new HistoryProcess(project, lastDoneID));
respond(response, "{ \"code\" : " + (done ? "\"ok\"" : "\"pending\"") + " }");
}

34
src/main/java/com/metaweb/gridworks/commands/info/GetExpressionHistoryCommand.java
View File

@ -36,23 +36,23 @@ public class GetExpressionHistoryCommand extends Command {
JSONWriter writer = new JSONWriter(response.getWriter());
writer.object();
writer.key("expressions");
writer.array();
for (String s : localExpressions) {
writer.object();
writer.key("code"); writer.value(s);
writer.key("global"); writer.value(false);
writer.endObject();
done.add(s);
}
for (String s : globalExpressions) {
if (!done.contains(s)) {
writer.object();
writer.key("code"); writer.value(s);
writer.key("global"); writer.value(true);
writer.endObject();
}
}
writer.endArray();
writer.array();
for (String s : localExpressions) {
writer.object();
writer.key("code"); writer.value(s);
writer.key("global"); writer.value(false);
writer.endObject();
done.add(s);
}
for (String s : globalExpressions) {
if (!done.contains(s)) {
writer.object();
writer.key("code"); writer.value(s);
writer.key("global"); writer.value(true);
writer.endObject();
}
}
writer.endArray();
writer.endObject();
} catch (Exception e) {
respondException(response, e);

8
src/main/java/com/metaweb/gridworks/commands/recon/ReconJudgeOneCellCommand.java
View File

@ -154,10 +154,10 @@ public class ReconJudgeOneCellCommand extends Command {
newCell.recon.match = this.match;
for (int m = 0; m < newCell.recon.candidates.size(); m++) {
if (newCell.recon.candidates.get(m).topicGUID.equals(this.match.topicGUID)) {
newCell.recon.matchRank = m;
break;
}
if (newCell.recon.candidates.get(m).topicGUID.equals(this.match.topicGUID)) {
newCell.recon.matchRank = m;
break;
}
}
description = "Match " + this.match.topicName +

22
src/main/java/com/metaweb/gridworks/commands/util/PreviewExtendDataCommand.java
View File

@ -82,19 +82,19 @@ public class PreviewExtendDataCommand extends Command {
writer.key("columns");
writer.array();
for (ColumnInfo info : job.columns) {
writer.object();
writer.object();
writer.key("names");
writer.array();
for (String name : info.names) {
writer.value(name);
}
writer.endArray();
writer.array();
for (String name : info.names) {
writer.value(name);
}
writer.endArray();
writer.key("path");
writer.array();
for (String id : info.path) {
writer.value(id);
}
writer.endArray();
writer.array();
for (String id : info.path) {
writer.value(id);
}
writer.endArray();
writer.endObject();
}
writer.endArray();

2
src/main/java/com/metaweb/gridworks/expr/JythonEvaluable.java
View File

@ -15,7 +15,7 @@ import org.python.core.PyString;
import org.python.util.PythonInterpreter;
public class JythonEvaluable implements Evaluable {
private static final String s_functionName = "___temp___";
private static final String s_functionName = "___temp___";
private static PythonInterpreter _engine;
static {

6
src/main/java/com/metaweb/gridworks/expr/functions/Get.java
View File

@ -51,11 +51,11 @@ public class Get implements Function {
if (end > start) {
if (v.getClass().isArray()) {
Object[] a2 = new Object[end - start];
Object[] a2 = new Object[end - start];
System.arraycopy((Object[]) v, start, a2, 0, end - start);
System.arraycopy((Object[]) v, start, a2, 0, end - start);
return a2;
return a2;
} else {
return ExpressionUtils.toObjectList(v).subList(start, end);
}

2
src/main/java/com/metaweb/gridworks/expr/functions/Length.java
View File

@ -21,7 +21,7 @@ public class Length implements Function {
Object[] a = (Object[]) v;
return a.length;
} else if (v instanceof Collection<?>) {
return ((Collection<?>) v).size();
return ((Collection<?>) v).size();
} else {
String s = (v instanceof String ? (String) v : v.toString());
return s.length();

32
src/main/java/com/metaweb/gridworks/expr/functions/arrays/Join.java
View File

@ -25,23 +25,23 @@ public class Join implements Function {
StringBuffer sb = new StringBuffer();
if (v.getClass().isArray()) {
for (Object o : (Object[]) v) {
if (o != null) {
if (sb.length() > 0) {
sb.append(separator);
}
sb.append(o.toString());
}
}
for (Object o : (Object[]) v) {
if (o != null) {
if (sb.length() > 0) {
sb.append(separator);
}
sb.append(o.toString());
}
}
} else {
for (Object o : ExpressionUtils.toObjectList(v)) {
if (o != null) {
if (sb.length() > 0) {
sb.append(separator);
}
sb.append(o.toString());
}
}
for (Object o : ExpressionUtils.toObjectList(v)) {
if (o != null) {
if (sb.length() > 0) {
sb.append(separator);
}
sb.append(o.toString());
}
}
}
return sb.toString();

4
src/main/java/com/metaweb/gridworks/expr/functions/arrays/Sort.java
View File

@ -14,8 +14,8 @@ import com.metaweb.gridworks.gel.Function;
public class Sort implements Function {
@SuppressWarnings("unchecked")
public Object call(Properties bindings, Object[] args) {
@SuppressWarnings("unchecked")
public Object call(Properties bindings, Object[] args) {
if (args.length == 1) {
Object v = args[0];

36
src/main/java/com/metaweb/gridworks/gel/Scanner.java
View File

@ -146,18 +146,18 @@ public class Scanner {
} else if (c == '\\') {
_index++; // skip escaping marker
if (_index < _limit) {
char c2 = _text.charAt(_index);
if (c2 == 't') {
sb.append('\t');
} else if (c2 == 'n') {
sb.append('\n');
} else if (c2 == 'r') {
sb.append('\r');
} else if (c2 == '\\') {
sb.append('\\');
} else {
sb.append(c2);
}
char c2 = _text.charAt(_index);
if (c2 == 't') {
sb.append('\t');
} else if (c2 == 'n') {
sb.append('\n');
} else if (c2 == 'r') {
sb.append('\r');
} else if (c2 == '\\') {
sb.append('\\');
} else {
sb.append(c2);
}
}
} else {
sb.append(c);
@ -170,12 +170,12 @@ public class Scanner {
} else if (Character.isLetter(c) || c == '_') { // identifier
while (_index < _limit) {
char c1 = _text.charAt(_index);
if (c1 == '_' || Character.isLetterOrDigit(c1)) {
_index++;
} else {
break;
}
char c1 = _text.charAt(_index);
if (c1 == '_' || Character.isLetterOrDigit(c1)) {
_index++;
} else {
break;
}
}
return new Token(

16
src/main/java/com/metaweb/gridworks/history/History.java
View File

@ -140,7 +140,7 @@ public class History implements Jsonizable {
public long getPrecedingEntryID(long entryID) {
if (entryID == 0) {
return -1;
return -1;
} else {
for (int i = 0; i < _pastEntries.size(); i++) {
if (_pastEntries.get(i).id == entryID) {
@ -150,13 +150,13 @@ public class History implements Jsonizable {
for (int i = 0; i < _futureEntries.size(); i++) {
if (_futureEntries.get(i).id == entryID) {
if (i > 0) {
return _futureEntries.get(i - 1).id;
} else if (_pastEntries.size() > 0) {
return _pastEntries.get(_pastEntries.size() - 1).id;
} else {
return 0;
}
if (i > 0) {
return _futureEntries.get(i - 1).id;
} else if (_pastEntries.size() > 0) {
return _pastEntries.get(_pastEntries.size() - 1).id;
} else {
return 0;
}
}
}
}

32
src/main/java/com/metaweb/gridworks/importers/ExcelImporter.java
View File

@ -114,14 +114,14 @@ public class ExcelImporter implements Importer {
*/
Map<String, Integer> nameToIndex = new HashMap<String, Integer>();
for (int c = 0; c < nonBlankIndices.size(); c++) {
String cell = nonBlankHeaderStrings.get(c);
String cell = nonBlankHeaderStrings.get(c);
if (nameToIndex.containsKey(cell)) {
int index = nameToIndex.get(cell);
nameToIndex.put(cell, index + 1);
int index = nameToIndex.get(cell);
nameToIndex.put(cell, index + 1);
cell = cell.contains(" ") ? (cell + " " + index) : (cell + index);
cell = cell.contains(" ") ? (cell + " " + index) : (cell + index);
} else {
nameToIndex.put(cell, 2);
nameToIndex.put(cell, 2);
}
Column column = new Column(c, cell);
@ -209,19 +209,19 @@ public class ExcelImporter implements Importer {
}
if (reconMap.containsKey(id)) {
recon = reconMap.get(id);
recon.judgmentBatchSize++;
recon = reconMap.get(id);
recon.judgmentBatchSize++;
} else {
recon = new Recon(0);
recon.service = "import";
recon.match = new ReconCandidate(id, "", value.toString(), new String[0], 100);
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
recon.addCandidate(recon.match);
recon = new Recon(0);
recon.service = "import";
recon.match = new ReconCandidate(id, "", value.toString(), new String[0], 100);
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
recon.addCandidate(recon.match);
reconMap.put(id, recon);
reconMap.put(id, recon);
}
}

4
src/main/java/com/metaweb/gridworks/model/ColumnModel.java
View File

@ -27,7 +27,7 @@ public class ColumnModel implements Jsonizable {
transient protected Map<String, Column> _nameToColumn;
transient protected Map<Integer, Column> _cellIndexToColumn;
transient protected List<ColumnGroup> _rootColumnGroups;
transient protected List<String> _columnNames;
transient protected List<String> _columnNames;
transient boolean _hasDependentRows;
public ColumnModel() {
@ -96,7 +96,7 @@ public class ColumnModel implements Jsonizable {
}
public List<String> getColumnNames() {
return _columnNames;
return _columnNames;
}
public void write(JSONWriter writer, Properties options)

4
src/main/java/com/metaweb/gridworks/model/Project.java
View File

@ -59,7 +59,7 @@ public class Project {
}
public ProjectMetadata getMetadata() {
return ProjectManager.singleton.getProjectMetadata(id);
return ProjectManager.singleton.getProjectMetadata(id);
}
public void save() {
@ -232,7 +232,7 @@ public class Project {
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
Row row = Row.load(line, pool);
Row row = Row.load(line, pool);
project.rows.add(row);
maxCellCount = Math.max(maxCellCount, row.cells.size());
}

22
src/main/java/com/metaweb/gridworks/model/Recon.java
View File

@ -60,16 +60,16 @@ public class Recon implements HasFields, Jsonizable {
final public long id;
public Object[] features = new Object[Feature_max];
public String service = "unknown";
public String service = "unknown";
public List<ReconCandidate> candidates;
public Judgment judgment = Judgment.None;
public String judgmentAction = "unknown";
public long judgmentHistoryEntry;
public int judgmentBatchSize = 0;
public String judgmentAction = "unknown";
public long judgmentHistoryEntry;
public int judgmentBatchSize = 0;
public ReconCandidate match = null;
public int matchRank = -1;
public int matchRank = -1;
public Recon(long judgmentHistoryEntry) {
id = System.currentTimeMillis() * 1000000 + Math.round(Math.random() * 1000000);
@ -189,7 +189,7 @@ public class Recon implements HasFields, Jsonizable {
public void write(JSONWriter writer, Properties options)
throws JSONException {
boolean saveMode = "save".equals(options.getProperty("mode"));
boolean saveMode = "save".equals(options.getProperty("mode"));
writer.object();
writer.key("id"); writer.value(id);
@ -225,7 +225,7 @@ public class Recon implements HasFields, Jsonizable {
writer.key("judgmentBatchSize"); writer.value(judgmentBatchSize);
if (match != null) {
writer.key("matchRank"); writer.value(matchRank);
writer.key("matchRank"); writer.value(matchRank);
}
}
@ -314,13 +314,13 @@ public class Recon implements HasFields, Jsonizable {
}
}
} else if ("service".equals(fieldName)) {
recon.service = jp.getText();
recon.service = jp.getText();
} else if ("judgmentAction".equals(fieldName)) {
recon.judgmentAction = jp.getText();
recon.judgmentAction = jp.getText();
} else if ("judgmentBatchSize".equals(fieldName)) {
recon.judgmentBatchSize = jp.getIntValue();
recon.judgmentBatchSize = jp.getIntValue();
} else if ("matchRank".equals(fieldName)) {
recon.matchRank = jp.getIntValue();
recon.matchRank = jp.getIntValue();
}
}
}

10
src/main/java/com/metaweb/gridworks/model/ReconCandidate.java
View File

@ -57,11 +57,11 @@ public class ReconCandidate implements HasFields, Jsonizable {
writer.key("score"); writer.value(score);
/* if (!options.containsKey("reconCandidateOmitTypes")) */ {
writer.key("types"); writer.array();
for (String typeID : typeIDs) {
writer.value(typeID);
}
writer.endArray();
writer.key("types"); writer.array();
for (String typeID : typeIDs) {
writer.value(typeID);
}
writer.endArray();
}
writer.endObject();

112
src/main/java/com/metaweb/gridworks/model/changes/ColumnSplitChange.java
View File

@ -19,10 +19,10 @@ import com.metaweb.gridworks.model.Row;
import com.metaweb.gridworks.util.Pool;
public class ColumnSplitChange implements Change {
final protected String _columnName;
final protected String _columnName;
final protected List<String> _columnNames;
final protected List<Integer> _rowIndices;
final protected List<String> _columnNames;
final protected List<Integer> _rowIndices;
final protected List<List<Serializable>> _tuples;
final protected boolean _removeOriginalColumn;
@ -30,20 +30,20 @@ public class ColumnSplitChange implements Change {
protected Column _column;
protected int _columnIndex;
protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows;
protected List<Row> _newRows;
protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows;
protected List<Row> _newRows;
public ColumnSplitChange(
String columnName,
List<String> columnNames,
List<Integer> rowIndices,
List<List<Serializable>> tuples,
boolean removeOriginalColumn
) {
_columnName = columnName;
String columnName,
List<String> columnNames,
List<Integer> rowIndices,
List<List<Serializable>> tuples,
boolean removeOriginalColumn
) {
_columnName = columnName;
_columnNames = columnNames;
_columnNames = columnNames;
_rowIndices = rowIndices;
_tuples = tuples;
@ -60,10 +60,10 @@ public class ColumnSplitChange implements Change {
Column column,
int columnIndex,
int firstNewCellIndex,
List<Row> oldRows,
List<Row> newRows
) {
int firstNewCellIndex,
List<Row> oldRows,
List<Row> newRows
) {
_columnName = columnName;
_columnNames = columnNames;
@ -83,13 +83,13 @@ public class ColumnSplitChange implements Change {
public void apply(Project project) {
synchronized (project) {
if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_column = project.columnModel.getColumnByName(_columnName);
_columnIndex = project.columnModel.getColumnIndexByName(_columnName);
_column = project.columnModel.getColumnByName(_columnName);
_columnIndex = project.columnModel.getColumnIndexByName(_columnName);
_oldRows = new ArrayList<Row>(_rowIndices.size());
_newRows = new ArrayList<Row>(_rowIndices.size());
@ -100,22 +100,22 @@ public class ColumnSplitChange implements Change {
int r = _rowIndices.get(i);
List<Serializable> tuple = _tuples.get(i);
Row oldRow = project.rows.get(r);
Row newRow = oldRow.dup();
Row oldRow = project.rows.get(r);
Row newRow = oldRow.dup();
_oldRows.add(oldRow);
_newRows.add(newRow);
_oldRows.add(oldRow);
_newRows.add(newRow);
for (int c = 0; c < tuple.size(); c++) {
Serializable value = tuple.get(c);
if (value != null) {
newRow.setCell(_firstNewCellIndex + c, new Cell(value, null));
}
}
for (int c = 0; c < tuple.size(); c++) {
Serializable value = tuple.get(c);
if (value != null) {
newRow.setCell(_firstNewCellIndex + c, new Cell(value, null));
}
}
if (_removeOriginalColumn) {
newRow.setCell(cellIndex, null);
}
if (_removeOriginalColumn) {
newRow.setCell(cellIndex, null);
}
}
}
@ -127,12 +127,12 @@ public class ColumnSplitChange implements Change {
}
for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name);
Column column = new Column(cellIndex, name);
project.columnModel.columns.add(_columnIndex + 1 + i, column);
project.columnModel.columns.add(_columnIndex + 1 + i, column);
}
if (_removeOriginalColumn) {
@ -158,7 +158,7 @@ public class ColumnSplitChange implements Change {
}
for (int i = 0; i < _columnNames.size(); i++) {
project.columnModel.columns.remove(_columnIndex + 1);
project.columnModel.columns.remove(_columnIndex + 1);
}
project.columnModel.update();
@ -167,29 +167,29 @@ public class ColumnSplitChange implements Change {
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("columnName="); writer.write(_columnName); writer.write('\n');
writer.write("columnName="); writer.write(_columnName); writer.write('\n');
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String name : _columnNames) {
writer.write(name); writer.write('\n');
writer.write(name); writer.write('\n');
}
writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n');
writer.write(rowIndex.toString()); writer.write('\n');
}
writer.write("tupleCount="); writer.write(Integer.toString(_tuples.size())); writer.write('\n');
for (List<Serializable> tuple : _tuples) {
writer.write(Integer.toString(tuple.size())); writer.write('\n');
for (Serializable value : tuple) {
if (value == null) {
writer.write("null");
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
if (value == null) {
writer.write("null");
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
}
writer.write("removeOriginalColumn="); writer.write(Boolean.toString(_removeOriginalColumn)); writer.write('\n');
@ -197,7 +197,7 @@ public class ColumnSplitChange implements Change {
writer.write("column="); _column.save(writer); writer.write('\n');
writer.write("columnIndex="); writer.write(Integer.toString(_columnIndex)); writer.write('\n');
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
for (Row row : _newRows) {
@ -233,7 +233,7 @@ public class ColumnSplitChange implements Change {
String value = line.substring(equal + 1);
if ("columnName".equals(field)) {
columnName = value;
columnName = value;
} else if ("columnNameCount".equals(field)) {
int count = Integer.parseInt(value);
@ -267,7 +267,7 @@ public class ColumnSplitChange implements Change {
List<Serializable> tuple = new ArrayList<Serializable>(valueCount);
for (int r = 0; r < valueCount; r++) {
line = reader.readLine();
line = reader.readLine();
JSONTokener t = new JSONTokener(line);
Object o = t.nextValue();

252
src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java
View File

@ -30,34 +30,34 @@ import com.metaweb.gridworks.util.Pool;
import com.metaweb.gridworks.util.FreebaseDataExtensionJob.DataExtension;
public class DataExtensionChange implements Change {
final protected String _baseColumnName;
final protected int _columnInsertIndex;
final protected String _baseColumnName;
final protected int _columnInsertIndex;
final protected List<String> _columnNames;
final protected List<FreebaseType> _columnTypes;
final protected List<String> _columnNames;
final protected List<FreebaseType> _columnTypes;
final protected List<Integer> _rowIndices;
final protected List<Integer> _rowIndices;
final protected List<DataExtension> _dataExtensions;
protected long _historyEntryID;
protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows;
protected List<Row> _newRows;
protected int _firstNewCellIndex = -1;
protected List<Row> _oldRows;
protected List<Row> _newRows;
public DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
long historyEntryID
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
long historyEntryID
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnTypes = columnTypes;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
_dataExtensions = dataExtensions;
@ -66,22 +66,22 @@ public class DataExtensionChange implements Change {
}
protected DataExtensionChange(
String baseColumnName,
int columnInsertIndex,
String baseColumnName,
int columnInsertIndex,
List<String> columnNames,
List<String> columnNames,
List<FreebaseType> columnTypes,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int firstNewCellIndex,
List<Row> oldRows,
List<Row> newRows
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int firstNewCellIndex,
List<Row> oldRows,
List<Row> newRows
) {
_baseColumnName = baseColumnName;
_columnInsertIndex = columnInsertIndex;
_columnNames = columnNames;
_columnNames = columnNames;
_columnTypes = columnTypes;
_rowIndices = rowIndices;
@ -95,10 +95,10 @@ public class DataExtensionChange implements Change {
public void apply(Project project) {
synchronized (project) {
if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_oldRows = new ArrayList<Row>(project.rows);
@ -116,18 +116,18 @@ public class DataExtensionChange implements Change {
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (int r = 0; r < _oldRows.size(); r++) {
Row oldRow = _oldRows.get(r);
if (r < rowIndex) {
_newRows.add(oldRow.dup());
continue;
}
Row oldRow = _oldRows.get(r);
if (r < rowIndex) {
_newRows.add(oldRow.dup());
continue;
}
if (dataExtension == null || dataExtension.data.length == 0) {
_newRows.add(oldRow);
} else {
Row firstNewRow = oldRow.dup();
extendRow(firstNewRow, dataExtension, 0, reconMap);
_newRows.add(firstNewRow);
if (dataExtension == null || dataExtension.data.length == 0) {
_newRows.add(oldRow);
} else {
Row firstNewRow = oldRow.dup();
extendRow(firstNewRow, dataExtension, 0, reconMap);
_newRows.add(firstNewRow);
int r2 = r + 1;
for (int subR = 1; subR < dataExtension.data.length; subR++) {
@ -153,7 +153,7 @@ public class DataExtensionChange implements Change {
}
r = r2 - 1; // r will be incremented by the for loop anyway
}
}
rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
@ -165,14 +165,14 @@ public class DataExtensionChange implements Change {
project.rows.addAll(_newRows);
for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name);
column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
column.setReconStats(ReconStats.create(project, cellIndex));
Column column = new Column(cellIndex, name);
column.setReconConfig(new DataExtensionReconConfig(_columnTypes.get(i)));
column.setReconStats(ReconStats.create(project, cellIndex));
project.columnModel.columns.add(_columnInsertIndex + i, column);
project.columnModel.columns.add(_columnInsertIndex + i, column);
}
project.columnModel.update();
@ -181,40 +181,40 @@ public class DataExtensionChange implements Change {
}
protected void extendRow(
Row row,
DataExtension dataExtension,
int extensionRowIndex,
Map<String, Recon> reconMap
) {
Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) {
Object value = values[c];
Cell cell = null;
Row row,
DataExtension dataExtension,
int extensionRowIndex,
Map<String, Recon> reconMap
) {
Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) {
Object value = values[c];
Cell cell = null;
if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value;
Recon recon;
if (reconMap.containsKey(rc.topicGUID)) {
recon = reconMap.get(rc.topicGUID);
} else {
recon = new Recon(_historyEntryID);
recon.addCandidate(rc);
recon.service = "mql";
recon.match = rc;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value;
Recon recon;
if (reconMap.containsKey(rc.topicGUID)) {
recon = reconMap.get(rc.topicGUID);
} else {
recon = new Recon(_historyEntryID);
recon.addCandidate(rc);
recon.service = "mql";
recon.match = rc;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
reconMap.put(rc.topicGUID, recon);
}
cell = new Cell(rc.topicName, recon);
} else {
cell = new Cell((Serializable) value, null);
}
reconMap.put(rc.topicGUID, recon);
}
cell = new Cell(rc.topicName, recon);
} else {
cell = new Cell((Serializable) value, null);
}
row.setCell(_firstNewCellIndex + c, cell);
}
row.setCell(_firstNewCellIndex + c, cell);
}
}
public void revert(Project project) {
@ -223,7 +223,7 @@ public class DataExtensionChange implements Change {
project.rows.addAll(_oldRows);
for (int i = 0; i < _columnNames.size(); i++) {
project.columnModel.columns.remove(_columnInsertIndex);
project.columnModel.columns.remove(_columnInsertIndex);
}
project.columnModel.update();
@ -232,11 +232,11 @@ public class DataExtensionChange implements Change {
}
public void save(Writer writer, Properties options) throws IOException {
writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
writer.write("baseColumnName="); writer.write(_baseColumnName); writer.write('\n');
writer.write("columnInsertIndex="); writer.write(Integer.toString(_columnInsertIndex)); writer.write('\n');
writer.write("columnNameCount="); writer.write(Integer.toString(_columnNames.size())); writer.write('\n');
for (String name : _columnNames) {
writer.write(name); writer.write('\n');
writer.write(name); writer.write('\n');
}
writer.write("columnTypeCount="); writer.write(Integer.toString(_columnTypes.size())); writer.write('\n');
for (FreebaseType type : _columnTypes) {
@ -251,34 +251,34 @@ public class DataExtensionChange implements Change {
}
writer.write("rowIndexCount="); writer.write(Integer.toString(_rowIndices.size())); writer.write('\n');
for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString()); writer.write('\n');
writer.write(rowIndex.toString()); writer.write('\n');
}
writer.write("dataExtensionCount="); writer.write(Integer.toString(_dataExtensions.size())); writer.write('\n');
for (DataExtension dataExtension : _dataExtensions) {
writer.write(Integer.toString(dataExtension.data.length)); writer.write('\n');
for (Object[] values : dataExtension.data) {
for (Object value : values) {
if (value == null) {
writer.write("null");
} else if (value instanceof ReconCandidate) {
for (Object value : values) {
if (value == null) {
writer.write("null");
} else if (value instanceof ReconCandidate) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
((ReconCandidate) value).write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
}
}
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("firstNewCellIndex="); writer.write(Integer.toString(_firstNewCellIndex)); writer.write('\n');
writer.write("newRowCount="); writer.write(Integer.toString(_newRows.size())); writer.write('\n');
for (Row row : _newRows) {
@ -294,14 +294,14 @@ public class DataExtensionChange implements Change {
}
static public Change load(LineNumberReader reader, Pool pool) throws Exception {
String baseColumnName = null;
int columnInsertIndex = -1;
String baseColumnName = null;
int columnInsertIndex = -1;
List<String> columnNames = null;
List<FreebaseType> columnTypes = null;
List<String> columnNames = null;
List<FreebaseType> columnTypes = null;
List<Integer> rowIndices = null;
List<DataExtension> dataExtensions = null;
List<Integer> rowIndices = null;
List<DataExtension> dataExtensions = null;
List<Row> oldRows = null;
List<Row> newRows = null;
@ -315,11 +315,11 @@ public class DataExtensionChange implements Change {
String value = line.substring(equal + 1);
if ("baseColumnName".equals(field)) {
baseColumnName = value;
baseColumnName = value;
} else if ("columnInsertIndex".equals(field)) {
columnInsertIndex = Integer.parseInt(value);
columnInsertIndex = Integer.parseInt(value);
} else if ("firstNewCellIndex".equals(field)) {
firstNewCellIndex = Integer.parseInt(value);
firstNewCellIndex = Integer.parseInt(value);
} else if ("rowIndexCount".equals(field)) {
int count = Integer.parseInt(value);
@ -361,14 +361,14 @@ public class DataExtensionChange implements Change {
Object[][] data = new Object[rowCount][];
for (int r = 0; r < rowCount; r++) {
Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine();
Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine();
row[c] = ReconCandidate.loadStreaming(line);
}
row[c] = ReconCandidate.loadStreaming(line);
}
data[r] = row;
data[r] = row;
}
dataExtensions.add(new DataExtension(data));
@ -398,15 +398,15 @@ public class DataExtensionChange implements Change {
}
DataExtensionChange change = new DataExtensionChange(
baseColumnName,
columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
firstNewCellIndex,
oldRows,
newRows
baseColumnName,
columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
firstNewCellIndex,
oldRows,
newRows
);

16
src/main/java/com/metaweb/gridworks/model/recon/HeuristicReconConfig.java
View File

@ -300,7 +300,7 @@ public class HeuristicReconConfig extends ReconConfig {
}
if (count > 0) {
ReconCandidate candidate = recon.candidates.get(0);
ReconCandidate candidate = recon.candidates.get(0);
recon.setFeature(Recon.Feature_nameMatch, text.equalsIgnoreCase(candidate.topicName));
recon.setFeature(Recon.Feature_nameLevenshtein, StringUtils.getLevenshteinDistance(text, candidate.topicName));
@ -311,14 +311,14 @@ public class HeuristicReconConfig extends ReconConfig {
if (this.typeID.equals(typeID)) {
recon.setFeature(Recon.Feature_typeMatch, true);
if (autoMatch && candidate.score >= 100) {
if (count == 1 ||
candidate.score / recon.candidates.get(1).score >= 1.5) {
if (count == 1 ||
candidate.score / recon.candidates.get(1).score >= 1.5) {
recon.match = candidate;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
}
recon.match = candidate;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
}
}
break;
}

4
src/main/java/com/metaweb/gridworks/operations/EngineDependentMassCellOperation.java
View File

@ -38,9 +38,9 @@ abstract public class EngineDependentMassCellOperation extends EngineDependentOp
FilteredRows filteredRows = engine.getAllFilteredRows(false);
try {
filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID));
filteredRows.accept(project, createRowVisitor(project, cellChanges, historyEntryID));
} catch (Exception e) {
e.printStackTrace();
e.printStackTrace();
}
String description = createDescription(column, cellChanges);

2
src/main/java/com/metaweb/gridworks/operations/EngineDependentOperation.java
View File

@ -16,7 +16,7 @@ abstract public class EngineDependentOperation extends AbstractOperation {
protected EngineDependentOperation(JSONObject engineConfig) {
_engineConfig = engineConfig;
_engineConfigString = engineConfig == null || engineConfig.length() == 0
? null : engineConfig.toString();
? null : engineConfig.toString();
}
protected Engine createEngine(Project project) throws Exception {

98
src/main/java/com/metaweb/gridworks/operations/ExtendDataOperation.java
View File

@ -51,7 +51,7 @@ public class ExtendDataOperation extends EngineDependentOperation {
public ExtendDataOperation(
JSONObject engineConfig,
String baseColumnName,
JSONObject extension,
JSONObject extension,
int columnInsertIndex
) {
super(engineConfig);
@ -142,17 +142,17 @@ public class ExtendDataOperation extends EngineDependentOperation {
FilteredRows filteredRows = engine.getAllFilteredRows(false);
filteredRows.accept(_project, new RowVisitor() {
List<Integer> _rowIndices;
List<Integer> _rowIndices;
public RowVisitor init(List<Integer> rowIndices) {
_rowIndices = rowIndices;
return this;
}
public RowVisitor init(List<Integer> rowIndices) {
_rowIndices = rowIndices;
return this;
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
if (!includeContextual) {
Cell cell = row.getCell(_cellIndex);
if (cell != null && cell.recon != null && cell.recon.match != null) {
_rowIndices.add(rowIndex);
_rowIndices.add(rowIndex);
}
}
return false;
@ -161,49 +161,49 @@ public class ExtendDataOperation extends EngineDependentOperation {
}
protected int extendRows(
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int from,
int limit,
List<Integer> rowIndices,
List<DataExtension> dataExtensions,
int from,
int limit,
Map<String, ReconCandidate> reconCandidateMap
) {
) {
Set<String> guids = new HashSet<String>();
int end;
for (end = from; end < limit && guids.size() < 10; end++) {
int index = rowIndices.get(end);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
for (end = from; end < limit && guids.size() < 10; end++) {
int index = rowIndices.get(end);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
guids.add(cell.recon.match.topicGUID);
}
}
Map<String, DataExtension> map = null;
Map<String, DataExtension> map = null;
try {
map = _job.extend(guids, reconCandidateMap);
} catch (Exception e) {
map = new HashMap<String, DataExtension>();
}
map = _job.extend(guids, reconCandidateMap);
} catch (Exception e) {
map = new HashMap<String, DataExtension>();
}
for (int i = from; i < end; i++) {
int index = rowIndices.get(i);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
String guid = cell.recon.match.topicGUID;
for (int i = from; i < end; i++) {
int index = rowIndices.get(i);
Row row = _project.rows.get(index);
Cell cell = row.getCell(_cellIndex);
String guid = cell.recon.match.topicGUID;
if (map.containsKey(guid)) {
dataExtensions.add(map.get(guid));
} else {
dataExtensions.add(null);
}
}
if (map.containsKey(guid)) {
dataExtensions.add(map.get(guid));
} else {
dataExtensions.add(null);
}
}
return end;
return end;
}
public void run() {
List<Integer> rowIndices = new ArrayList<Integer>();
List<DataExtension> dataExtensions = new ArrayList<DataExtension>();
List<Integer> rowIndices = new ArrayList<Integer>();
List<DataExtension> dataExtensions = new ArrayList<DataExtension>();
try {
populateRowsWithMatches(rowIndices);
@ -216,8 +216,8 @@ public class ExtendDataOperation extends EngineDependentOperation {
Map<String, ReconCandidate> reconCandidateMap = new HashMap<String, ReconCandidate>();
while (start < rowIndices.size()) {
int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
start = end;
int end = extendRows(rowIndices, dataExtensions, start, rowIndices.size(), reconCandidateMap);
start = end;
_progress = end * 100 / rowIndices.size();
try {
@ -230,10 +230,10 @@ public class ExtendDataOperation extends EngineDependentOperation {
}
if (!_canceled) {
List<String> columnNames = new ArrayList<String>();
for (ColumnInfo info : _job.columns) {
columnNames.add(StringUtils.join(info.names, " - "));
}
List<String> columnNames = new ArrayList<String>();
for (ColumnInfo info : _job.columns) {
columnNames.add(StringUtils.join(info.names, " - "));
}
List<FreebaseType> columnTypes = new ArrayList<FreebaseType>();
for (ColumnInfo info : _job.columns) {
@ -246,13 +246,13 @@ public class ExtendDataOperation extends EngineDependentOperation {
_description,
ExtendDataOperation.this,
new DataExtensionChange(
_baseColumnName,
_columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
_historyEntryID)
_baseColumnName,
_columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
_historyEntryID)
);
_project.history.addEntry(historyEntry);

6
src/main/java/com/metaweb/gridworks/operations/OnError.java
View File

@ -4,7 +4,7 @@
package com.metaweb.gridworks.operations;
public enum OnError {
KeepOriginal,
SetToBlank,
StoreError
KeepOriginal,
SetToBlank,
StoreError
}

18
src/main/java/com/metaweb/gridworks/operations/ReconDiscardJudgmentsOperation.java
View File

@ -77,20 +77,20 @@ public class ReconDiscardJudgmentsOperation extends EngineDependentMassCellOpera
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
Cell cell = row.getCell(cellIndex);
if (cell != null && cell.recon != null) {
Recon newRecon;
if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon.dup(historyEntryID);
Recon newRecon;
if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon.dup(historyEntryID);
newRecon.match = null;
newRecon.matchRank = -1;
newRecon.judgment = Judgment.None;
newRecon.judgmentAction = "mass";
newRecon.judgmentBatchSize = 1;
newRecon.judgmentBatchSize = 1;
dupReconMap.put(cell.recon.id, newRecon);
}
dupReconMap.put(cell.recon.id, newRecon);
}
Cell newCell = new Cell(cell.value, newRecon);

48
src/main/java/com/metaweb/gridworks/operations/ReconJudgeSimilarCellsOperation.java
View File

@ -151,8 +151,8 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
return new RowVisitor() {
int _cellIndex;
List<CellChange> _cellChanges;
Recon _sharedNewRecon = null;
Map<Long, Recon> _dupReconMap = new HashMap<Long, Recon>();
Recon _sharedNewRecon = null;
Map<Long, Recon> _dupReconMap = new HashMap<Long, Recon>();
long _historyEntryID;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) {
@ -170,22 +170,22 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
Recon recon = null;
if (_judgment == Judgment.New && _shareNewTopics) {
if (_sharedNewRecon == null) {
_sharedNewRecon = new Recon(_historyEntryID);
_sharedNewRecon.judgment = Judgment.New;
_sharedNewRecon.judgmentBatchSize = 0;
_sharedNewRecon.judgmentAction = "similar";
}
_sharedNewRecon.judgmentBatchSize++;
if (_sharedNewRecon == null) {
_sharedNewRecon = new Recon(_historyEntryID);
_sharedNewRecon.judgment = Judgment.New;
_sharedNewRecon.judgmentBatchSize = 0;
_sharedNewRecon.judgmentAction = "similar";
}
_sharedNewRecon.judgmentBatchSize++;
recon = _sharedNewRecon;
recon = _sharedNewRecon;
} else {
if (_dupReconMap.containsKey(cell.recon.id)) {
recon = _dupReconMap.get(cell.recon.id);
recon.judgmentBatchSize++;
} else {
recon = cell.recon.dup(_historyEntryID);
recon.judgmentBatchSize = 1;
if (_dupReconMap.containsKey(cell.recon.id)) {
recon = _dupReconMap.get(cell.recon.id);
recon.judgmentBatchSize++;
} else {
recon = cell.recon.dup(_historyEntryID);
recon.judgmentBatchSize = 1;
recon.matchRank = -1;
recon.judgmentAction = "similar";
@ -194,12 +194,12 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
recon.match = _match;
if (recon.candidates != null) {
for (int m = 0; m < recon.candidates.size(); m++) {
if (recon.candidates.get(m).topicGUID.equals(_match.topicGUID)) {
recon.matchRank = m;
break;
}
}
for (int m = 0; m < recon.candidates.size(); m++) {
if (recon.candidates.get(m).topicGUID.equals(_match.topicGUID)) {
recon.matchRank = m;
break;
}
}
}
} else if (_judgment == Judgment.New) {
recon.judgment = Recon.Judgment.New;
@ -209,8 +209,8 @@ public class ReconJudgeSimilarCellsOperation extends EngineDependentMassCellOper
recon.match = null;
}
_dupReconMap.put(cell.recon.id, recon);
}
_dupReconMap.put(cell.recon.id, recon);
}
}
Cell newCell = new Cell(cell.value, recon);

34
src/main/java/com/metaweb/gridworks/operations/ReconMarkNewTopicsOperation.java
View File

@ -72,10 +72,10 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio
Column column = project.columnModel.getColumnByName(_columnName);
return new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
int cellIndex;
List<CellChange> cellChanges;
Map<String, Recon> sharedRecons = new HashMap<String, Recon>();
Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>();
Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>();
long historyEntryID;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) {
@ -98,25 +98,25 @@ public class ReconMarkNewTopicsOperation extends EngineDependentMassCellOperatio
recon = new Recon(historyEntryID);
recon.judgment = Judgment.New;
recon.judgmentBatchSize = 1;
recon.judgmentAction = "mass";
recon.judgmentAction = "mass";
sharedRecons.put(s, recon);
}
} else {
long reconID = cell.recon == null ? 0 : cell.recon.id;
if (dupReconMap.containsKey(reconID)) {
recon = dupReconMap.get(reconID);
recon.judgmentBatchSize++;
} else {
recon = cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID);
recon.match = null;
recon.matchRank = -1;
recon.judgment = Judgment.New;
recon.judgmentBatchSize = 1;
recon.judgmentAction = "mass";
long reconID = cell.recon == null ? 0 : cell.recon.id;
if (dupReconMap.containsKey(reconID)) {
recon = dupReconMap.get(reconID);
recon.judgmentBatchSize++;
} else {
recon = cell.recon == null ? new Recon(historyEntryID) : cell.recon.dup(historyEntryID);
recon.match = null;
recon.matchRank = -1;
recon.judgment = Judgment.New;
recon.judgmentBatchSize = 1;
recon.judgmentAction = "mass";
dupReconMap.put(reconID, recon);
}
dupReconMap.put(reconID, recon);
}
}
Cell newCell = new Cell(cell.value, recon);

24
src/main/java/com/metaweb/gridworks/operations/ReconMatchBestCandidatesOperation.java
View File

@ -63,9 +63,9 @@ public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOp
Column column = project.columnModel.getColumnByName(_columnName);
return new RowVisitor() {
int cellIndex;
List<CellChange> cellChanges;
Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>();
int cellIndex;
List<CellChange> cellChanges;
Map<Long, Recon> dupReconMap = new HashMap<Long, Recon>();
long historyEntryID;
public RowVisitor init(int cellIndex, List<CellChange> cellChanges, long historyEntryID) {
@ -81,20 +81,20 @@ public class ReconMatchBestCandidatesOperation extends EngineDependentMassCellOp
if (cell != null && cell.recon != null) {
ReconCandidate candidate = cell.recon.getBestCandidate();
if (candidate != null) {
Recon newRecon;
if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon.dup(historyEntryID);
newRecon.judgmentBatchSize = 1;
Recon newRecon;
if (dupReconMap.containsKey(cell.recon.id)) {
newRecon = dupReconMap.get(cell.recon.id);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon.dup(historyEntryID);
newRecon.judgmentBatchSize = 1;
newRecon.match = candidate;
newRecon.matchRank = 0;
newRecon.judgment = Judgment.Matched;
newRecon.judgmentAction = "mass";
dupReconMap.put(cell.recon.id, newRecon);
}
dupReconMap.put(cell.recon.id, newRecon);
}
Cell newCell = new Cell(
cell.value,
newRecon

22
src/main/java/com/metaweb/gridworks/operations/ReconMatchSpecificTopicOperation.java
View File

@ -109,24 +109,24 @@ public class ReconMatchSpecificTopicOperation extends EngineDependentMassCellOpe
}
public boolean visit(Project project, int rowIndex, Row row, boolean includeContextual, boolean includeDependent) {
Cell cell = row.getCell(cellIndex);
Cell cell = row.getCell(cellIndex);
if (cell != null) {
long reconID = cell.recon != null ? cell.recon.id : 0;
long reconID = cell.recon != null ? cell.recon.id : 0;
Recon newRecon;
if (dupReconMap.containsKey(reconID)) {
newRecon = dupReconMap.get(reconID);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon != null ? cell.recon.dup(historyEntryID) : new Recon(historyEntryID);
Recon newRecon;
if (dupReconMap.containsKey(reconID)) {
newRecon = dupReconMap.get(reconID);
newRecon.judgmentBatchSize++;
} else {
newRecon = cell.recon != null ? cell.recon.dup(historyEntryID) : new Recon(historyEntryID);
newRecon.match = match;
newRecon.matchRank = -1;
newRecon.judgment = Judgment.Matched;
newRecon.judgmentAction = "mass";
newRecon.judgmentBatchSize = 1;
newRecon.judgmentBatchSize = 1;
dupReconMap.put(reconID, newRecon);
}
dupReconMap.put(reconID, newRecon);
}
Cell newCell = new Cell(
cell.value,

24
src/main/java/com/metaweb/gridworks/operations/ReconOperation.java
View File

@ -142,17 +142,17 @@ public class ReconOperation extends EngineDependentOperation {
writer.endObject();
if (_reconConfig instanceof HeuristicReconConfig) {
writer.object();
writer.key("action"); writer.value("createFacet");
writer.key("facetType"); writer.value("range");
writer.key("facetConfig");
writer.object();
writer.key("name"); writer.value(_columnName + ": best candidate's score");
writer.key("columnName"); writer.value(_columnName);
writer.key("expression"); writer.value("cell.recon.best.score");
writer.key("mode"); writer.value("range");
writer.endObject();
writer.endObject();
writer.object();
writer.key("action"); writer.value("createFacet");
writer.key("facetType"); writer.value("range");
writer.key("facetConfig");
writer.object();
writer.key("name"); writer.value(_columnName + ": best candidate's score");
writer.key("columnName"); writer.value(_columnName);
writer.key("expression"); writer.value("cell.recon.best.score");
writer.key("mode"); writer.value("range");
writer.endObject();
writer.endObject();
}
writer.endArray();
writer.endObject();
@ -234,7 +234,7 @@ public class ReconOperation extends EngineDependentOperation {
List<ReconEntry> entries = groups.get(j).entries;
if (recon != null) {
recon.judgmentBatchSize = entries.size();
recon.judgmentBatchSize = entries.size();
}
for (ReconEntry entry : entries) {

38
src/main/java/com/metaweb/gridworks/operations/TextTransformOperation.java
View File

@ -40,22 +40,22 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
}
static public OnError stringToOnError(String s) {
if ("set-to-blank".equalsIgnoreCase(s)) {
return OnError.SetToBlank;
} else if ("store-error".equalsIgnoreCase(s)) {
return OnError.StoreError;
} else {
return OnError.KeepOriginal;
}
if ("set-to-blank".equalsIgnoreCase(s)) {
return OnError.SetToBlank;
} else if ("store-error".equalsIgnoreCase(s)) {
return OnError.StoreError;
} else {
return OnError.KeepOriginal;
}
}
static public String onErrorToString(OnError onError) {
if (onError == OnError.SetToBlank) {
return "set-to-blank";
} else if (onError == OnError.StoreError) {
return "store-error";
} else {
return "keep-original";
}
if (onError == OnError.SetToBlank) {
return "set-to-blank";
} else if (onError == OnError.StoreError) {
return "store-error";
} else {
return "keep-original";
}
}
public TextTransformOperation(
@ -136,11 +136,11 @@ public class TextTransformOperation extends EngineDependentMassCellOperation {
} else {
Serializable newValue = ExpressionUtils.wrapStorable(o);
if (ExpressionUtils.isError(newValue)) {
if (_onError == OnError.KeepOriginal) {
return false;
} else if (_onError == OnError.SetToBlank) {
newValue = null;
}
if (_onError == OnError.KeepOriginal) {
return false;
} else if (_onError == OnError.SetToBlank) {
newValue = null;
}
}
if (!ExpressionUtils.sameValue(oldValue, newValue)) {

64
src/main/java/com/metaweb/gridworks/util/FreebaseDataExtensionJob.java
View File

@ -35,19 +35,19 @@ public class FreebaseDataExtensionJob {
}
static public class ColumnInfo {
final public List<String> names;
final public List<String> path;
final public FreebaseType expectedType;
final public List<String> names;
final public List<String> path;
final public FreebaseType expectedType;
protected ColumnInfo(List<String> names, List<String> path, FreebaseType expectedType) {
this.names = names;
this.path = path;
this.expectedType = expectedType;
}
protected ColumnInfo(List<String> names, List<String> path, FreebaseType expectedType) {
this.names = names;
this.path = path;
this.expectedType = expectedType;
}
}
final public JSONObject extension;
final public int columnCount;
final public JSONObject extension;
final public int columnCount;
final public List<ColumnInfo> columns = new ArrayList<ColumnInfo>();
public FreebaseDataExtensionJob(JSONObject obj) throws JSONException {
@ -57,9 +57,9 @@ public class FreebaseDataExtensionJob {
}
public Map<String, FreebaseDataExtensionJob.DataExtension> extend(
Set<String> guids,
Set<String> guids,
Map<String, ReconCandidate> reconCandidateMap
) throws Exception {
) throws Exception {
StringWriter writer = new StringWriter();
formulateQuery(guids, extension, writer);
@ -92,9 +92,9 @@ public class FreebaseDataExtensionJob {
}
protected FreebaseDataExtensionJob.DataExtension collectResult(
JSONObject obj,
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
) throws JSONException {
List<Object[]> rows = new ArrayList<Object[]>();
collectResult(rows, extension.getJSONArray("properties"), obj, 0, 0, reconCandidateMap);
@ -125,12 +125,12 @@ public class FreebaseDataExtensionJob {
JSONObject obj,
Map<String, ReconCandidate> reconCandidateMap
) throws JSONException {
String guid = obj.getString("guid");
ReconCandidate rc;
if (reconCandidateMap.containsKey(guid)) {
rc = reconCandidateMap.get(guid);
} else {
rc = new ReconCandidate(
String guid = obj.getString("guid");
ReconCandidate rc;
if (reconCandidateMap.containsKey(guid)) {
rc = reconCandidateMap.get(guid);
} else {
rc = new ReconCandidate(
obj.getString("id"),
obj.getString("guid"),
obj.getString("name"),
@ -138,8 +138,8 @@ public class FreebaseDataExtensionJob {
100
);
reconCandidateMap.put(guid, rc);
}
reconCandidateMap.put(guid, rc);
}
storeCell(rows, row, col, rc, reconCandidateMap);
}
@ -372,17 +372,17 @@ public class FreebaseDataExtensionJob {
}
static protected int countColumns(JSONObject obj, List<ColumnInfo> columns, List<String> names, List<String> path) throws JSONException {
String name = obj.getString("name");
String name = obj.getString("name");
List<String> names2 = null;
List<String> path2 = null;
if (columns != null) {
names2 = new ArrayList<String>(names);
names2.add(name);
List<String> names2 = null;
List<String> path2 = null;
if (columns != null) {
names2 = new ArrayList<String>(names);
names2.add(name);
path2 = new ArrayList<String>(path);
path2.add(obj.getString("id"));
}
path2 = new ArrayList<String>(path);
path2.add(obj.getString("id"));
}
if (obj.has("properties") && !obj.isNull("properties")) {
boolean included = (obj.has("included") && obj.getBoolean("included"));
@ -394,7 +394,7 @@ public class FreebaseDataExtensionJob {
}
return (included ? 1 : 0) +
countColumns(obj.getJSONArray("properties"), columns, names2, path2);
countColumns(obj.getJSONArray("properties"), columns, names2, path2);
} else {
if (columns != null) {
JSONObject expected = obj.getJSONObject("expected");