s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

When re-loading existing projects from disk, cache recon objects by their IDs to lower memory consumption.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@437 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
David Huynh 2010-04-09 19:32:50 +00:00
parent dff870519e
commit 8950e87e02
13 changed files with 75 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/com/metaweb/gridworks/model/Cell.java
View File

@ -4,6 +4,7 @@ import java.io.Serializable;
import java.io.Writer;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
@ -76,11 +77,11 @@ public class Cell implements HasFields, Jsonizable {
}
}
static public Cell load(String s) throws Exception {
return s.length() == 0 ? null : load(ParsingUtilities.evaluateJsonStringToObject(s));
static public Cell load(String s, Map<Long, Recon> reconCache) throws Exception {
return s.length() == 0 ? null : load(ParsingUtilities.evaluateJsonStringToObject(s), reconCache);
}
static public Cell load(JSONObject obj) throws Exception {
static public Cell load(JSONObject obj, Map<Long, Recon> reconCache) throws Exception {
Serializable value = null;
Recon recon = null;
@ -97,7 +98,7 @@ public class Cell implements HasFields, Jsonizable {
}
if (obj.has("r") && !obj.isNull("r")) {
recon = Recon.load(obj.getJSONObject("r"));
recon = Recon.load(obj.getJSONObject("r"), reconCache);
}
return new Cell(value, recon);

5
src/main/java/com/metaweb/gridworks/model/Project.java
View File

@ -12,7 +12,9 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@ -195,10 +197,11 @@ public class Project {
} else if ("rowCount".equals(field)) {
int count = Integer.parseInt(value);
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
Row row = Row.load(line);
Row row = Row.load(line, reconCache);
project.rows.add(row);
maxCellCount = Math.max(maxCellCount, row.cells.size());
}

11
src/main/java/com/metaweb/gridworks/model/Recon.java
View File

@ -193,12 +193,17 @@ public class Recon implements HasFields, Jsonizable {
writer.endObject();
}
static public Recon load(JSONObject obj) throws Exception {
static public Recon load(JSONObject obj, Map<Long, Recon> reconCache) throws Exception {
if (obj == null) {
return null;
}
Recon recon = new Recon(obj.getLong("id"));
long id = obj.getLong("id");
if (reconCache.containsKey(id)) {
return reconCache.get(id);
}
Recon recon = new Recon(id);
if (obj.has("j")) {
recon.judgment = stringToJudgment(obj.getString("j"));
@ -225,6 +230,8 @@ public class Recon implements HasFields, Jsonizable {
}
}
reconCache.put(id, recon);
return recon;
}

9
src/main/java/com/metaweb/gridworks/model/Row.java
View File

@ -3,6 +3,7 @@ package com.metaweb.gridworks.model;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Map.Entry;
@ -152,11 +153,11 @@ public class Row implements HasFields, Jsonizable {
}
}
static public Row load(String s) throws Exception {
return s.length() == 0 ? null : load(ParsingUtilities.evaluateJsonStringToObject(s));
static public Row load(String s, Map<Long, Recon> reconCache) throws Exception {
return s.length() == 0 ? null : load(ParsingUtilities.evaluateJsonStringToObject(s), reconCache);
}
static public Row load(JSONObject obj) throws Exception {
static public Row load(JSONObject obj, Map<Long, Recon> reconCache) throws Exception {
JSONArray a = obj.getJSONArray("cells");
int count = a.length();
@ -165,7 +166,7 @@ public class Row implements HasFields, Jsonizable {
if (!a.isNull(i)) {
JSONObject o = a.getJSONObject(i);
row.setCell(i, Cell.load(o));
row.setCell(i, Cell.load(o, reconCache));
}
}

6
src/main/java/com/metaweb/gridworks/model/changes/CellAtRow.java
View File

@ -2,9 +2,11 @@ package com.metaweb.gridworks.model.changes;
import java.io.IOException;
import java.io.Writer;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Recon;
public class CellAtRow {
@ -24,10 +26,10 @@ public class CellAtRow {
}
}
static public CellAtRow load(String s) throws Exception {
static public CellAtRow load(String s, Map<Long, Recon> reconCache) throws Exception {
int semicolon = s.indexOf(';');
int row = Integer.parseInt(s.substring(0, semicolon));
Cell cell = semicolon < s.length() - 1 ? Cell.load(s.substring(semicolon + 1)) : null;
Cell cell = semicolon < s.length() - 1 ? Cell.load(s.substring(semicolon + 1), reconCache) : null;
return new CellAtRow(row, cell);
}

8
src/main/java/com/metaweb/gridworks/model/changes/CellChange.java
View File

@ -3,11 +3,13 @@ package com.metaweb.gridworks.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
public class CellChange implements Change {
final public int row;
@ -53,7 +55,7 @@ public class CellChange implements Change {
writer.write("/ec/\n"); // end of change marker
}
static public CellChange load(LineNumberReader reader) throws Exception {
static public CellChange load(LineNumberReader reader, Map<Long, Recon> reconCache) throws Exception {
int row = -1;
int cellIndex = -1;
Cell oldCell = null;
@ -70,9 +72,9 @@ public class CellChange implements Change {
} else if ("cell".equals(field)) {
cellIndex = Integer.parseInt(value);
} else if ("new".equals(field) && value.length() > 0) {
newCell = Cell.load(value);
newCell = Cell.load(value, reconCache);
} else if ("old".equals(field) && value.length() > 0) {
oldCell = Cell.load(value);
oldCell = Cell.load(value, reconCache);
}
}

7
src/main/java/com/metaweb/gridworks/model/changes/ColumnAdditionChange.java
View File

@ -4,12 +4,15 @@ import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.Row;
public class ColumnAdditionChange extends ColumnChange {
@ -92,11 +95,13 @@ public class ColumnAdditionChange extends ColumnChange {
} else if ("newCellCount".equals(field)) {
int newCellCount = Integer.parseInt(line.substring(equal + 1));
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
newCells = new ArrayList<CellAtRow>(newCellCount);
for (int i = 0; i < newCellCount; i++) {
line = reader.readLine();
if (line != null) {
newCells.add(CellAtRow.load(line));
newCells.add(CellAtRow.load(line, reconCache));
}
}
}

7
src/main/java/com/metaweb/gridworks/model/changes/ColumnRemovalChange.java
View File

@ -3,12 +3,15 @@ package com.metaweb.gridworks.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.model.Cell;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.Row;
public class ColumnRemovalChange extends ColumnChange {
@ -85,11 +88,13 @@ public class ColumnRemovalChange extends ColumnChange {
} else if ("oldCellCount".equals(field)) {
int oldCellCount = Integer.parseInt(line.substring(equal + 1));
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
oldCells = new CellAtRow[oldCellCount];
for (int i = 0; i < oldCellCount; i++) {
line = reader.readLine();
if (line != null) {
oldCells[i] = CellAtRow.load(line);
oldCells[i] = CellAtRow.load(line, reconCache);
}
}
}

8
src/main/java/com/metaweb/gridworks/model/changes/DataExtensionChange.java
View File

@ -5,7 +5,9 @@ import java.io.LineNumberReader;
import java.io.Serializable;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.json.JSONException;
@ -284,6 +286,8 @@ public class DataExtensionChange implements Change {
int firstNewCellIndex = -1;
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
@ -365,7 +369,7 @@ public class DataExtensionChange implements Change {
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
oldRows.add(Row.load(line));
oldRows.add(Row.load(line, reconCache));
}
}
} else if ("newRowCount".equals(field)) {
@ -375,7 +379,7 @@ public class DataExtensionChange implements Change {
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
newRows.add(Row.load(line));
newRows.add(Row.load(line, reconCache));
}
}
}

7
src/main/java/com/metaweb/gridworks/model/changes/MassCellChange.java
View File

@ -3,12 +3,15 @@ package com.metaweb.gridworks.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.Row;
public class MassCellChange implements Change {
@ -112,9 +115,11 @@ public class MassCellChange implements Change {
} else if ("cellChangeCount".equals(field)) {
int cellChangeCount = Integer.parseInt(line.substring(equal + 1));
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
cellChanges = new CellChange[cellChangeCount];
for (int i = 0; i < cellChangeCount; i++) {
cellChanges[i] = CellChange.load(reader);
cellChanges[i] = CellChange.load(reader, reconCache);
}
}
}

9
src/main/java/com/metaweb/gridworks/model/changes/MassRowChange.java
View File

@ -4,11 +4,14 @@ import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.Row;
public class MassRowChange implements Change {
@ -56,6 +59,8 @@ public class MassRowChange implements Change {
List<Row> oldRows = null;
List<Row> newRows = null;
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
@ -68,7 +73,7 @@ public class MassRowChange implements Change {
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
oldRows.add(Row.load(line));
oldRows.add(Row.load(line, reconCache));
}
}
} else if ("newRowCount".equals(field)) {
@ -78,7 +83,7 @@ public class MassRowChange implements Change {
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
newRows.add(Row.load(line));
newRows.add(Row.load(line, reconCache));
}
}
}

7
src/main/java/com/metaweb/gridworks/model/changes/ReconChange.java
View File

@ -6,12 +6,15 @@ package com.metaweb.gridworks.model.changes;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.model.Column;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.ReconStats;
import com.metaweb.gridworks.model.recon.ReconConfig;
import com.metaweb.gridworks.util.ParsingUtilities;
@ -155,9 +158,11 @@ public class ReconChange extends MassCellChange {
} else if ("cellChangeCount".equals(field)) {
int cellChangeCount = Integer.parseInt(value);
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
cellChanges = new CellChange[cellChangeCount];
for (int i = 0; i < cellChangeCount; i++) {
cellChanges[i] = CellChange.load(reader);
cellChanges[i] = CellChange.load(reader, reconCache);
}
}
}

7
src/main/java/com/metaweb/gridworks/model/changes/RowRemovalChange.java
View File

@ -4,11 +4,14 @@ import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.metaweb.gridworks.history.Change;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.model.Recon;
import com.metaweb.gridworks.model.Row;
public class RowRemovalChange implements Change {
@ -90,11 +93,13 @@ public class RowRemovalChange implements Change {
} else if ("rowCount".equals(field)) {
int count = Integer.parseInt(line.substring(equal + 1));
Map<Long, Recon> reconCache = new HashMap<Long, Recon>();
rows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
rows.add(Row.load(line));
rows.add(Row.load(line, reconCache));
}
}
}