s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Cache String, not URL, in ColumnAdditionByFetchingURLsOperation

Browse Source
This commit is contained in:
Antonin Delpeuch 2017-03-08 07:45:11 +00:00
parent 3297e6e7ee
commit a9c4b0af16
1 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java
View File

@ -172,7 +172,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
final protected Evaluable _eval; final protected Evaluable _eval;
final protected long _historyEntryID; final protected long _historyEntryID;
protected int _cellIndex; protected int _cellIndex;
protected Map<URL, Serializable> _urlCache; protected Map<String, Serializable> _urlCache;
public ColumnAdditionByFetchingURLsProcess( public ColumnAdditionByFetchingURLsProcess(
Project project, Project project,
@ -185,7 +185,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
_engine = engine; _engine = engine;
_eval = eval; _eval = eval;
_historyEntryID = HistoryEntry.allocateID(); _historyEntryID = HistoryEntry.allocateID();
_urlCache = new HashMap<URL, Serializable>(); _urlCache = new HashMap<String, Serializable>();
} }
@Override @Override
@ -259,18 +259,12 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
CellAtRow cachedFetch(CellAtRow urlData) { CellAtRow cachedFetch(CellAtRow urlData) {
String urlString = urlData.cell.value.toString(); String urlString = urlData.cell.value.toString();
URL url = null;
try {
url = new URL(urlString);
} catch (MalformedURLException e) {
return null;
}
Serializable cellResult = _urlCache.get(url); Serializable cellResult = _urlCache.get(urlString);
if (cellResult == null) { if (cellResult == null) {
cellResult = fetch(url); cellResult = fetch(urlString);
if (cellResult != null) { if (cellResult != null) {
_urlCache.put(url, cellResult); _urlCache.put(urlString, cellResult);
} }
try { try {
@ -295,7 +289,14 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
return null; return null;
} }
Serializable fetch(URL url) { Serializable fetch(String urlString) {
URL url = null;
try {
url = new URL(urlString);
} catch (MalformedURLException e) {
return null;
}
try { try {
URLConnection urlConnection = url.openConnection(); URLConnection urlConnection = url.openConnection();
// urlConnection.setRequestProperty(_headerKey, _headerValue); // urlConnection.setRequestProperty(_headerKey, _headerValue);