Add checkbox to disable caching

main/src/com/google/refine/commands/column/AddColumnByFetchingURLsCommand.java

@@ -54,6 +54,7 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
         int columnInsertIndex = Integer.parseInt(request.getParameter("columnInsertIndex"));
         int delay = Integer.parseInt(request.getParameter("delay"));
         String onError = request.getParameter("onError");
+        boolean cacheResponses = Boolean.parseBoolean(request.getParameter("cacheResponses"));
         
         return new ColumnAdditionByFetchingURLsOperation(
             engineConfig, 
@@ -62,7 +63,8 @@ public class AddColumnByFetchingURLsCommand extends EngineDependentCommand {
             TextTransformOperation.stringToOnError(onError),
             newColumnName,
             columnInsertIndex,
-            delay
+            delay,
+            cacheResponses
         );
     }
 

main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java

@@ -85,6 +85,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
     final protected String     _newColumnName;
     final protected int        _columnInsertIndex;
     final protected int        _delay;
+    final protected boolean    _cacheResponses;
 
     static public AbstractOperation reconstruct(Project project, JSONObject obj) throws Exception {
         JSONObject engineConfig = obj.getJSONObject("engineConfig");
@@ -96,7 +97,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             TextTransformOperation.stringToOnError(obj.getString("onError")),
             obj.getString("newColumnName"),
             obj.getInt("columnInsertIndex"),
-            obj.getInt("delay")
+            obj.getInt("delay"),
+            obj.optBoolean("cacheResponses", false) // false for retro-compatibility
         );
     }
 
@@ -107,7 +109,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
         OnError        onError,
         String         newColumnName,
         int            columnInsertIndex,
-        int            delay
+        int            delay,
+        boolean        cacheResponses
     ) {
         super(engineConfig);
 
@@ -119,6 +122,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
         _columnInsertIndex = columnInsertIndex;
 
         _delay = delay;
+        _cacheResponses = cacheResponses;
     }
 
     @Override
@@ -135,6 +139,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
         writer.key("urlExpression"); writer.value(_urlExpression);
         writer.key("onError"); writer.value(TextTransformOperation.onErrorToString(_onError));
         writer.key("delay"); writer.value(_delay);
+        writer.key("cacheResponses"); writer.value(_cacheResponses);
         writer.endObject();
     }
 
@@ -165,7 +170,8 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             project,
             engine,
             eval,
-            getBriefDescription(null)
+            getBriefDescription(null),
+            _cacheResponses
         );
     }
 
@@ -181,35 +187,39 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             Project project,
             Engine engine,
             Evaluable eval,
-            String description
+            String description,
+            boolean cacheResponses
         ) throws JSONException {
             super(description);
             _project = project;
             _engine = engine;
             _eval = eval;
             _historyEntryID = HistoryEntry.allocateID();
-            _urlCache = CacheBuilder.newBuilder()
+            _urlCache = null;
-		.maximumSize(2048)
+            if (cacheResponses) {
-		.expireAfterWrite(10, TimeUnit.MINUTES)
+                _urlCache = CacheBuilder.newBuilder()
-		.build(
+                .maximumSize(2048)
-		     new CacheLoader<String, Serializable>() {
+                .expireAfterWrite(10, TimeUnit.MINUTES)
-			public Serializable load(String urlString) {
+                .build(
-			    Serializable result = fetch(urlString);
+                     new CacheLoader<String, Serializable>() {
-			    try {
+                        public Serializable load(String urlString) {
-				// Always sleep for the delay, no matter how long the
+                            Serializable result = fetch(urlString);
-				// request took. This is more responsible than substracting
+                            try {
-				// the time spend requesting the URL, because it naturally
+                                // Always sleep for the delay, no matter how long the
-				// slows us down if the server is busy and takes a long time
+                                // request took. This is more responsible than substracting
-				// to reply.
+                                // the time spend requesting the URL, because it naturally
-				if (_delay > 0) {
+                                // slows us down if the server is busy and takes a long time
-				    Thread.sleep(_delay);
+                                // to reply.
-				}
+                                if (_delay > 0) {
-			    } catch (InterruptedException e) {
+                                    Thread.sleep(_delay);
-				return null;
+                                }
-			    }
+                            } catch (InterruptedException e) {
-			    return result;
+                                return null;
-			}
+                            }
-		     });
+                            return result;
+                        }
+                     });
+            }
         }
 
         @Override
@@ -250,8 +260,20 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             List<CellAtRow> responseBodies = new ArrayList<CellAtRow>(urls.size());
             for (int i = 0; i < urls.size(); i++) {
                 CellAtRow urlData = urls.get(i);
-                CellAtRow cellAtRow = cachedFetch(urlData);
+                String urlString = urlData.cell.value.toString();
-                if (cellAtRow != null) {
+
+                Serializable response = null;
+                if (_urlCache != null) {
+                    response = cachedFetch(urlString);
+                } else {
+                    response = fetch(urlString);
+                }
+
+                if (response != null) {
+                    CellAtRow cellAtRow = new CellAtRow(
+                            urlData.row,
+                            new Cell(response, null));
+
                     responseBodies.add(cellAtRow);
                 }
 
@@ -259,7 +281,7 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
 
                 if (_canceled) {
                     break;
-	        }
+                }
             }
 
             if (!_canceled) {
@@ -279,30 +301,21 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
             }
         }
 
-	CellAtRow cachedFetch(CellAtRow urlData) {
+        Serializable cachedFetch(String urlString) {
-	    String urlString = urlData.cell.value.toString();
+            try {
-
+                return  _urlCache.get(urlString);
-	    Serializable cellResult = null;
+            } catch(ExecutionException e) {
-	    try {
+                return null;
-		cellResult = _urlCache.get(urlString);
+            }
-	    } catch(ExecutionException e) {
+        }
-	    }
-
-	    if (cellResult != null) {
-		return new CellAtRow(
-				urlData.row,
-				new Cell(cellResult, null));
-	    }
-	    return null;
-	}
 
         Serializable fetch(String urlString) {
-	    URL url = null;
+            URL url = null;
-	    try {
+            try {
-		url = new URL(urlString);
+                url = new URL(urlString);
-	    } catch (MalformedURLException e) {
+            } catch (MalformedURLException e) {
-		return null;
+                return null;
-	    }
+            }
 
             try {
                 URLConnection urlConnection = url.openConnection();

main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java

@@ -134,7 +134,8 @@ public class UrlFetchingTests extends RefineTest {
 		OnError.SetToBlank,
 		"rand",
 		1,
-		500);
+		500,
+		true);
 	ProcessManager pm = project.getProcessManager();
         Process process = op.createProcess(project, options);
 	process.startPerforming(pm);

main/webapp/modules/core/langs/translation-default.json

@@ -510,6 +510,7 @@
 		"on-error": "On error",
 		"set-blank": "set to blank",
 		"store-err": "store error",
+		"cache-responses": "Cache responses",
 		"copy-val": "copy value from original column",
 		"warning-col-name": "You must enter a column name.",
 		"add-col-fetch": "Add column by fetching URLs based on column",

main/webapp/modules/core/langs/translation-en.json

@@ -510,6 +510,7 @@
 		"on-error": "On error",
 		"set-blank": "set to blank",
 		"store-err": "store error",
+		"cache-responses": "Cache responses",
 		"copy-val": "copy value from original column",
 		"warning-col-name": "You must enter a column name.",
 		"add-col-fetch": "Add column by fetching URLs based on column",

main/webapp/modules/core/langs/translation-fr.json

@@ -510,6 +510,7 @@
 		"on-error": "En cas d’erreur",
 		"set-blank": "vider la cellule",
 		"store-err": "conserver l’erreur",
+		"cache-responses": "Mettre les réponses en cache",
 		"copy-val": "copier la valeur depuis la colonne originale",
 		"warning-col-name": "Vous devez indiquer un nom de colonne.",
 		"add-col-fetch": "Ajouter une colonne en moissonnant les données depuis les URL d’une colonne",

main/webapp/modules/core/scripts/views/data-table/add-column-by-fetching-urls-dialog.html

@@ -12,11 +12,14 @@
         </tr>
         <tr>
           <td width="1%" style="white-space: pre;" bind="or_views_onErr"></td>
-          <td colspan="3">
+          <td>
             <input type="radio" name="dialog-onerror-choice" value="set-to-blank" checked id="$add-column-error-set-to-blank"/>
               <label for="$add-column-error-set-to-blank" bind="or_views_setBlank"></label>
             <input type="radio" name="dialog-onerror-choice" value="store-error" id="$add-column-error-store-error" />
               <label for="$add-column-error-store-error" bind="or_views_storeErr"></label></td>
+	   <td colspan="2">
+	     <input type="checkbox" name="dialog-cache-responses" id="$add-column-cache-responses" checked="checked" />
+		<label for="$add-column-cache-responses" bind="or_views_cacheResponses"></label></td>
         </tr>
         <tr><td colspan="4"><h3><span bind="or_views_urlFetch"></span></h3></td></tr>
         <tr><td colspan="4">$EXPRESSION_PREVIEW_WIDGET$</td></tr>
@@ -27,4 +30,4 @@
       <button class="button" bind="cancelButton"></button>
     </div>
   </div>
-</div>
+</div>

main/webapp/modules/core/scripts/views/data-table/menu-edit-column.js

@@ -103,6 +103,7 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
     elmts.or_views_onErr.text($.i18n._('core-views')["on-error"]);
     elmts.or_views_setBlank.text($.i18n._('core-views')["set-blank"]);
     elmts.or_views_storeErr.text($.i18n._('core-views')["store-err"]);
+    elmts.or_views_cacheResponses.text($.i18n._('core-views')["cache-responses"]);
     elmts.or_views_urlFetch.text($.i18n._('core-views')["url-fetch"]);
     elmts.okButton.html($.i18n._('core-buttons')["ok"]);
     elmts.cancelButton.text($.i18n._('core-buttons')["cancel"]);
@@ -135,7 +136,8 @@ DataTableColumnHeaderUI.extendMenu(function(column, columnHeaderUI, menu) {
           newColumnName: columnName, 
           columnInsertIndex: columnIndex + 1,
           delay: elmts.throttleDelayInput[0].value,
-          onError: $('input[name="dialog-onerror-choice"]:checked')[0].value
+          onError: $('input[name="dialog-onerror-choice"]:checked')[0].value,
+          cacheResponses: $('input[name="dialog-cache-responses"]')[0].checked,
         },
         null,
         { modelsChanged: true }