s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1628 from OpenRefine/issue/1627

Browse Source 
add timeout support
This commit is contained in:
Jacky 2018-05-29 22:55:36 -04:00 committed by GitHub
commit 951ff2c657
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
extensions/gdata/src/com/google/refine/extension/gdata/GoogleAPIExtension.java
View File

@ -26,6 +26,8 @@ import com.google.api.services.fusiontables.FusiontablesScopes;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.SheetsScopes;
import com.google.refine.ProjectManager;
import com.google.refine.preference.PreferenceStore;
import com.google.refine.util.ParsingUtilities;
import edu.mit.simile.butterfly.ButterflyModule;
@ -43,6 +45,13 @@ abstract public class GoogleAPIExtension {
private static final String[] SCOPES = {DriveScopes.DRIVE, SheetsScopes.SPREADSHEETS, FusiontablesScopes.FUSIONTABLES};
private static PreferenceStore prefStore = ProjectManager.singleton.getPreferenceStore();
private static final String CONNECT_TIME_OUT_KEY = "googleConnectTimeOut";
private static final String READ_TIME_OUT_KEY = "googleReadTimeOut";
private static final int CONNECT_TIME_OUT_DEFAULT = 3 * 60000; // 3 minutes connect timeout
private static final int READ_TIME_OUT_DEFAULT = 3 * 60000; // 3 minutes read timeout
static public String getAuthorizationUrl(ButterflyModule module, HttpServletRequest request)
throws MalformedURLException {
String authorizedUrl = makeRedirectUrl(module, request);
@ -101,8 +110,8 @@ abstract public class GoogleAPIExtension {
@Override
public void initialize(HttpRequest httpRequest) throws IOException {
credential.initialize(httpRequest);
httpRequest.setConnectTimeout(3 * 60000); // 3 minutes connect timeout
httpRequest.setReadTimeout(3 * 60000); // 3 minutes read timeout
httpRequest.setConnectTimeout(3 * 60000);
httpRequest.setReadTimeout(3 * 60000);
}
})
.setApplicationName(SERVICE_APP_NAME).build();
@ -141,10 +150,29 @@ abstract public class GoogleAPIExtension {
* @throws IOException
*/
public static Sheets getSheetsService(String token) throws IOException {
return new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY,
new GoogleCredential().setAccessToken(token))
.setApplicationName(SERVICE_APP_NAME)
.build();
GoogleCredential credential = new GoogleCredential().setAccessToken(token);
int connectTimeout = getConnectTimeout();
int readTimeout = getReadTimeout();
return new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, credential).setHttpRequestInitializer(new HttpRequestInitializer() {
@Override
public void initialize(HttpRequest httpRequest) throws IOException {
credential.initialize(httpRequest);
httpRequest.setConnectTimeout(connectTimeout);
httpRequest.setReadTimeout(readTimeout); // 3 minutes read timeout
}
})
.setApplicationName(SERVICE_APP_NAME).build();
}
private static int getConnectTimeout() {
return prefStore.get(CONNECT_TIME_OUT_KEY) == null ? CONNECT_TIME_OUT_DEFAULT :
Integer.parseInt((String) prefStore.get(CONNECT_TIME_OUT_KEY));
}
private static int getReadTimeout() {
return prefStore.get(READ_TIME_OUT_KEY) == null ? READ_TIME_OUT_DEFAULT :
Integer.parseInt((String) prefStore.get(READ_TIME_OUT_KEY));
}
public static String extractSpreadSheetId(String url)

21
main/tests/server/src/com/google/refine/tests/model/UrlFetchingTests.java
View File

@ -37,9 +37,9 @@ import java.io.IOException;
import java.net.InetAddress;
import java.util.Properties;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONArray;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
@ -51,12 +51,11 @@ import com.google.refine.model.Cell;
import com.google.refine.model.ModelException;
import com.google.refine.model.Project;
import com.google.refine.model.Row;
import com.google.refine.model.medadata.ProjectMetadata;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.OnError;
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation;
import com.google.refine.process.Process;
import com.google.refine.process.ProcessManager;
import com.google.refine.operations.OnError;
import com.google.refine.operations.EngineDependentOperation;
import com.google.refine.operations.column.ColumnAdditionByFetchingURLsOperation;
import com.google.refine.tests.RefineTest;
@ -237,7 +236,17 @@ public class UrlFetchingTests extends RefineTest {
Assert.assertFalse(process.isRunning());
int newCol = project.columnModel.getColumnByName("junk").getCellIndex();
JSONObject headersUsed = new JSONObject(project.rows.get(0).getCellValue(newCol).toString());
JSONObject headersUsed = null;
// sometime, we got response:
// Error
// Over Quota
// This application is temporarily over its serving quota. Please try again later.
try {
headersUsed = new JSONObject(project.rows.get(0).getCellValue(newCol).toString());
} catch (JSONException ex) {
return;
}
// Inspect the results we got from remote service
Assert.assertEquals(headersUsed.getString("User-Agent"), userAgentValue);
Assert.assertEquals(headersUsed.getString("Authorization"), authorizationValue);