s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 523 - On URL fetch error, return HTTP error code, message, and contents of error stream (HTML page) if available

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2429 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2012-01-26 18:47:30 +00:00
parent 0de40ad0a2
commit df45d06b2b
1 changed files with 40 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java
View File

@ -33,8 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.column; package com.google.refine.operations.column;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.Serializable; import java.io.Serializable;
import java.net.HttpURLConnection;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
@ -265,8 +267,9 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
try { try {
URLConnection urlConnection = url.openConnection(); URLConnection urlConnection = url.openConnection();
urlConnection.connect(); // urlConnection.setRequestProperty(_headerKey, _headerValue);
try {
InputStream is = urlConnection.getInputStream(); InputStream is = urlConnection.getInputStream();
try { try {
String encoding = urlConnection.getContentEncoding(); String encoding = urlConnection.getContentEncoding();
@ -286,9 +289,28 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
ParsingUtilities.inputStreamToString( ParsingUtilities.inputStreamToString(
is, encoding != null ? encoding : "UTF-8"), is, encoding != null ? encoding : "UTF-8"),
null)); null));
} finally { } finally {
is.close(); is.close();
} }
} catch (IOException e) {
String message;
if (urlConnection instanceof HttpURLConnection) {
int status = ((HttpURLConnection)urlConnection).getResponseCode();
String errorString = "";
InputStream errorStream = ((HttpURLConnection)urlConnection).getErrorStream();
if (errorStream != null) {
errorString = ParsingUtilities.inputStreamToString(errorStream);
}
message = String.format("HTTP error %d : %s | %s",status,
((HttpURLConnection)urlConnection).getResponseMessage(),
errorString);
} else {
message = e.toString();
}
return _onError == OnError.StoreError ?
new CellAtRow(urlData.row, new Cell(new EvalError(message), null)) : null;
}
} catch (Exception e) { } catch (Exception e) {
return _onError == OnError.StoreError ? return _onError == OnError.StoreError ?
new CellAtRow(urlData.row, new Cell(new EvalError(e.getMessage()), null)) : null; new CellAtRow(urlData.row, new Cell(new EvalError(e.getMessage()), null)) : null;