s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 523 - On URL fetch error, return HTTP error code, message, and contents of error stream (HTML page) if available

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2429 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2012-01-26 18:47:30 +00:00
parent 0de40ad0a2
commit df45d06b2b
1 changed files with 40 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java
View File

@ -33,8 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.column; package com.google.refine.operations.column;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.Serializable; import java.io.Serializable;
import java.net.HttpURLConnection;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
@ -265,29 +267,49 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
try { try {
URLConnection urlConnection = url.openConnection(); URLConnection urlConnection = url.openConnection();
urlConnection.connect(); // urlConnection.setRequestProperty(_headerKey, _headerValue);
InputStream is = urlConnection.getInputStream();
try { try {
String encoding = urlConnection.getContentEncoding(); InputStream is = urlConnection.getInputStream();
if (encoding == null) { try {
String contentType = urlConnection.getContentType(); String encoding = urlConnection.getContentEncoding();
if (contentType != null) { if (encoding == null) {
final String charsetEqual = "charset="; String contentType = urlConnection.getContentType();
int c = contentType.lastIndexOf(charsetEqual); if (contentType != null) {
if (c > 0) { final String charsetEqual = "charset=";
encoding = contentType.substring(c + charsetEqual.length()); int c = contentType.lastIndexOf(charsetEqual);
if (c > 0) {
encoding = contentType.substring(c + charsetEqual.length());
}
} }
} }
return new CellAtRow(
urlData.row,
new Cell(
ParsingUtilities.inputStreamToString(
is, encoding != null ? encoding : "UTF-8"),
null));
} finally {
is.close();
} }
return new CellAtRow( } catch (IOException e) {
urlData.row, String message;
new Cell( if (urlConnection instanceof HttpURLConnection) {
ParsingUtilities.inputStreamToString( int status = ((HttpURLConnection)urlConnection).getResponseCode();
is, encoding != null ? encoding : "UTF-8"), String errorString = "";
null)); InputStream errorStream = ((HttpURLConnection)urlConnection).getErrorStream();
} finally { if (errorStream != null) {
is.close(); errorString = ParsingUtilities.inputStreamToString(errorStream);
}
message = String.format("HTTP error %d : %s | %s",status,
((HttpURLConnection)urlConnection).getResponseMessage(),
errorString);
} else {
message = e.toString();
}
return _onError == OnError.StoreError ?
new CellAtRow(urlData.row, new Cell(new EvalError(message), null)) : null;
} }
} catch (Exception e) { } catch (Exception e) {
return _onError == OnError.StoreError ? return _onError == OnError.StoreError ?