s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 523 - On URL fetch error, return HTTP error code, message, and contents of error stream (HTML page) if available

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2429 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2012-01-26 18:47:30 +00:00
parent 0de40ad0a2
commit df45d06b2b
1 changed files with 40 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
main/src/com/google/refine/operations/column/ColumnAdditionByFetchingURLsOperation.java
View File

@ -33,8 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.operations.column;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
@ -265,29 +267,49 @@ public class ColumnAdditionByFetchingURLsOperation extends EngineDependentOperat
try {
URLConnection urlConnection = url.openConnection();
urlConnection.connect();
// urlConnection.setRequestProperty(_headerKey, _headerValue);
InputStream is = urlConnection.getInputStream();
try {
String encoding = urlConnection.getContentEncoding();
if (encoding == null) {
String contentType = urlConnection.getContentType();
if (contentType != null) {
final String charsetEqual = "charset=";
int c = contentType.lastIndexOf(charsetEqual);
if (c > 0) {
encoding = contentType.substring(c + charsetEqual.length());
InputStream is = urlConnection.getInputStream();
try {
String encoding = urlConnection.getContentEncoding();
if (encoding == null) {
String contentType = urlConnection.getContentType();
if (contentType != null) {
final String charsetEqual = "charset=";
int c = contentType.lastIndexOf(charsetEqual);
if (c > 0) {
encoding = contentType.substring(c + charsetEqual.length());
}
}
}
return new CellAtRow(
urlData.row,
new Cell(
ParsingUtilities.inputStreamToString(
is, encoding != null ? encoding : "UTF-8"),
null));
} finally {
is.close();
}
return new CellAtRow(
urlData.row,
new Cell(
ParsingUtilities.inputStreamToString(
is, encoding != null ? encoding : "UTF-8"),
null));
} finally {
is.close();
} catch (IOException e) {
String message;
if (urlConnection instanceof HttpURLConnection) {
int status = ((HttpURLConnection)urlConnection).getResponseCode();
String errorString = "";
InputStream errorStream = ((HttpURLConnection)urlConnection).getErrorStream();
if (errorStream != null) {
errorString = ParsingUtilities.inputStreamToString(errorStream);
}
message = String.format("HTTP error %d : %s | %s",status,
((HttpURLConnection)urlConnection).getResponseMessage(),
errorString);
} else {
message = e.toString();
}
return _onError == OnError.StoreError ?
new CellAtRow(urlData.row, new Cell(new EvalError(message), null)) : null;
}
} catch (Exception e) {
return _onError == OnError.StoreError ?