diff --git a/main/pom.xml b/main/pom.xml index abfef9497..44fbfd614 100644 --- a/main/pom.xml +++ b/main/pom.xml @@ -392,6 +392,12 @@ mockwebserver 4.7.2 test + + + org.apache.httpcomponents + httpmime + 4.5.2 + test diff --git a/main/src/com/google/refine/importing/ImportingUtilities.java b/main/src/com/google/refine/importing/ImportingUtilities.java index 827fce6f4..6b165a88e 100644 --- a/main/src/com/google/refine/importing/ImportingUtilities.java +++ b/main/src/com/google/refine/importing/ImportingUtilities.java @@ -81,6 +81,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; +import org.apache.http.StatusLine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -309,11 +310,18 @@ public class ImportingUtilities { CloseableHttpResponse response = httpclient.execute(httpGet); try { - response.getStatusLine(); HttpEntity entity = response.getEntity(); if (entity == null) { throw new Exception("No content found in " + url.toString()); } + StatusLine status = response.getStatusLine(); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode >= 400) { + String errorString = ParsingUtilities.inputStreamToString(entity.getContent()); + String message = String.format("HTTP error %d : %s | %s", statusCode, + status.getReasonPhrase(), errorString); + throw new Exception(message); + } InputStream stream2 = entity.getContent(); String mimeType = null; diff --git a/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java b/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java index b0b725f81..20d2d6e66 100644 --- a/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java +++ b/main/tests/server/src/com/google/refine/importing/ImportingUtilitiesTests.java @@ -28,8 +28,20 @@ package com.google.refine.importing; import java.io.File; import java.io.IOException; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.util.LinkedList; +import java.util.Properties; +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.entity.mime.content.StringBody; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -45,6 +57,11 @@ import com.google.refine.util.JSONUtilities; import com.google.refine.util.ParsingUtilities; import com.google.refine.util.TestUtils; +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; + +import static org.mockito.Mockito.when; + public class ImportingUtilitiesTests extends ImporterTest { @Override @@ -82,4 +99,76 @@ public class ImportingUtilitiesTests extends ImporterTest { JSONUtilities.safePut(options, "recordPath", path); return options; } + + @Test + public void urlImporting() throws IOException { + + String RESPONSE_BODY = "{code:401,message:Unauthorised}"; + String MESSAGE = String.format("HTTP error %d : %s | %s", 401, + "Client Error", RESPONSE_BODY); + + MockWebServer server = new MockWebServer(); + MockResponse mockResponse = new MockResponse(); + mockResponse.setBody(RESPONSE_BODY); + mockResponse.setResponseCode(401); + server.start(); + server.enqueue(mockResponse); + HttpUrl url = server.url("/random"); + + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + StringBody stringBody = new StringBody(url.toString(), ContentType.MULTIPART_FORM_DATA); + builder = builder.addPart("download", stringBody); + HttpEntity entity = builder.build(); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + entity.writeTo(os); + ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray()); + + HttpServletRequest req = Mockito.mock(HttpServletRequest.class); + when(req.getContentType()).thenReturn(entity.getContentType().getValue()); + when(req.getParameter("download")).thenReturn(url.toString()); + when(req.getMethod()).thenReturn("POST"); + when(req.getContentLength()).thenReturn((int) entity.getContentLength()); + when(req.getInputStream()).thenReturn(new MockServletInputStream(is)); + + + ImportingJob job = ImportingManager.createJob(); + Properties parameters = ParsingUtilities.parseUrlParameters(req); + ObjectNode retrievalRecord = ParsingUtilities.mapper.createObjectNode(); + ObjectNode progress = ParsingUtilities.mapper.createObjectNode(); + try { + ImportingUtilities.retrieveContentFromPostRequest(req, parameters, job.getRawDataDir(), retrievalRecord, new ImportingUtilities.Progress() { + @Override + public void setProgress(String message, int percent) { + if (message != null) { + JSONUtilities.safePut(progress, "message", message); + } + JSONUtilities.safePut(progress, "percent", percent); + } + + @Override + public boolean isCanceled() { + return job.canceled; + } + }); + Assert.fail("No Exception was thrown"); + } catch (Exception exception) { + Assert.assertEquals(MESSAGE, exception.getMessage()); + } + } + + public static class MockServletInputStream extends ServletInputStream { + + private final InputStream delegate; + + public MockServletInputStream(InputStream delegate) { + this.delegate = delegate; + } + + @Override + public int read() throws IOException { + return delegate.read(); + } + + } }