s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve recon error handling

Browse Source
This commit is contained in:
Tom Morris 2013-02-26 00:00:03 -05:00
parent a2b60d3d4b
commit 95e13eac50
2 changed files with 110 additions and 96 deletions
main/src/com/google/refine
commands/recon
GuessTypesOfColumnCommand.java
model/recon
StandardReconConfig.java

35
main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -37,8 +37,8 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -86,11 +86,11 @@ public class GuessTypesOfColumnCommand extends Command {
writer.key("code"); writer.value("error");
writer.key("message"); writer.value("No such column");
} else {
try {
List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl);
writer.key("code"); writer.value("ok");
writer.key("types"); writer.array();
List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl);
for (TypeGroup tg : typeGroups) {
writer.object();
writer.key("id"); writer.value(tg.id);
@ -101,9 +101,6 @@ public class GuessTypesOfColumnCommand extends Command {
}
writer.endArray();
} catch (Exception e) {
writer.key("code"); writer.value("error");
}
}
writer.endObject();
@ -112,7 +109,7 @@ public class GuessTypesOfColumnCommand extends Command {
}
}
final static int s_sampleSize = 10;
final static int SAMPLE_SIZE = 10;
/**
* Run relevance searches for the first n cells in the given column and
@ -122,13 +119,15 @@ public class GuessTypesOfColumnCommand extends Command {
* @param project
* @param column
* @return
* @throws JSONException, IOException
*/
protected List<TypeGroup> guessTypes(Project project, Column column, String serviceUrl) {
protected List<TypeGroup> guessTypes(Project project, Column column, String serviceUrl)
throws JSONException, IOException {
Map<String, TypeGroup> map = new HashMap<String, TypeGroup>();
int cellIndex = column.getCellIndex();
List<String> samples = new ArrayList<String>(s_sampleSize);
List<String> samples = new ArrayList<String>(SAMPLE_SIZE);
Set<String> sampleSet = new HashSet<String>();
for (Row row : project.rows) {
@ -138,7 +137,7 @@ public class GuessTypesOfColumnCommand extends Command {
if (!sampleSet.contains(s)) {
samples.add(s);
sampleSet.add(s);
if (samples.size() >= s_sampleSize) {
if (samples.size() >= SAMPLE_SIZE) {
break;
}
}
@ -160,13 +159,13 @@ public class GuessTypesOfColumnCommand extends Command {
}
jsonWriter.endObject();
} catch (JSONException e) {
// ignore
logger.error("Error constructing query", e);
}
String queriesString = stringWriter.toString();
try {
URL url = new URL(serviceUrl);
URLConnection connection = url.openConnection();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
{
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(30000);
@ -185,6 +184,12 @@ public class GuessTypesOfColumnCommand extends Command {
connection.connect();
}
if (connection.getResponseCode() >= 400) {
InputStream is = connection.getErrorStream();
throw new IOException("Failed - code:"
+ Integer.toString(connection.getResponseCode())
+ " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is));
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
@ -237,15 +242,17 @@ public class GuessTypesOfColumnCommand extends Command {
} finally {
is.close();
}
} catch (Exception e) {
}
} catch (IOException e) {
logger.error("Failed to guess cell types for load\n" + queriesString, e);
throw e;
}
List<TypeGroup> types = new ArrayList<TypeGroup>(map.values());
Collections.sort(types, new Comparator<TypeGroup>() {
@Override
public int compare(TypeGroup o1, TypeGroup o2) {
int c = Math.min(s_sampleSize, o2.count) - Math.min(s_sampleSize, o1.count);
int c = Math.min(SAMPLE_SIZE, o2.count) - Math.min(SAMPLE_SIZE, o1.count);
if (c != 0) {
return c;
}

15
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -34,10 +34,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model.recon;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@ -289,7 +290,7 @@ public class StandardReconConfig extends ReconConfig {
try {
URL url = new URL(service);
URLConnection connection = url.openConnection();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
{
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(30000);
@ -308,6 +309,13 @@ public class StandardReconConfig extends ReconConfig {
connection.connect();
}
if (connection.getResponseCode() >= 400) {
// TODO: Retry with backoff on 500 errors?
InputStream is = connection.getErrorStream();
throw new IOException("Failed - code:"
+ Integer.toString(connection.getResponseCode())
+ " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is));
} else {
InputStream is = connection.getInputStream();
try {
String s = ParsingUtilities.inputStreamToString(is);
@ -340,8 +348,7 @@ public class StandardReconConfig extends ReconConfig {
} finally {
is.close();
}
// } catch (IOException e) {
// // TODO: Retry on HTTP 500 errors?
}
} catch (Exception e) {
logger.error("Failed to batch recon with load:\n" + queriesString, e);
}