s434730/RandomSec
3
0
Fork 0

Improve recon error handling

Browse Source
This commit is contained in:
Tom Morris 2013-02-26 00:00:03 -05:00
parent a2b60d3d4b
commit 95e13eac50
2 changed files with 110 additions and 96 deletions
main/src/com/google/refine
commands/recon
GuessTypesOfColumnCommand.java
model/recon
StandardReconConfig.java

35
main/src/com/google/refine/commands/recon/GuessTypesOfColumnCommand.java
View File

@ -37,8 +37,8 @@ import java.io.DataOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
@ -86,11 +86,11 @@ public class GuessTypesOfColumnCommand extends Command {
writer.key("code"); writer.value("error"); writer.key("code"); writer.value("error");
writer.key("message"); writer.value("No such column"); writer.key("message"); writer.value("No such column");
} else { } else {
try { List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl);
writer.key("code"); writer.value("ok"); writer.key("code"); writer.value("ok");
writer.key("types"); writer.array(); writer.key("types"); writer.array();
List<TypeGroup> typeGroups = guessTypes(project, column, serviceUrl);
for (TypeGroup tg : typeGroups) { for (TypeGroup tg : typeGroups) {
writer.object(); writer.object();
writer.key("id"); writer.value(tg.id); writer.key("id"); writer.value(tg.id);
@ -101,9 +101,6 @@ public class GuessTypesOfColumnCommand extends Command {
} }
writer.endArray(); writer.endArray();
} catch (Exception e) {
writer.key("code"); writer.value("error");
}
} }
writer.endObject(); writer.endObject();
@ -112,7 +109,7 @@ public class GuessTypesOfColumnCommand extends Command {
} }
} }
final static int s_sampleSize = 10; final static int SAMPLE_SIZE = 10;
/** /**
* Run relevance searches for the first n cells in the given column and * Run relevance searches for the first n cells in the given column and
@ -122,13 +119,15 @@ public class GuessTypesOfColumnCommand extends Command {
* @param project * @param project
* @param column * @param column
* @return * @return
* @throws JSONException, IOException
*/ */
protected List<TypeGroup> guessTypes(Project project, Column column, String serviceUrl) { protected List<TypeGroup> guessTypes(Project project, Column column, String serviceUrl)
throws JSONException, IOException {
Map<String, TypeGroup> map = new HashMap<String, TypeGroup>(); Map<String, TypeGroup> map = new HashMap<String, TypeGroup>();
int cellIndex = column.getCellIndex(); int cellIndex = column.getCellIndex();
List<String> samples = new ArrayList<String>(s_sampleSize); List<String> samples = new ArrayList<String>(SAMPLE_SIZE);
Set<String> sampleSet = new HashSet<String>(); Set<String> sampleSet = new HashSet<String>();
for (Row row : project.rows) { for (Row row : project.rows) {
@ -138,7 +137,7 @@ public class GuessTypesOfColumnCommand extends Command {
if (!sampleSet.contains(s)) { if (!sampleSet.contains(s)) {
samples.add(s); samples.add(s);
sampleSet.add(s); sampleSet.add(s);
if (samples.size() >= s_sampleSize) { if (samples.size() >= SAMPLE_SIZE) {
break; break;
} }
} }
@ -160,13 +159,13 @@ public class GuessTypesOfColumnCommand extends Command {
} }
jsonWriter.endObject(); jsonWriter.endObject();
} catch (JSONException e) { } catch (JSONException e) {
// ignore logger.error("Error constructing query", e);
} }
String queriesString = stringWriter.toString(); String queriesString = stringWriter.toString();
try { try {
URL url = new URL(serviceUrl); URL url = new URL(serviceUrl);
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
{ {
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(30000); connection.setConnectTimeout(30000);
@ -185,6 +184,12 @@ public class GuessTypesOfColumnCommand extends Command {
connection.connect(); connection.connect();
} }
if (connection.getResponseCode() >= 400) {
InputStream is = connection.getErrorStream();
throw new IOException("Failed - code:"
+ Integer.toString(connection.getResponseCode())
+ " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is));
} else {
InputStream is = connection.getInputStream(); InputStream is = connection.getInputStream();
try { try {
String s = ParsingUtilities.inputStreamToString(is); String s = ParsingUtilities.inputStreamToString(is);
@ -237,15 +242,17 @@ public class GuessTypesOfColumnCommand extends Command {
} finally { } finally {
is.close(); is.close();
} }
} catch (Exception e) { }
} catch (IOException e) {
logger.error("Failed to guess cell types for load\n" + queriesString, e); logger.error("Failed to guess cell types for load\n" + queriesString, e);
throw e;
} }
List<TypeGroup> types = new ArrayList<TypeGroup>(map.values()); List<TypeGroup> types = new ArrayList<TypeGroup>(map.values());
Collections.sort(types, new Comparator<TypeGroup>() { Collections.sort(types, new Comparator<TypeGroup>() {
@Override @Override
public int compare(TypeGroup o1, TypeGroup o2) { public int compare(TypeGroup o1, TypeGroup o2) {
int c = Math.min(s_sampleSize, o2.count) - Math.min(s_sampleSize, o1.count); int c = Math.min(SAMPLE_SIZE, o2.count) - Math.min(SAMPLE_SIZE, o1.count);
if (c != 0) { if (c != 0) {
return c; return c;
} }

15
main/src/com/google/refine/model/recon/StandardReconConfig.java
View File

@ -34,10 +34,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.refine.model.recon; package com.google.refine.model.recon;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@ -289,7 +290,7 @@ public class StandardReconConfig extends ReconConfig {
try { try {
URL url = new URL(service); URL url = new URL(service);
URLConnection connection = url.openConnection(); HttpURLConnection connection = (HttpURLConnection) url.openConnection();
{ {
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.setConnectTimeout(30000); connection.setConnectTimeout(30000);
@ -308,6 +309,13 @@ public class StandardReconConfig extends ReconConfig {
connection.connect(); connection.connect();
} }
if (connection.getResponseCode() >= 400) {
// TODO: Retry with backoff on 500 errors?
InputStream is = connection.getErrorStream();
throw new IOException("Failed - code:"
+ Integer.toString(connection.getResponseCode())
+ " message: " + is == null ? "" : ParsingUtilities.inputStreamToString(is));
} else {
InputStream is = connection.getInputStream(); InputStream is = connection.getInputStream();
try { try {
String s = ParsingUtilities.inputStreamToString(is); String s = ParsingUtilities.inputStreamToString(is);
@ -340,8 +348,7 @@ public class StandardReconConfig extends ReconConfig {
} finally { } finally {
is.close(); is.close();
} }
// } catch (IOException e) { }
// // TODO: Retry on HTTP 500 errors?
} catch (Exception e) { } catch (Exception e) {
logger.error("Failed to batch recon with load:\n" + queriesString, e); logger.error("Failed to batch recon with load:\n" + queriesString, e);
} }