s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 404 - Fix indeterminate behavior in character encoding guesser. Thanks to Paul Makepeace.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@2120 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2011-06-14 04:29:44 +00:00
parent 5fa5e7b28a
commit 2af22f9485
1 changed files with 16 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
main/src/com/google/refine/commands/project/CreateProjectCommand.java
View File

@ -1,6 +1,6 @@
/*
Copyright 2010, Google Inc.
Copyright 2010,2011. Google Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -447,38 +447,35 @@ public class CreateProjectCommand extends Command {
) throws Exception {
if (importer instanceof ReaderImporter) {
BufferedInputStream inputStream = new BufferedInputStream(rawInputStream);
// NOTE(SM): The ICU4J char detection code requires the input stream to support mark/reset.
// Unfortunately, not all ServletInputStream implementations are marking, so we need do
// this memory-expensive wrapping to make it work. It's far from ideal but I don't have
// a more efficient solution.
byte[] bytes = new byte[1024 * 4];
inputStream.mark(bytes.length);
inputStream.read(bytes);
inputStream.reset();
// NOTE: The ICU4J char detection code requires the input stream to support mark/reset.
InputStream inputStream = rawInputStream;
if (!inputStream.markSupported()) {
inputStream = new BufferedInputStream(rawInputStream);
}
CharsetDetector detector = new CharsetDetector();
detector.setDeclaredEncoding("utf8"); // most of the content on the web is encoded in UTF-8 so start with that
options.setProperty("encoding_confidence", "0"); // in case we don't find anything suitable
InputStreamReader reader = null;
CharsetMatch[] charsetMatches = detector.setText(bytes).detectAll();
if (charsetMatches.length > 0) {
CharsetMatch charsetMatch = charsetMatches[0]; // matches are ordered - first is best match
CharsetMatch[] charsetMatches = detector.setText(inputStream).detectAll();
for (CharsetMatch charsetMatch : charsetMatches) { // matches are ordered - first is best match
String matchName = charsetMatch.getName();
int confidence = charsetMatch.getConfidence();
// Threshold was 50. Do we ever want to not use our best guess even if it's low confidence? - tfmorris
if (confidence >= 20) {
logger.info("Encoding guess: {} [confidence: {}]", matchName, confidence);
try {
reader = new InputStreamReader(inputStream, charsetMatch.getName());
reader = new InputStreamReader(inputStream, matchName);
} catch (UnsupportedEncodingException e) {
// ignored - we'll fall back to a different reader later
logger.debug("Unsupported InputStreamReader charset encoding: {} [confidence: {}]; skipping", matchName, confidence);
continue;
}
// Encoding will be set later at common exit point
options.setProperty("encoding_confidence", Integer.toString(confidence));
logger.info("Best encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence());
options.setProperty("encoding_confidence", Integer.toString(confidence));
break;
} else {
logger.debug("Poor encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence());
logger.debug("Poor encoding guess: {} [confidence: {}]; skipping", matchName, confidence);
}
}