s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue 237 - Make sure project's character encoding is always set. Lower minimum confidence threshold for guesser.

Browse Source 
git-svn-id: http://google-refine.googlecode.com/svn/trunk@1931 7d457c2a-affb-35e4-300a-418c747d4874
This commit is contained in:
Tom Morris 2010-11-26 22:23:31 +00:00
parent d5a742944e
commit 080ec5332e
1 changed files with 26 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
main/src/com/google/refine/commands/project/CreateProjectCommand.java
View File

@ -448,35 +448,44 @@ public class CreateProjectCommand extends Command {
CharsetDetector detector = new CharsetDetector();
detector.setDeclaredEncoding("utf8"); // most of the content on the web is encoded in UTF-8 so start with that
options.setProperty("encoding_confidence", "0"); // in case we don't find anything suitable
Reader reader = null;
InputStreamReader reader = null;
CharsetMatch[] charsetMatches = detector.setText(bytes).detectAll();
for (CharsetMatch charsetMatch : charsetMatches) {
try {
if (charsetMatches.length > 0) {
CharsetMatch charsetMatch = charsetMatches[0]; // matches are ordered - first is best match
int confidence = charsetMatch.getConfidence();
if (confidence >= 50) {
// Threshold was 50. Do we ever want to not use our best guess even if it's low confidence? - tfmorris
if (confidence >= 20) {
try {
reader = new InputStreamReader(inputStream, charsetMatch.getName());
options.setProperty("encoding", charsetMatch.getName());
options.setProperty("encoding_confidence", Integer.toString(confidence));
logger.info("Best encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence());
}
break;
} catch (UnsupportedEncodingException e) {
// silent
// ignored - we'll fall back to a different reader later
}
// Encoding will be set later at common exit point
options.setProperty("encoding_confidence", Integer.toString(confidence));
logger.info("Best encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence());
} else {
logger.debug("Poor encoding guess: {} [confidence: {}]", charsetMatch.getName(), charsetMatch.getConfidence());
}
}
if (reader == null) { // when all else fails
reader = encoding != null ?
new InputStreamReader(inputStream, encoding) :
new InputStreamReader(inputStream);
if (encoding != null) {
reader = new InputStreamReader(inputStream, encoding);
} else {
reader = new InputStreamReader(inputStream);
}
}
// Get the actual encoding which will be used and save it for project metadata
options.setProperty("encoding", reader.getEncoding());
((ReaderImporter) importer).read(reader, project, metadata, options);
} else {
// TODO: How do we set character encoding here?
// Things won't work right if it's not set, so pick some arbitrary values
options.setProperty("encoding", encoding);
options.setProperty("encoding_confidence", "0");
((StreamImporter) importer).read(rawInputStream, project, metadata, options);
}
}