s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Perform ASCII normalization earlier in FingerprintKeyer.

Browse Source 
This closes #1256.
This commit is contained in:
Antonin Delpeuch 2017-09-27 09:23:40 +01:00
parent 5d387b1df8
commit 1da3c00cb1
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
main/src/com/google/refine/clustering/binning/FingerprintKeyer.java
View File

@ -52,6 +52,7 @@ public class FingerprintKeyer extends Keyer {
s = s.trim(); // first off, remove whitespace around the string s = s.trim(); // first off, remove whitespace around the string
s = s.toLowerCase(); // then lowercase it s = s.toLowerCase(); // then lowercase it
s = punctctrl.matcher(s).replaceAll(""); // then remove all punctuation and control chars s = punctctrl.matcher(s).replaceAll(""); // then remove all punctuation and control chars
s = asciify(s); // find ASCII equivalent to characters
String[] frags = StringUtils.split(s); // split by whitespace String[] frags = StringUtils.split(s); // split by whitespace
TreeSet<String> set = new TreeSet<String>(); TreeSet<String> set = new TreeSet<String>();
for (String ss : frags) { for (String ss : frags) {
@ -65,7 +66,7 @@ public class FingerprintKeyer extends Keyer {
b.append(' '); b.append(' ');
} }
} }
return asciify(b.toString()); // find ASCII equivalent to characters return b.toString();
} }
protected String asciify(String s) { protected String asciify(String s) {

1
main/tests/server/src/com/google/refine/tests/clustering/binning/KeyerTests.java
View File

@ -50,6 +50,7 @@ public class KeyerTests extends RefineTest {
private static final String[][] testStrings = { private static final String[][] testStrings = {
{"the multi multi word test","multi test the word"}, {"the multi multi word test","multi test the word"},
{" école ÉCole ecoLe ", "ecole"},
{"a b c d","a b c d"}, {"a b c d","a b c d"},
{" d c b a ","a b c d"}, {" d c b a ","a b c d"},
{"\tABC \t DEF ","abc def"}, // test leading and trailing whitespace {"\tABC \t DEF ","abc def"}, // test leading and trailing whitespace