s434730/RandomSec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #3349 from thadguidry/fingerprint-wynn-thadguidry

Browse Source 
Add Wynn to fingerprint to support Old English texts
This commit is contained in:
Tom Morris 2020-11-28 22:08:00 -05:00 committed by GitHub
commit 2872ceeb7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
main/src/com/google/refine/clustering/binning/FingerprintKeyer.java
View File

@ -72,6 +72,7 @@ public class FingerprintKeyer extends Keyer {
.put("\u0111", "d") // Small letter D with stroke
.put("\u0256", "d") // Small letter African D
.put("\u00FE", "th") // Lower case Icelandic thorn þ
.put("ƿ","w") // Lower case Wynn from Old English modernly transliterated to w
// Visually similar replacements from our private former asciify() method
// (only need lower case forms since we're already downcased)
.put("\u0127", "h") // small H with stroke
@ -141,4 +142,4 @@ public class FingerprintKeyer extends Keyer {
return result.toString();
}
}
}

2
main/tests/server/src/com/google/refine/clustering/binning/KeyerTests.java
View File

@ -55,7 +55,7 @@ public class KeyerTests extends RefineTest {
// {"å","aa"}, // Requested by issue #650, but conflicts with diacritic folding
{"æø","aeoe"}, // Norwegian replacements from #650
{"©ß","css"}, // issue #409 esszet
{"\u00D0\u00DE", "dth"}, // Icelandic eth and thorn
{"\u00D0\u00F0\u00DE\u00FEǷƿ", "ddththww"}, // eth, thorn, & wynn for Icelandic / Olde English
{"fffiflffiſtst", "fffiflffistst"}, // ligatures
// Test legacy replacements
{"\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u0100\u0101\u0102\u0103\u0104\u0105", "aaaaaaaaaaaaaaaaaa"},