Change the meaning of WER

This commit is contained in:
Filip Gralinski 2019-12-21 16:03:52 +01:00
parent 170cc417ca
commit 01486d23aa
3 changed files with 28 additions and 6 deletions

View File

@ -516,7 +516,21 @@ gevalCoreOnSources (Mean (MultiLabelFMeasure beta)) _
intoWords (RawItemTarget t) = Prelude.map unpack $ Data.Text.words t intoWords (RawItemTarget t) = Prelude.map unpack $ Data.Text.words t
intoWords (PartiallyParsedItemTarget ts) = Prelude.map unpack ts intoWords (PartiallyParsedItemTarget ts) = Prelude.map unpack ts
gevalCoreOnSources (Mean _) _ = error $ "Mean/ meta-metric defined only for MultiLabel-F1 for the time being" gevalCoreOnSources (Mean WER) _
= gevalCoreWithoutInputOnItemTargets (Right . intoWords)
(Right . getWords)
((uncurry (/.)) . (uncurry werStep))
averageC
id
noGraph
where
-- repeated as below, as it will be refactored into dependent types soon anyway
getWords (RawItemTarget t) = Prelude.map unpack $ selectByStandardThreshold $ parseIntoProbList t
getWords (PartiallyParsedItemTarget ts) = Prelude.map unpack ts
intoWords (RawItemTarget t) = Prelude.map unpack $ Data.Text.words t
intoWords (PartiallyParsedItemTarget ts) = Prelude.map unpack ts
gevalCoreOnSources (Mean _) _ = error $ "Mean/ meta-metric defined only for MultiLabel-F1 and WER for the time being"
-- only MultiLabel-F1 handled for JSONs for the time being... -- only MultiLabel-F1 handled for JSONs for the time being...
gevalCoreOnSources (MultiLabelFMeasure beta) _ = gevalCoreWithoutInputOnItemTargets (Right . intoWords) gevalCoreOnSources (MultiLabelFMeasure beta) _ = gevalCoreWithoutInputOnItemTargets (Right . intoWords)
@ -588,7 +602,15 @@ gevalCoreOnSources GLEU _ = gevalCoreWithoutInput (Right . Prelude.map Prelude.w
gleuAgg = CC.foldl gleuFuse (0, 0) gleuAgg = CC.foldl gleuFuse (0, 0)
gleuFuse (a1, a2) (b1, b2) = (a1 + b1, a2 + b2) gleuFuse (a1, a2) (b1, b2) = (a1 + b1, a2 + b2)
gevalCoreOnSources WER _ = gevalCoreWithoutInput (Right . Prelude.words . unpack) (Right . Prelude.words . unpack) (uncurry werStep) averageC id noGraph gevalCoreOnSources WER _ = gevalCoreWithoutInput (Right . Prelude.words . unpack)
(Right . Prelude.words . unpack)
(uncurry werStep)
werAgg
werFinal
noGraph
where werAgg = CC.foldl werFuse (0, 0)
werFuse (a1, a2) (b1, b2) = (a1 + b1, a2 + b2)
werFinal (errors, ref) = errors /. ref
gevalCoreOnSources Accuracy _ = gevalCoreWithoutInput (Right . strip) (Right . strip) hitOrMiss averageC id noGraph gevalCoreOnSources Accuracy _ = gevalCoreWithoutInput (Right . strip) (Right . strip) hitOrMiss averageC id noGraph
where hitOrMiss (exp, got) = where hitOrMiss (exp, got) =

View File

@ -5,8 +5,8 @@ module GEval.WER
import Data.Array import Data.Array
import GEval.Common import GEval.Common
werStep :: Eq a => [a] -> [a] -> Double werStep :: Eq a => [a] -> [a] -> (Int, Int)
werStep expected got = (fromIntegral $ distance expected got) `safeDoubleDiv` (fromIntegral $ length expected) werStep expected got = (distance expected got, length expected)
-- see https://stackoverflow.com/questions/6718787/levenshtein-distance-cost -- see https://stackoverflow.com/questions/6718787/levenshtein-distance-cost
distance u v = memo ! (m, n) distance u v = memo ! (m, n)

View File

@ -1 +1 @@
--metric WER --metric Mean/WER