This commit is contained in:
Filip Gralinski 2019-11-07 12:33:46 +01:00
parent 41fe0d2283
commit 234bac19ce

View File

@ -471,7 +471,6 @@ gevalCoreOnSources Spearman _ = gevalCoreByCorrelationMeasure spearman
gevalCoreOnSources (ProbabilisticMultiLabelFMeasure beta) _ = generalizedProbabilisticFMeasure beta
SAProbabilisticMultiLabelFMeasure
where intoWords = Right . Data.Text.words
gevalCoreOnSources (ProbabilisticSoftFMeasure beta) _ = generalizedProbabilisticFMeasure beta
SAProbabilisticSoftFMeasure
@ -489,8 +488,6 @@ gevalCoreOnSources MultiLabelLikelihood _ = gevalCoreWithoutInput SAMultiLabelLi
averageC
logLossToLikehood
noGraph
where
intoWords = Right . Data.Text.words
gevalCoreOnSources MSE _ = gevalCoreWithoutInput SAMSE averageC id noGraph
@ -695,6 +692,112 @@ gevalCoreGeneralized' parserSpec itemStep aggregator finalStep generateGraph con
<*> (ZipSource $ recordSource context parserSpec)) .| CL.map (checkStep (Proxy :: Proxy m) itemStep)) .| CL.catMaybes .| aggregator)
return $ MetricOutput (SimpleRun $ finalStep v) (generateGraph v)
continueGEvalCalculations :: (MonadIO m) =>
SAMetric t
-> Metric
-> ConduitT (ItemIntermediateRepresentationType t) Void (ResourceT m) MetricOutput
continueGEvalCalculations SALikelihood Likelihood = defineContinuation averageC logLossToLikehood noGraph
continueGEvalCalculations SAMultiLabelLikelihood MultiLabelLikelihood = defineContinuation averageC
logLossToLikehood
noGraph
continueGEvalCalculations SAMSE MSE = defineContinuation averageC id noGraph
continueGEvalCalculations SARMSE RMSE = defineContinuation averageC (** 0.5) noGraph
continueGEvalCalculations SAMAE MAE = defineContinuation averageC id noGraph
continueGEvalCalculations SASMAPE SMAPE = defineContinuation averageC (* 100.0) noGraph
continueGEvalCalculations SALogLoss LogLoss = defineContinuation averageC id noGraph
continueGEvalCalculations SABLEU BLEU = defineContinuation bleuAgg bleuFinal noGraph
where bleuFinal (p1, p2, p3, p4, rl, l1, l2, l3, l4) = ((p1 /. l1) * (p2 /. l2) * (p3 /. l3) * (p4 /. l4)) ** 0.25 * (brevityPenalty l1 rl)
bleuAgg = CC.foldl bleuFuse (0, 0, 0, 0, 0, 0, 0, 0, 0)
bleuFuse (a1, a2, a3, a4, a5, a6, a7, a8, a9) (b1, b2, b3, b4, b5, b6, b7, b8, b9) = (a1+b1, a2+b2, a3+b3, a4+b4, a5+b5, a6+b6, a7+b7, a8+b8, a9+b9)
brevityPenalty c r
| c >= r = 1.0
| c == 0 && r > 0 = 0.0
| otherwise = exp (1.0 - (r /. c))
continueGEvalCalculations GLEU _ = defineContinuation SAGLEU gleuAgg gleuFinal noGraph
where gleuFinal (m, t) = m /. t
gleuAgg = CC.foldl gleuFuse (0, 0)
gleuFuse (a1, a2) (b1, b2) = (a1+b1, a2+b2)
continueGEvalCalculations WER _ = defineContinuation SAWER averageC id noGraph
continueGEvalCalculations Accuracy _ = defineContinuation SAAccuracy averageC id noGraph
continueGEvalCalculations (FMeasure beta) _ = defineContinuation SAFMeasure countAgg (fMeasureOnCounts beta) noGraph
continueGEvalCalculations (MacroFMeasure beta) _ = defineContinuation SAMacroFMeasure gatherClassC macroAverageOnCounts noGraph
where gatherClassC = CC.foldl gatherClassCombiner (M.empty, M.empty, M.empty)
gatherClassCombiner (tpMap, expectedMap, gotMap) (tp, expected, got) =
(insertMaybeToMap tp tpMap,
insertMaybeToMap expected expectedMap,
insertMaybeToMap got gotMap)
insertMaybeToMap Nothing m = m
insertMaybeToMap (Just c) m = M.insertWith (+) c 1 m
macroAverageOnCounts (tpMap, expectedMap, gotMap) =
(Prelude.sum
$ Prelude.map (\c -> fMeasureOnCounts beta (M.lookupDefault (0::Int) c tpMap,
M.lookupDefault 0 c expectedMap,
M.lookupDefault 0 c gotMap))
$ M.keys expectedMap) / (fromIntegral $ Prelude.length $ M.keys expectedMap)
continueGEvalCalculations (SoftFMeasure beta) _ = defineContinuation SASoftFMeasure
countAgg
(fMeasureOnCounts beta)
noGraph
continueGEvalCalculations (Soft2DFMeasure beta) _ = defineContinuation SASoft2DFMeasure
(CC.map (fMeasureOnCounts beta) .| averageC)
id
noGraph
continueGEvalCalculations ClippEU _ = defineContinuation SAClippEU clippeuAgg finalStep noGraph
where
clippeuAgg = CC.foldl countFolder (0, 0, 0)
finalStep counts = f2MeasureOnCounts counts
continueGEvalCalculations NMI _ = defineContinuation SANMI (CC.foldl updateConfusionMatrix M.empty) normalizedMutualInformationFromConfusionMatrix noGraph
continueGEvalCalculations MAP _ = defineContinuation SAMAP
averageC
id
noGraph
continueGEvalCalculations BIOF1 _ = defineContinuation SABIOF1 countAgg f1MeasureOnCounts noGraph
continueGEvalCalculations BIOF1Labels _ = defineContinuation SABIOF1Labels countAgg f1MeasureOnCounts noGraph
continueGEvalCalculations TokenAccuracy _ = defineContinuation SATokenAccuracy
hitsAndTotalsAgg
(\(hits, total) -> hits /. total)
noGraph
where
hitsAndTotalsAgg = CC.foldl (\(h1, t1) (h2, t2) -> (h1 + h2, t1 + t2)) (0, 0)
continueGEvalCalculations MultiLabelLogLoss _ = defineContinuation SAMultiLabelLogLoss
averageC
id
noGraph
defineContinuation :: (ConduitT c Void (ResourceT m) d) -- ^ a Conduit which aggregates all the combined values into
-- a "total" value
-> (d -> Double) -- ^ function to transform the "total" value into the final score
-> (d -> Maybe GraphSeries)
-> ConduitT c Void (ResourceT m) MetricOutput
defineContinuation aggregator finalStep generateGraph = do
v <- aggregator
return $ MetricOutput (SimpleRun $ finalStep v) (generateGraph v)
-- | A type family to handle all the evaluation "context".
--
-- This is needed as for some metrics the output and the expected metric is enough