filipg/geval
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement Probabilistic-MultiLabel-F1

Browse Source
This commit is contained in:
Filip Gralinski 2019-09-07 14:16:06 +02:00
parent c011ba3962
commit b540cba7da
10 changed files with 57 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/GEval/Core.hs
View File

@ -629,25 +629,14 @@ gevalCore' (SoftFMeasure beta) _ = gevalCoreWithoutInput parseAnnotations
Prelude.length expected,
Prelude.length got)
gevalCore' (ProbabilisticSoftFMeasure beta) _ = gevalCoreWithoutInput parseAnnotations
gevalCore' (ProbabilisticMultiLabelFMeasure beta) _ = generalizedProbabilisticFMeasure beta
intoWords
(Right . (\(ProbList es) -> es) . parseIntoProbList)
where intoWords = Right . Data.Text.words
gevalCore' (ProbabilisticSoftFMeasure beta) _ = generalizedProbabilisticFMeasure beta
parseAnnotations
parseObtainedAnnotations
getProbabilisticCounts
probabilisticSoftAgg
(fMeasureOnProbabilisticCounts beta)
loessGraph
where probabilisticSoftAgg :: Monad m => ConduitM ([Double], [Double], Double, Int) o m ([Double], [Double], Double, Int)
probabilisticSoftAgg = CC.foldl probabilisticSoftFolder ([], [], fromInteger 0, 0)
probabilisticSoftFolder (r1, p1, g1, e1) (r2, p2, g2, e2) = (r1 ++ r2, p1 ++ p2, g1 + g2, e1 + e2)
loessGraph :: ([Double], [Double], Double, Int) -> Maybe GraphSeries
loessGraph (results, probs, _, _) = Just $ GraphSeries $ Prelude.map (\x -> (x, clippedLoess probs' results' x)) $ Prelude.filter (\p -> p > lowest && p < highest) $ Prelude.map (\d -> 0.01 * (fromIntegral d)) [1..99]
where results' = DVU.fromList results
probs' = DVU.fromList probs
lowest = Data.List.minimum probs
highest = Data.List.maximum probs
fMeasureOnProbabilisticCounts :: Double -> ([Double], [Double], Double, Int) -> Double
fMeasureOnProbabilisticCounts beta (results, probs, got, nbExpected) = weightedHarmonicMean beta calibrationMeasure recall
where calibrationMeasure = softCalibration results probs
recall = got /. nbExpected
gevalCore' (Soft2DFMeasure beta) _ = gevalCoreWithoutInput parseLabeledClippings
parseLabeledClippings
@ -751,6 +740,27 @@ gevalCore' MultiLabelLogLoss _ = gevalCoreWithoutInput intoWords
where
intoWords = Right . Data.Text.words
generalizedProbabilisticFMeasure beta parseBareEntities parseEntities = gevalCoreWithoutInput parseBareEntities
parseEntities
getProbabilisticCounts
probabilisticSoftAgg
(fMeasureOnProbabilisticCounts beta)
loessGraph
where probabilisticSoftAgg :: Monad m => ConduitM ([Double], [Double], Double, Int) o m ([Double], [Double], Double, Int)
probabilisticSoftAgg = CC.foldl probabilisticSoftFolder ([], [], fromInteger 0, 0)
probabilisticSoftFolder (r1, p1, g1, e1) (r2, p2, g2, e2) = (r1 ++ r2, p1 ++ p2, g1 + g2, e1 + e2)
loessGraph :: ([Double], [Double], Double, Int) -> Maybe GraphSeries
loessGraph (results, probs, _, _) = Just $ GraphSeries $ Prelude.map (\x -> (x, clippedLoess probs' results' x)) $ Prelude.filter (\p -> p > lowest && p < highest) $ Prelude.map (\d -> 0.01 * (fromIntegral d)) [1..99]
where results' = DVU.fromList results
probs' = DVU.fromList probs
lowest = Data.List.minimum probs
highest = Data.List.maximum probs
fMeasureOnProbabilisticCounts :: Double -> ([Double], [Double], Double, Int) -> Double
fMeasureOnProbabilisticCounts beta (results, probs, got, nbExpected) = weightedHarmonicMean beta calibrationMeasure recall
where calibrationMeasure = softCalibration results probs
recall = got /. nbExpected
countAgg :: (Num n, Num v, Monad m) => ConduitM (n, v, v) o m (n, v, v)
countAgg = CC.foldl countFolder (fromInteger 0, fromInteger 0, fromInteger 0)

7
src/GEval/Metric.hs
View File

@ -28,7 +28,7 @@ data Metric = RMSE | MSE | Pearson | Spearman | BLEU | GLEU | WER | Accuracy | C
| LogLossHashed Word32 | CharMatch | MAP | LogLoss | Likelihood
| BIOF1 | BIOF1Labels | TokenAccuracy | LikelihoodHashed Word32 | MAE | SMAPE | MultiLabelFMeasure Double
| MultiLabelLogLoss | MultiLabelLikelihood
| SoftFMeasure Double | ProbabilisticSoftFMeasure Double | Soft2DFMeasure Double
| SoftFMeasure Double | ProbabilisticMultiLabelFMeasure Double | ProbabilisticSoftFMeasure Double | Soft2DFMeasure Double
deriving (Eq)
instance Show Metric where
@ -44,6 +44,7 @@ instance Show Metric where
show (FMeasure beta) = "F" ++ (show beta)
show (MacroFMeasure beta) = "Macro-F" ++ (show beta)
show (SoftFMeasure beta) = "Soft-F" ++ (show beta)
show (ProbabilisticMultiLabelFMeasure beta) = "Probabilistic-MultiLabel-F" ++ (show beta)
show (ProbabilisticSoftFMeasure beta) = "Probabilistic-Soft-F" ++ (show beta)
show (Soft2DFMeasure beta) = "Soft2D-F" ++ (show beta)
show NMI = "NMI"
@ -98,6 +99,9 @@ instance Read Metric where
readsPrec p ('S':'o':'f':'t':'-':'F':theRest) = case readsPrec p theRest of
[(beta, theRest)] -> [(SoftFMeasure beta, theRest)]
_ -> []
readsPrec p ('P':'r':'o':'b':'a':'b':'i':'l':'i':'s':'t':'i':'c':'-':'M':'u':'l':'t':'i':'L':'a':'b':'e':'l':'-':'F':theRest) = case readsPrec p theRest of
[(beta, theRest)] -> [(ProbabilisticMultiLabelFMeasure beta, theRest)]
_ -> []
readsPrec p ('P':'r':'o':'b':'a':'b':'i':'l':'i':'s':'t':'i':'c':'-':'S':'o':'f':'t':'-':'F':theRest) = case readsPrec p theRest of
[(beta, theRest)] -> [(ProbabilisticSoftFMeasure beta, theRest)]
_ -> []
@ -137,6 +141,7 @@ getMetricOrdering ClippEU = TheHigherTheBetter
getMetricOrdering (FMeasure _) = TheHigherTheBetter
getMetricOrdering (MacroFMeasure _) = TheHigherTheBetter
getMetricOrdering (SoftFMeasure _) = TheHigherTheBetter
getMetricOrdering (ProbabilisticMultiLabelFMeasure _) = TheHigherTheBetter
getMetricOrdering (ProbabilisticSoftFMeasure _) = TheHigherTheBetter
getMetricOrdering (Soft2DFMeasure _) = TheHigherTheBetter
getMetricOrdering NMI = TheHigherTheBetter

2
src/GEval/ProbList.hs
View File

@ -2,7 +2,7 @@
{-# LANGUAGE TypeFamilies #-}
module GEval.ProbList
(parseIntoProbList, selectByStandardThreshold, countLogLossOnProbList)
(parseIntoProbList, selectByStandardThreshold, countLogLossOnProbList, ProbList(..))
where
import qualified Data.Text as T

5
test/Spec.hs
View File

@ -264,6 +264,11 @@ main = hspec $ do
read "F2" `shouldBe` (FMeasure 2.0)
read "F1" `shouldBe` (FMeasure 1.0)
read "F0.5" `shouldBe` (FMeasure 0.5)
describe "Probabilistic-F1" $ do
it "simple test" $ do
runGEvalTest "probabilistic-f1-simple" `shouldReturnAlmost` 0.5
it "with probs" $ do
runGEvalTest "probabilistic-f1-probs" `shouldReturnAlmost` 0.5451223333805993
describe "Soft-F1" $ do
it "simple test" $ do
runGEvalTest "soft-f1-simple" `shouldReturnAlmost` 0.33333333333333

4
test/probabilistic-f1-probs/probabilistic-f1-probs-solution/test-A/out.tsv Normal file
View File

@ -0,0 +1,4 @@
foo bar:0.7
baz:0.2 foo:0.5
foo:0.7 foo:0.8 baq:0.8
1 foo bar:0.7
2 baz:0.2 foo:0.5
3 foo:0.7 foo:0.8 baq:0.8

1
test/probabilistic-f1-probs/probabilistic-f1-probs/config.txt Normal file
View File

@ -0,0 +1 @@
--metric Probabilistic-MultiLabel-F1

4
test/probabilistic-f1-probs/probabilistic-f1-probs/test-A/expected.tsv Normal file
View File

@ -0,0 +1,4 @@
foo bar
baz
baq foo foo
1 foo bar
2 baz
3 baq foo foo

3
test/probabilistic-f1-simple/probabilistic-f1-simple-solution/test-A/out.tsv Normal file
View File

@ -0,0 +1,3 @@
bar:1.0
baz:1.0
foo baz:1.0 bar:1.0 foo:1.0 foo
1 bar:1.0
2 baz:1.0
3 foo baz:1.0 bar:1.0 foo:1.0 foo

1
test/probabilistic-f1-simple/probabilistic-f1-simple/config.txt Normal file
View File

@ -0,0 +1 @@
--metric Probabilistic-MultiLabel-F1

3
test/probabilistic-f1-simple/probabilistic-f1-simple/test-A/expected.tsv Normal file
View File

@ -0,0 +1,3 @@
foo
bar baz baz foo
1 foo
2 bar baz baz foo