From cea084c789dc02291eaaab74aae17b685bc552ac Mon Sep 17 00:00:00 2001
From: Filip Gralinski <filipg@amu.edu.pl>
Date: Sat, 7 Apr 2018 21:13:37 +0200
Subject: [PATCH] accuracy can work on probs now

---
 geval.cabal                                      |  2 +-
 src/GEval/Core.hs                                | 16 +++++++++++++++-
 test/Spec.hs                                     |  2 ++
 .../accuracy-probs-solution/test-A/out.tsv       | 10 ++++++++++
 test/accuracy-probs/accuracy-probs/config.txt    |  1 +
 .../accuracy-probs/test-A/expected.tsv           | 10 ++++++++++
 6 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 test/accuracy-probs/accuracy-probs-solution/test-A/out.tsv
 create mode 100644 test/accuracy-probs/accuracy-probs/config.txt
 create mode 100644 test/accuracy-probs/accuracy-probs/test-A/expected.tsv

diff --git a/geval.cabal b/geval.cabal
index 52bd563..42531ec 100644
--- a/geval.cabal
+++ b/geval.cabal
@@ -1,5 +1,5 @@
 name:                geval
-version:             0.5.3.0
+version:             0.5.4.0
 synopsis:            Machine learning evaluation tools
 description:         Please see README.md
 homepage:            http://github.com/name/project
diff --git a/src/GEval/Core.hs b/src/GEval/Core.hs
index 219db1e..6b29f7e 100644
--- a/src/GEval/Core.hs
+++ b/src/GEval/Core.hs
@@ -47,6 +47,7 @@ import Data.Text
 import Data.Text.Read as TR
 import Control.Applicative
 import Control.Exception
+import Text.Read (readMaybe)
 import Control.Conditional (unlessM, whenM)
 import qualified System.Directory as D
 import System.Posix
@@ -312,7 +313,20 @@ gevalCore' BLEU _ = gevalCoreWithoutInput (Right . Prelude.map Prelude.words . D
           | otherwise = exp (1.0 - (r /. c))
 
 gevalCore' Accuracy _ = gevalCoreWithoutInput (Right . strip) (Right . strip) hitOrMiss averageC id
-                      where hitOrMiss (x,y) = if x == y then 1.0 else 0.0
+                      where hitOrMiss (exp,got) = if (normalizeProbForAccuracy exp got) == exp then 1.0 else 0.0
+                            -- if the expected value is 0 or 1 treat values between 0.0 and 1.0 as probabilities
+                            -- for the positive outcome
+                            normalizeProbForAccuracy :: Text -> Text -> Text
+                            normalizeProbForAccuracy exp got
+                              | exp == (pack "1") = case tryReadingAsFloat got of
+                                            Just p -> if p >= 0.5 && p <= 1.0 then exp else got
+                                            Nothing -> got
+                              | exp == (pack "0") = case tryReadingAsFloat got of
+                                            Just p -> if p < 0.5 && p >= 0.0 then exp else got
+                                            Nothing -> got
+                              | otherwise = got
+                            tryReadingAsFloat :: Text -> Maybe Float
+                            tryReadingAsFloat = readMaybe . unpack
 
 gevalCore' (FMeasure beta) _ = gevalCoreWithoutInput outParser outParser getCount countAgg (fMeasureOnCounts beta)
   where outParser = detected <=< (getValue . TR.double)
diff --git a/test/Spec.hs b/test/Spec.hs
index c190b4d..9be9045 100644
--- a/test/Spec.hs
+++ b/test/Spec.hs
@@ -53,6 +53,8 @@ main = hspec $ do
   describe "Accuracy" $ do
     it "simple example" $
       runGEvalTest "accuracy-simple" `shouldReturnAlmost` 0.6
+    it "with probs" $
+      runGEvalTest "accuracy-probs" `shouldReturnAlmost` 0.4
   describe "F-measure" $ do
     it "simple example" $
       runGEvalTest "f-measure-simple" `shouldReturnAlmost` 0.57142857
diff --git a/test/accuracy-probs/accuracy-probs-solution/test-A/out.tsv b/test/accuracy-probs/accuracy-probs-solution/test-A/out.tsv
new file mode 100644
index 0000000..588f41d
--- /dev/null
+++ b/test/accuracy-probs/accuracy-probs-solution/test-A/out.tsv
@@ -0,0 +1,10 @@
+1
+0
+foo
+0
+-1.5
+0.7
+1.001
+0.3
+0.45
+0.45
diff --git a/test/accuracy-probs/accuracy-probs/config.txt b/test/accuracy-probs/accuracy-probs/config.txt
new file mode 100644
index 0000000..337a0cc
--- /dev/null
+++ b/test/accuracy-probs/accuracy-probs/config.txt
@@ -0,0 +1 @@
+--metric Accuracy
diff --git a/test/accuracy-probs/accuracy-probs/test-A/expected.tsv b/test/accuracy-probs/accuracy-probs/test-A/expected.tsv
new file mode 100644
index 0000000..f76fa70
--- /dev/null
+++ b/test/accuracy-probs/accuracy-probs/test-A/expected.tsv
@@ -0,0 +1,10 @@
+1
+0
+0
+1
+0
+1
+1
+1
+1
+0