diff --git a/README.md b/README.md index 11de003..9edb568 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,6 @@ Then let Gonito pull them and evaluate your results. evaluated (default: "out.tsv") --expected-file EXPECTED The name of the file with expected results (default: "expected.tsv") - --metric METRIC Metric to be used - RMSE, MSE or BLEU (default: RMSE) + --metric METRIC Metric to be used - RMSE, MSE, Accuracy or BLEU (default: RMSE) If you need another metric, let me know, or do it yourself! diff --git a/geval.cabal b/geval.cabal index dc6c7d5..1895440 100644 --- a/geval.cabal +++ b/geval.cabal @@ -1,5 +1,5 @@ name: geval -version: 0.1.1.1 +version: 0.1.2.1 synopsis: Machine learning evaluation tools description: Please see README.md homepage: http://github.com/name/project diff --git a/src/GEval/Core.hs b/src/GEval/Core.hs index 60927e9..0668328 100644 --- a/src/GEval/Core.hs +++ b/src/GEval/Core.hs @@ -38,7 +38,7 @@ import GEval.BLEU type MetricValue = Double -data Metric = RMSE | MSE | BLEU +data Metric = RMSE | MSE | BLEU | Accuracy deriving (Show, Read) defaultOutDirectory = "." @@ -141,6 +141,9 @@ gevalCore' BLEU = gevalCore'' (Prelude.map Prelude.words . DLS.splitOn "\t" . un | c >= r = 1.0 | otherwise = exp (1.0 - (r /. c)) +gevalCore' Accuracy = gevalCore'' strip strip hitOrMiss averageC id + where hitOrMiss (x,y) = if x == y then 1.0 else 0.0 + (/.) :: Int -> Int -> Double x /. 0 = 1.0 x /. y = (fromIntegral x) / (fromIntegral y) diff --git a/src/GEval/OptionsParser.hs b/src/GEval/OptionsParser.hs index d86571f..cca4605 100644 --- a/src/GEval/OptionsParser.hs +++ b/src/GEval/OptionsParser.hs @@ -73,7 +73,7 @@ metricReader = option auto <> value defaultMetric <> showDefault <> metavar "METRIC" - <> help "Metric to be used - RMSE, MSE or BLEU" ) + <> help "Metric to be used - RMSE, MSE, Accuracy or BLEU" ) runGEval :: [String] -> IO (Either (ParserResult GEvalOptions) (Maybe MetricValue)) runGEval args = do diff --git a/test/Spec.hs b/test/Spec.hs index 009ea99..927f886 100644 --- a/test/Spec.hs +++ b/test/Spec.hs @@ -33,6 +33,12 @@ main = hspec $ do "test/bleu-perfect/bleu-perfect", "--out-directory", "test/bleu-perfect/bleu-perfect-solution"]) >>= extractVal) `shouldReturnAlmost` 1.0000 + describe "Accuracy" $ do + it "simple example" $ do + ((runGEval ["--expected-directory", + "test/accuracy-simple/accuracy-simple", + "--out-directory", + "test/accuracy-simple/accuracy-simple-solution"]) >>= extractVal) `shouldReturnAlmost` 0.6 describe "precision count" $ do it "simple test" $ do precisionCount [["Alice", "has", "a", "cat" ]] ["Ala", "has", "cat"] `shouldBe` 2 diff --git a/test/accuracy-simple/accuracy-simple-solution/test-A/out.tsv b/test/accuracy-simple/accuracy-simple-solution/test-A/out.tsv new file mode 100644 index 0000000..483a307 --- /dev/null +++ b/test/accuracy-simple/accuracy-simple-solution/test-A/out.tsv @@ -0,0 +1,5 @@ +A +B +A +B +B diff --git a/test/accuracy-simple/accuracy-simple/config.txt b/test/accuracy-simple/accuracy-simple/config.txt new file mode 100644 index 0000000..337a0cc --- /dev/null +++ b/test/accuracy-simple/accuracy-simple/config.txt @@ -0,0 +1 @@ +--metric Accuracy diff --git a/test/accuracy-simple/accuracy-simple/test-A/expected.tsv b/test/accuracy-simple/accuracy-simple/test-A/expected.tsv new file mode 100644 index 0000000..613e83c --- /dev/null +++ b/test/accuracy-simple/accuracy-simple/test-A/expected.tsv @@ -0,0 +1,5 @@ +A +B +B +A +B