From 5dc6e13191b1e48239828edee9ced790acac546c Mon Sep 17 00:00:00 2001 From: Filip Gralinski Date: Thu, 27 Sep 2018 21:52:02 +0200 Subject: [PATCH] Add Pearson and Spearman correlation measures --- src/GEval/Core.hs | 26 ++++++++++++++++++- src/GEval/OptionsParser.hs | 2 +- test/Spec.hs | 3 +++ .../spearman-simple-solution/test-A/out.tsv | 5 ++++ .../spearman-simple/config.txt | 1 + .../spearman-simple/test-A/expected.tsv | 5 ++++ 6 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 test/spearman-simple/spearman-simple-solution/test-A/out.tsv create mode 100644 test/spearman-simple/spearman-simple/config.txt create mode 100644 test/spearman-simple/spearman-simple/test-A/expected.tsv diff --git a/src/GEval/Core.hs b/src/GEval/Core.hs index 380332f..56e6293 100644 --- a/src/GEval/Core.hs +++ b/src/GEval/Core.hs @@ -85,6 +85,10 @@ import Data.Conduit.AutoDecompress import Text.Tokenizer import qualified Data.HashMap.Strict as M +import qualified Data.Vector as V +import qualified Data.Vector.Generic as VG + +import Statistics.Correlation import Data.Proxy @@ -98,7 +102,8 @@ defaultLogLossHashedSize :: Word32 defaultLogLossHashedSize = 10 -- | evaluation metric -data Metric = RMSE | MSE | BLEU | GLEU | WER | Accuracy | ClippEU | FMeasure Double | MacroFMeasure Double | NMI +data Metric = RMSE | MSE | Pearson | Spearman | BLEU | GLEU | WER | Accuracy | ClippEU + | FMeasure Double | MacroFMeasure Double | NMI | LogLossHashed Word32 | CharMatch | MAP | LogLoss | Likelihood | BIOF1 | BIOF1Labels | LikelihoodHashed Word32 | MAE | MultiLabelFMeasure Double | MultiLabelLogLoss | MultiLabelLikelihood @@ -107,6 +112,8 @@ data Metric = RMSE | MSE | BLEU | GLEU | WER | Accuracy | ClippEU | FMeasure Dou instance Show Metric where show RMSE = "RMSE" show MSE = "MSE" + show Pearson = "Pearson" + show Spearman = "Spearman" show BLEU = "BLEU" show GLEU = "GLEU" show WER = "WER" @@ -141,6 +148,8 @@ instance Show Metric where instance Read Metric where readsPrec _ ('R':'M':'S':'E':theRest) = [(RMSE, theRest)] readsPrec _ ('M':'S':'E':theRest) = [(MSE, theRest)] + readsPrec _ ('P':'e':'a':'r':'s':'o':'n':theRest) = [(Pearson, theRest)] + readsPrec _ ('S':'p':'e':'a':'r':'m':'a':'n':theRest) = [(Spearman, theRest)] readsPrec _ ('B':'L':'E':'U':theRest) = [(BLEU, theRest)] readsPrec _ ('G':'L':'E':'U':theRest) = [(GLEU, theRest)] readsPrec _ ('W':'E':'R':theRest) = [(WER, theRest)] @@ -180,6 +189,8 @@ data MetricOrdering = TheLowerTheBetter | TheHigherTheBetter getMetricOrdering :: Metric -> MetricOrdering getMetricOrdering RMSE = TheLowerTheBetter getMetricOrdering MSE = TheLowerTheBetter +getMetricOrdering Pearson = TheHigherTheBetter +getMetricOrdering Spearman = TheHigherTheBetter getMetricOrdering BLEU = TheHigherTheBetter getMetricOrdering GLEU = TheHigherTheBetter getMetricOrdering WER = TheLowerTheBetter @@ -514,6 +525,8 @@ gevalCore' MSE _ = gevalCoreWithoutInput outParser outParser itemSquaredError av gevalCore' MAE _ = gevalCoreWithoutInput outParser outParser itemAbsoluteError averageC id where outParser = getValue . TR.double +gevalCore' Pearson _ = gevalCoreByCorrelationMeasure pearson +gevalCore' Spearman _ = gevalCoreByCorrelationMeasure spearman gevalCore' LogLoss _ = gevalCoreWithoutInput outParser outParser itemLogLossError averageC id where outParser = getValue . TR.double @@ -670,6 +683,17 @@ gevalCore' MultiLabelLogLoss _ = gevalCoreWithoutInput intoWords countAgg :: Monad m => ConduitM (Int, Int, Int) o m (Int, Int, Int) countAgg = CC.foldl countFolder (0, 0, 0) +gevalCoreByCorrelationMeasure :: (MonadUnliftIO m, MonadThrow m, MonadIO m) => + (V.Vector (Double, Double) -> Double) -> -- ^ correlation function + LineSource (ResourceT m) -> -- ^ source to read the expected output + LineSource (ResourceT m) -> -- ^ source to read the output + m (MetricValue) -- ^ metric values for the output against the expected output +gevalCoreByCorrelationMeasure correlationFunction = + gevalCoreWithoutInput outParser outParser id correlationC finalStep + where outParser = getValue . TR.double + correlationC = CC.foldl (flip (:)) [] + finalStep pairs = correlationFunction $ V.fromList pairs + parseDistributionWrapper :: Word32 -> Word32 -> Text -> HashedDistribution parseDistributionWrapper nbOfBits seed distroSpec = case parseDistribution nbOfBits seed distroSpec of Right distro -> distro diff --git a/src/GEval/OptionsParser.hs b/src/GEval/OptionsParser.hs index 7a9097f..e842d18 100644 --- a/src/GEval/OptionsParser.hs +++ b/src/GEval/OptionsParser.hs @@ -169,7 +169,7 @@ metricReader = many $ option auto -- actually `some` should be used inst ( long "metric" -- --metric might be in the config.txt file... <> short 'm' <> metavar "METRIC" - <> help "Metric to be used - RMSE, MSE, Accuracy, LogLoss, Likelihood, F-measure (specify as F1, F2, F0.25, etc.), macro F-measure (specify as Macro-F1, Macro-F2, Macro-F0.25, etc.), multi-label F-measure (specify as MultiLabel-F1, MultiLabel-F2, MultiLabel-F0.25, etc.), MAP, BLEU, GLEU (\"Google GLEU\" not the grammar correction metric), WER, NMI, ClippEU, LogLossHashed, LikelihoodHashed, BIO-F1, BIO-F1-Labels or CharMatch" ) + <> help "Metric to be used - RMSE, MSE, Pearson, Spearman, Accuracy, LogLoss, Likelihood, F-measure (specify as F1, F2, F0.25, etc.), macro F-measure (specify as Macro-F1, Macro-F2, Macro-F0.25, etc.), multi-label F-measure (specify as MultiLabel-F1, MultiLabel-F2, MultiLabel-F0.25, etc.), MAP, BLEU, GLEU (\"Google GLEU\" not the grammar correction metric), WER, NMI, ClippEU, LogLossHashed, LikelihoodHashed, BIO-F1, BIO-F1-Labels or CharMatch" ) altMetricReader :: Parser (Maybe Metric) altMetricReader = optional $ option auto diff --git a/test/Spec.hs b/test/Spec.hs index f4a3bfb..3134483 100644 --- a/test/Spec.hs +++ b/test/Spec.hs @@ -70,6 +70,9 @@ main = hspec $ do describe "mean absolute error" $ do it "simple test with arguments" $ runGEvalTest "mae-simple" `shouldReturnAlmost` 1.5 + describe "Spearman's rank correlation coefficient" $ do + it "simple test" $ do + runGEvalTest "spearman-simple" `shouldReturnAlmost` (- 0.5735) describe "BLEU" $ do it "trivial example from Wikipedia" $ runGEvalTest "bleu-trivial" `shouldReturnAlmost` 0.0 diff --git a/test/spearman-simple/spearman-simple-solution/test-A/out.tsv b/test/spearman-simple/spearman-simple-solution/test-A/out.tsv new file mode 100644 index 0000000..cd17e47 --- /dev/null +++ b/test/spearman-simple/spearman-simple-solution/test-A/out.tsv @@ -0,0 +1,5 @@ +1.2 +1 +2.3 +1 +18 diff --git a/test/spearman-simple/spearman-simple/config.txt b/test/spearman-simple/spearman-simple/config.txt new file mode 100644 index 0000000..00f42de --- /dev/null +++ b/test/spearman-simple/spearman-simple/config.txt @@ -0,0 +1 @@ +--metric Spearman diff --git a/test/spearman-simple/spearman-simple/test-A/expected.tsv b/test/spearman-simple/spearman-simple/test-A/expected.tsv new file mode 100644 index 0000000..20114a2 --- /dev/null +++ b/test/spearman-simple/spearman-simple/test-A/expected.tsv @@ -0,0 +1,5 @@ +1.1 +1.57 +0.51 +1.1 +1.1