Refactor (introduce GEval.Metric)

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.18.0.0
+
+* Add --validate option
+
 ## 1.17.0.0
 
 * Add Probabilistic-Soft-F-score

geval.cabal

@@ -16,7 +16,8 @@ cabal-version:       >=1.10
 library
   hs-source-dirs:      src
   exposed-modules:     GEval.Core
-                       GEval.CreateChallenge
+                     , GEval.Metric
+                     , GEval.CreateChallenge
                      , GEval.OptionsParser
                      , GEval.BLEU
                      , GEval.ClippEU

src/GEval/Core.hs

@@ -11,10 +11,8 @@ module GEval.Core
     ( geval,
       gevalCore,
       gevalCoreOnSingleLines,
+      module GEval.Metric,
       LineInFile(..),
-      Metric(..),
-      MetricOrdering(..),
-      getMetricOrdering,
       isBetter,
       isBetterOrEqual,
       GEvalSpecialCommand(..),
@@ -50,6 +48,8 @@ module GEval.Core
       somethingWrongWithFilesMessage
     ) where
 
+import GEval.Metric
+
 import Data.Conduit
 import Data.Conduit.Combinators as CC
 import qualified Data.Conduit.Binary as CB
@@ -114,136 +114,6 @@ import Data.Word
 
 import "Glob" System.FilePath.Glob
 
-defaultLogLossHashedSize :: Word32
-defaultLogLossHashedSize = 10
-
--- | evaluation metric
-data Metric = RMSE | MSE | Pearson | Spearman | BLEU | GLEU | WER | Accuracy | ClippEU
-              | FMeasure Double | MacroFMeasure Double | NMI
-              | LogLossHashed Word32 | CharMatch | MAP | LogLoss | Likelihood
-              | BIOF1 | BIOF1Labels | TokenAccuracy | LikelihoodHashed Word32 | MAE | SMAPE | MultiLabelFMeasure Double
-              | MultiLabelLogLoss | MultiLabelLikelihood
-              | SoftFMeasure Double | ProbabilisticSoftFMeasure Double
-              deriving (Eq)
-
-instance Show Metric where
-  show RMSE = "RMSE"
-  show MSE  = "MSE"
-  show Pearson = "Pearson"
-  show Spearman = "Spearman"
-  show BLEU = "BLEU"
-  show GLEU = "GLEU"
-  show WER = "WER"
-  show Accuracy = "Accuracy"
-  show ClippEU = "ClippEU"
-  show (FMeasure beta) = "F" ++ (show beta)
-  show (MacroFMeasure beta) = "Macro-F" ++ (show beta)
-  show (SoftFMeasure beta) = "Soft-F" ++ (show beta)
-  show (ProbabilisticSoftFMeasure beta) = "Probabilistic-Soft-F" ++ (show beta)
-  show NMI = "NMI"
-  show (LogLossHashed nbOfBits) = "LogLossHashed" ++ (if
-                                                       nbOfBits == defaultLogLossHashedSize
-                                                      then
-                                                       ""
-                                                      else
-                                                       (show nbOfBits))
-  show (LikelihoodHashed nbOfBits) = "LikelihoodHashed" ++ (if
-                                                               nbOfBits == defaultLogLossHashedSize
-                                                            then
-                                                              ""
-                                                            else
-                                                              (show nbOfBits))
-  show CharMatch = "CharMatch"
-  show MAP = "MAP"
-  show LogLoss = "LogLoss"
-  show Likelihood = "Likelihood"
-  show BIOF1 = "BIO-F1"
-  show BIOF1Labels = "BIO-F1-Labels"
-  show TokenAccuracy = "TokenAccuracy"
-  show MAE = "MAE"
-  show SMAPE = "SMAPE"
-  show (MultiLabelFMeasure beta) = "MultiLabel-F" ++ (show beta)
-  show MultiLabelLogLoss = "MultiLabel-Logloss"
-  show MultiLabelLikelihood = "MultiLabel-Likelihood"
-
-instance Read Metric where
-  readsPrec _ ('R':'M':'S':'E':theRest) = [(RMSE, theRest)]
-  readsPrec _ ('M':'S':'E':theRest) = [(MSE, theRest)]
-  readsPrec _ ('P':'e':'a':'r':'s':'o':'n':theRest) = [(Pearson, theRest)]
-  readsPrec _ ('S':'p':'e':'a':'r':'m':'a':'n':theRest) = [(Spearman, theRest)]
-  readsPrec _ ('B':'L':'E':'U':theRest) = [(BLEU, theRest)]
-  readsPrec _ ('G':'L':'E':'U':theRest) = [(GLEU, theRest)]
-  readsPrec _ ('W':'E':'R':theRest) = [(WER, theRest)]
-  readsPrec _ ('A':'c':'c':'u':'r':'a':'c':'y':theRest) = [(Accuracy, theRest)]
-  readsPrec _ ('C':'l':'i':'p':'p':'E':'U':theRest) = [(ClippEU, theRest)]
-  readsPrec _ ('N':'M':'I':theRest) = [(NMI, theRest)]
-  readsPrec p ('F':theRest) = case readsPrec p theRest of
-    [(beta, theRest)] -> [(FMeasure beta, theRest)]
-    _ -> []
-  readsPrec p ('M':'a':'c':'r':'o':'-':'F':theRest) = case readsPrec p theRest of
-    [(beta, theRest)] -> [(MacroFMeasure beta, theRest)]
-    _ -> []
-  readsPrec p ('M':'u':'l':'t':'i':'L':'a':'b':'e':'l':'-':'F':theRest) = case readsPrec p theRest of
-    [(beta, theRest)] -> [(MultiLabelFMeasure beta, theRest)]
-    _ -> []
-  readsPrec p ('S':'o':'f':'t':'-':'F':theRest) = case readsPrec p theRest of
-    [(beta, theRest)] -> [(SoftFMeasure beta, theRest)]
-    _ -> []
-  readsPrec p ('P':'r':'o':'b':'a':'b':'i':'l':'i':'s':'t':'i':'c':'-':'S':'o':'f':'t':'-':'F':theRest) = case readsPrec p theRest of
-    [(beta, theRest)] -> [(ProbabilisticSoftFMeasure beta, theRest)]
-    _ -> []
-  readsPrec p ('L':'o':'g':'L':'o':'s':'s':'H':'a':'s':'h':'e':'d':theRest) = case readsPrec p theRest of
-    [(nbOfBits, theRest)] -> [(LogLossHashed nbOfBits, theRest)]
-    _ -> [(LogLossHashed defaultLogLossHashedSize, theRest)]
-  readsPrec p ('L':'i':'k':'e':'l':'i':'h':'o':'o':'d':'H':'a':'s':'h':'e':'d':theRest) = case readsPrec p theRest of
-    [(nbOfBits, theRest)] -> [(LikelihoodHashed nbOfBits, theRest)]
-    _ -> [(LikelihoodHashed defaultLogLossHashedSize, theRest)]
-  readsPrec _ ('L':'o':'g':'L':'o':'s':'s':theRest) = [(LogLoss, theRest)]
-  readsPrec _ ('L':'i':'k':'e':'l':'i':'h':'o':'o':'d':theRest) = [(Likelihood, theRest)]
-  readsPrec p ('C':'h':'a':'r':'M':'a':'t':'c':'h':theRest) = [(CharMatch, theRest)]
-  readsPrec _ ('M':'A':'P':theRest) = [(MAP, theRest)]
-  readsPrec _ ('B':'I':'O':'-':'F':'1':'-':'L':'a':'b':'e':'l':'s':theRest) = [(BIOF1Labels, theRest)]
-  readsPrec _ ('B':'I':'O':'-':'F':'1':theRest) = [(BIOF1, theRest)]
-  readsPrec _ ('T':'o':'k':'e':'n':'A':'c':'c':'u':'r':'a':'c':'y':theRest) = [(TokenAccuracy, theRest)]
-  readsPrec _ ('M':'A':'E':theRest) = [(MAE, theRest)]
-  readsPrec _ ('S':'M':'A':'P':'E':theRest) = [(SMAPE, theRest)]
-  readsPrec _ ('M':'u':'l':'t':'i':'L':'a':'b':'e':'l':'-':'L':'o':'g':'L':'o':'s':'s':theRest) = [(MultiLabelLogLoss, theRest)]
-  readsPrec _ ('M':'u':'l':'t':'i':'L':'a':'b':'e':'l':'-':'L':'i':'k':'e':'l':'i':'h':'o':'o':'d':theRest) = [(MultiLabelLikelihood, theRest)]
-
-
-
-data MetricOrdering = TheLowerTheBetter | TheHigherTheBetter
-
--- | Returns what is preferred for a given metric: high values or low values.
-getMetricOrdering :: Metric -> MetricOrdering
-getMetricOrdering RMSE     = TheLowerTheBetter
-getMetricOrdering MSE      = TheLowerTheBetter
-getMetricOrdering Pearson  = TheHigherTheBetter
-getMetricOrdering Spearman = TheHigherTheBetter
-getMetricOrdering BLEU     = TheHigherTheBetter
-getMetricOrdering GLEU     = TheHigherTheBetter
-getMetricOrdering WER      = TheLowerTheBetter
-getMetricOrdering Accuracy = TheHigherTheBetter
-getMetricOrdering ClippEU  = TheHigherTheBetter
-getMetricOrdering (FMeasure _) = TheHigherTheBetter
-getMetricOrdering (MacroFMeasure _) = TheHigherTheBetter
-getMetricOrdering (SoftFMeasure _) = TheHigherTheBetter
-getMetricOrdering (ProbabilisticSoftFMeasure _) = TheHigherTheBetter
-getMetricOrdering NMI = TheHigherTheBetter
-getMetricOrdering (LogLossHashed _) = TheLowerTheBetter
-getMetricOrdering (LikelihoodHashed _) = TheHigherTheBetter
-getMetricOrdering CharMatch = TheHigherTheBetter
-getMetricOrdering MAP = TheHigherTheBetter
-getMetricOrdering LogLoss = TheLowerTheBetter
-getMetricOrdering Likelihood = TheHigherTheBetter
-getMetricOrdering BIOF1 = TheHigherTheBetter
-getMetricOrdering BIOF1Labels = TheHigherTheBetter
-getMetricOrdering TokenAccuracy = TheHigherTheBetter
-getMetricOrdering MAE = TheLowerTheBetter
-getMetricOrdering SMAPE = TheLowerTheBetter
-getMetricOrdering (MultiLabelFMeasure _) = TheHigherTheBetter
-getMetricOrdering MultiLabelLogLoss = TheLowerTheBetter
-getMetricOrdering MultiLabelLikelihood = TheHigherTheBetter
 
 isBetterOrEqual :: Metric -> MetricValue -> MetricValue -> Bool
 isBetterOrEqual metric valA valB = not (isBetter metric valB valA)

src/GEval/CreateChallenge.hs

@@ -4,7 +4,8 @@ module GEval.CreateChallenge
        (createChallenge)
        where
 
-import GEval.Core
+import GEval.Metric
+import GEval.Core (GEvalSpecification(..), GEvalException(..), configFileName, gesMainMetric, defaultTestName)
 import GEval.Submit (tokenFileName)
 import qualified System.Directory as D
 import Control.Conditional (whenM)

src/GEval/Validation.hs

@@ -4,7 +4,8 @@ module GEval.Validation
     ( validationChallenge
     ) where
 
-import GEval.Core
+import GEval.Metric
+import GEval.Core (GEvalSpecification(..), GEvalException(..), somethingWrongWithFilesMessage, isEmptyFile)
 import qualified System.Directory as D
 
 import System.FilePath.Find as SFF