diff --git a/src/GEval/Metric.hs b/src/GEval/Metric.hs new file mode 100644 index 0000000..4550a92 --- /dev/null +++ b/src/GEval/Metric.hs @@ -0,0 +1,184 @@ +module GEval.Metric + (Metric(..), + MetricOrdering(..), + defaultLogLossHashedSize, + getMetricOrdering, + listOfAvailableMetrics) + where + +import Data.Word + +-- here metrics and their basic properties are listed, +-- the evaluation procedures are defined in GEval.Core + +-- | evaluation metric +data Metric = RMSE | MSE | Pearson | Spearman | BLEU | GLEU | WER | Accuracy | ClippEU + | FMeasure Double | MacroFMeasure Double | NMI + | LogLossHashed Word32 | CharMatch | MAP | LogLoss | Likelihood + | BIOF1 | BIOF1Labels | TokenAccuracy | LikelihoodHashed Word32 | MAE | SMAPE | MultiLabelFMeasure Double + | MultiLabelLogLoss | MultiLabelLikelihood + | SoftFMeasure Double | ProbabilisticSoftFMeasure Double + deriving (Eq) + +-- | the list of available metrics, to be shown to the user or to be +-- | used for tests +listOfAvailableMetrics :: [Metric] +listOfAvailableMetrics = [RMSE, + MSE, + MAE, + SMAPE, + Pearson, + Spearman, + Accuracy, + LogLoss, + Likelihood, + FMeasure 1.0, + FMeasure 2.0, + FMeasure 0.25, + MacroFMeasure 1.0, + MacroFMeasure 2.0, + MacroFMeasure 0.25, + MultiLabelFMeasure 1.0, + MultiLabelFMeasure 2.0, + MultiLabelFMeasure 0.25, + MultiLabelLikelihood, + MAP, + BLEU, + GLEU, + WER, + NMI, + ClippEU, + LogLossHashed defaultLogLossHashedSize, + LikelihoodHashed defaultLogLossHashedSize, + BIOF1, + BIOF1Labels, + TokenAccuracy, + SoftFMeasure 1.0, + SoftFMeasure 2.0, + SoftFMeasure 0.25, + ProbabilisticSoftFMeasure 1.0, + ProbabilisticSoftFMeasure 2.0, + ProbabilisticSoftFMeasure 0.25, + CharMatch] + +instance Show Metric where + show RMSE = "RMSE" + show MSE = "MSE" + show Pearson = "Pearson" + show Spearman = "Spearman" + show BLEU = "BLEU" + show GLEU = "GLEU" + show WER = "WER" + show Accuracy = "Accuracy" + show ClippEU = "ClippEU" + show (FMeasure beta) = "F" ++ (show beta) + show (MacroFMeasure beta) = "Macro-F" ++ (show beta) + show (SoftFMeasure beta) = "Soft-F" ++ (show beta) + show (ProbabilisticSoftFMeasure beta) = "Probabilistic-Soft-F" ++ (show beta) + show NMI = "NMI" + show (LogLossHashed nbOfBits) = "LogLossHashed" ++ (if + nbOfBits == defaultLogLossHashedSize + then + "" + else + (show nbOfBits)) + show (LikelihoodHashed nbOfBits) = "LikelihoodHashed" ++ (if + nbOfBits == defaultLogLossHashedSize + then + "" + else + (show nbOfBits)) + show CharMatch = "CharMatch" + show MAP = "MAP" + show LogLoss = "LogLoss" + show Likelihood = "Likelihood" + show BIOF1 = "BIO-F1" + show BIOF1Labels = "BIO-F1-Labels" + show TokenAccuracy = "TokenAccuracy" + show MAE = "MAE" + show SMAPE = "SMAPE" + show (MultiLabelFMeasure beta) = "MultiLabel-F" ++ (show beta) + show MultiLabelLogLoss = "MultiLabel-Logloss" + show MultiLabelLikelihood = "MultiLabel-Likelihood" + +instance Read Metric where + readsPrec _ ('R':'M':'S':'E':theRest) = [(RMSE, theRest)] + readsPrec _ ('M':'S':'E':theRest) = [(MSE, theRest)] + readsPrec _ ('P':'e':'a':'r':'s':'o':'n':theRest) = [(Pearson, theRest)] + readsPrec _ ('S':'p':'e':'a':'r':'m':'a':'n':theRest) = [(Spearman, theRest)] + readsPrec _ ('B':'L':'E':'U':theRest) = [(BLEU, theRest)] + readsPrec _ ('G':'L':'E':'U':theRest) = [(GLEU, theRest)] + readsPrec _ ('W':'E':'R':theRest) = [(WER, theRest)] + readsPrec _ ('A':'c':'c':'u':'r':'a':'c':'y':theRest) = [(Accuracy, theRest)] + readsPrec _ ('C':'l':'i':'p':'p':'E':'U':theRest) = [(ClippEU, theRest)] + readsPrec _ ('N':'M':'I':theRest) = [(NMI, theRest)] + readsPrec p ('F':theRest) = case readsPrec p theRest of + [(beta, theRest)] -> [(FMeasure beta, theRest)] + _ -> [] + readsPrec p ('M':'a':'c':'r':'o':'-':'F':theRest) = case readsPrec p theRest of + [(beta, theRest)] -> [(MacroFMeasure beta, theRest)] + _ -> [] + readsPrec p ('M':'u':'l':'t':'i':'L':'a':'b':'e':'l':'-':'F':theRest) = case readsPrec p theRest of + [(beta, theRest)] -> [(MultiLabelFMeasure beta, theRest)] + _ -> [] + readsPrec p ('S':'o':'f':'t':'-':'F':theRest) = case readsPrec p theRest of + [(beta, theRest)] -> [(SoftFMeasure beta, theRest)] + _ -> [] + readsPrec p ('P':'r':'o':'b':'a':'b':'i':'l':'i':'s':'t':'i':'c':'-':'S':'o':'f':'t':'-':'F':theRest) = case readsPrec p theRest of + [(beta, theRest)] -> [(ProbabilisticSoftFMeasure beta, theRest)] + _ -> [] + readsPrec p ('L':'o':'g':'L':'o':'s':'s':'H':'a':'s':'h':'e':'d':theRest) = case readsPrec p theRest of + [(nbOfBits, theRest)] -> [(LogLossHashed nbOfBits, theRest)] + _ -> [(LogLossHashed defaultLogLossHashedSize, theRest)] + readsPrec p ('L':'i':'k':'e':'l':'i':'h':'o':'o':'d':'H':'a':'s':'h':'e':'d':theRest) = case readsPrec p theRest of + [(nbOfBits, theRest)] -> [(LikelihoodHashed nbOfBits, theRest)] + _ -> [(LikelihoodHashed defaultLogLossHashedSize, theRest)] + readsPrec _ ('L':'o':'g':'L':'o':'s':'s':theRest) = [(LogLoss, theRest)] + readsPrec _ ('L':'i':'k':'e':'l':'i':'h':'o':'o':'d':theRest) = [(Likelihood, theRest)] + readsPrec p ('C':'h':'a':'r':'M':'a':'t':'c':'h':theRest) = [(CharMatch, theRest)] + readsPrec _ ('M':'A':'P':theRest) = [(MAP, theRest)] + readsPrec _ ('B':'I':'O':'-':'F':'1':'-':'L':'a':'b':'e':'l':'s':theRest) = [(BIOF1Labels, theRest)] + readsPrec _ ('B':'I':'O':'-':'F':'1':theRest) = [(BIOF1, theRest)] + readsPrec _ ('T':'o':'k':'e':'n':'A':'c':'c':'u':'r':'a':'c':'y':theRest) = [(TokenAccuracy, theRest)] + readsPrec _ ('M':'A':'E':theRest) = [(MAE, theRest)] + readsPrec _ ('S':'M':'A':'P':'E':theRest) = [(SMAPE, theRest)] + readsPrec _ ('M':'u':'l':'t':'i':'L':'a':'b':'e':'l':'-':'L':'o':'g':'L':'o':'s':'s':theRest) = [(MultiLabelLogLoss, theRest)] + readsPrec _ ('M':'u':'l':'t':'i':'L':'a':'b':'e':'l':'-':'L':'i':'k':'e':'l':'i':'h':'o':'o':'d':theRest) = [(MultiLabelLikelihood, theRest)] + + + +data MetricOrdering = TheLowerTheBetter | TheHigherTheBetter + +-- | Returns what is preferred for a given metric: high values or low values. +getMetricOrdering :: Metric -> MetricOrdering +getMetricOrdering RMSE = TheLowerTheBetter +getMetricOrdering MSE = TheLowerTheBetter +getMetricOrdering Pearson = TheHigherTheBetter +getMetricOrdering Spearman = TheHigherTheBetter +getMetricOrdering BLEU = TheHigherTheBetter +getMetricOrdering GLEU = TheHigherTheBetter +getMetricOrdering WER = TheLowerTheBetter +getMetricOrdering Accuracy = TheHigherTheBetter +getMetricOrdering ClippEU = TheHigherTheBetter +getMetricOrdering (FMeasure _) = TheHigherTheBetter +getMetricOrdering (MacroFMeasure _) = TheHigherTheBetter +getMetricOrdering (SoftFMeasure _) = TheHigherTheBetter +getMetricOrdering (ProbabilisticSoftFMeasure _) = TheHigherTheBetter +getMetricOrdering NMI = TheHigherTheBetter +getMetricOrdering (LogLossHashed _) = TheLowerTheBetter +getMetricOrdering (LikelihoodHashed _) = TheHigherTheBetter +getMetricOrdering CharMatch = TheHigherTheBetter +getMetricOrdering MAP = TheHigherTheBetter +getMetricOrdering LogLoss = TheLowerTheBetter +getMetricOrdering Likelihood = TheHigherTheBetter +getMetricOrdering BIOF1 = TheHigherTheBetter +getMetricOrdering BIOF1Labels = TheHigherTheBetter +getMetricOrdering TokenAccuracy = TheHigherTheBetter +getMetricOrdering MAE = TheLowerTheBetter +getMetricOrdering SMAPE = TheLowerTheBetter +getMetricOrdering (MultiLabelFMeasure _) = TheHigherTheBetter +getMetricOrdering MultiLabelLogLoss = TheLowerTheBetter +getMetricOrdering MultiLabelLikelihood = TheHigherTheBetter + +defaultLogLossHashedSize :: Word32 +defaultLogLossHashedSize = 10