filipg/geval
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add --alt-metric option

Browse Source
This commit is contained in:
Filip Gralinski 2018-06-02 11:29:54 +02:00
parent 4768931221
commit d370e375a0
2 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/GEval/OptionsParser.hs
View File

@ -100,9 +100,13 @@ specParser = GEvalSpecification
<> showDefault <> showDefault
<> metavar "INPUT" <> metavar "INPUT"
<> help "The name of the file with the input (applicable only for some metrics)" ) <> help "The name of the file with the input (applicable only for some metrics)" )
<*> metricReader <*> ((flip fromMaybe) <$> altMetricReader <*> metricReader)
<*> optional precisionArgParser <*> optional precisionArgParser
sel :: Maybe Metric -> Metric -> Metric
sel Nothing m = m
sel (Just m) _ = m
metricReader :: Parser Metric metricReader :: Parser Metric
metricReader = option auto metricReader = option auto
( long "metric" ( long "metric"
@ -112,6 +116,13 @@ metricReader = option auto
<> metavar "METRIC" <> metavar "METRIC"
<> help "Metric to be used - RMSE, MSE, Accuracy, LogLoss, Likelihood, F-measure (specify as F1, F2, F0.25, etc.), MAP, BLEU, NMI, ClippEU, LogLossHashed, LikelihoodHashed, BIO-F1, BIO-F1-Labels or CharMatch" ) <> help "Metric to be used - RMSE, MSE, Accuracy, LogLoss, Likelihood, F-measure (specify as F1, F2, F0.25, etc.), MAP, BLEU, NMI, ClippEU, LogLossHashed, LikelihoodHashed, BIO-F1, BIO-F1-Labels or CharMatch" )
altMetricReader :: Parser (Maybe Metric)
altMetricReader = optional $ option auto
( long "alt-metric"
<> short 'a'
<> metavar "METRIC"
<> help "Alternative metric (overrides --metric option)" )
runGEval :: [String] -> IO (Either (ParserResult GEvalOptions) (Maybe MetricValue)) runGEval :: [String] -> IO (Either (ParserResult GEvalOptions) (Maybe MetricValue))
runGEval args = do runGEval args = do
ret <- runGEvalGetOptions args ret <- runGEvalGetOptions args

9
test/Spec.hs
View File

@ -290,6 +290,9 @@ main = hspec $ do
it "test sorting" $ do it "test sorting" $ do
results <- runLineByLineGeneralized FirstTheWorst sampleChallenge Data.Conduit.List.consume results <- runLineByLineGeneralized FirstTheWorst sampleChallenge Data.Conduit.List.consume
Prelude.head (Prelude.map (\(LineRecord inp _ _ _ _) -> inp) results) `shouldBe` "baq" Prelude.head (Prelude.map (\(LineRecord inp _ _ _ _) -> inp) results) `shouldBe` "baq"
describe "handle --alt-metric option" $ do
it "accuracy instead of likelihood" $ do
runGEvalTestExtraOptions ["--alt-metric", "Accuracy"] "likelihood-simple" `shouldReturnAlmost` 0.75
neverMatch :: Char -> Int -> Bool neverMatch :: Char -> Int -> Bool
@ -309,11 +312,13 @@ testMatchFun _ _ = False
extractVal :: (Either (ParserResult GEvalOptions) (Maybe MetricValue)) -> IO MetricValue extractVal :: (Either (ParserResult GEvalOptions) (Maybe MetricValue)) -> IO MetricValue
extractVal (Right (Just val)) = return val extractVal (Right (Just val)) = return val
runGEvalTest testName = (runGEval [ runGEvalTest = runGEvalTestExtraOptions []
runGEvalTestExtraOptions extraOptions testName = (runGEval ([
"--expected-directory", "--expected-directory",
"test/" ++ testName ++ "/" ++ testName, "test/" ++ testName ++ "/" ++ testName,
"--out-directory", "--out-directory",
"test/" ++ testName ++ "/" ++ testName ++ "-solution"]) >>= extractVal "test/" ++ testName ++ "/" ++ testName ++ "-solution"] ++ extraOptions)) >>= extractVal
extractMetric :: String -> IO (Maybe Metric) extractMetric :: String -> IO (Maybe Metric)
extractMetric testName = do extractMetric testName = do