handle multiple metrics for the same test

Handler/CreateChallenge.hs

@@ -144,18 +144,19 @@ checkTestDir chan challengeId challenge commit testDir = do
         "--expected-directory", challengeRepoDir,
         "--test-name", takeFileName testDir]
       case optionsParsingResult of
-       Left evalException -> do
+       Left _ -> do
          err chan "Cannot read metric"
          return ()
        Right opts -> do
-         _ <- runDB $ insert $ Test {
+         _ <- runDB $ mapM (\(priority, metric) -> insert $ Test {
            testChallenge=challengeId,
-           testMetric=gesMetric $ geoSpec opts,
+           testMetric=metric,
            testName=T.pack $ takeFileName testDir,
            testChecksum=(SHA1 checksum),
            testCommit=commit,
            testActive=True,
-           testPrecision=gesPrecision $ geoSpec opts}
+           testPrecision=gesPrecision $ geoSpec opts,
+           testPriority=Just priority}) $ zip [1..] (gesMetrics $ geoSpec opts)
          return ()
     else
       msg chan $ concat ["Test dir ", (T.pack testDir), " does not have expected results."]

Handler/Shared.hs

@@ -312,8 +312,14 @@ enableTriggerToken userId Nothing = do
   token <- newToken
   runDB $ update userId [UserTriggerToken =. Just token]
 
+thenCmp :: Ordering -> Ordering -> Ordering
+thenCmp EQ o2 = o2
+thenCmp o1 _  = o1
+
 getMainTest :: [Entity Test] -> Entity Test
-getMainTest tests = DL.maximumBy (\(Entity _ a) (Entity _ b) -> ((testName a) `compare` (testName b))) tests
+getMainTest tests = DL.maximumBy (\(Entity _ a) (Entity _ b) -> ( ((testName a) `compare` (testName b))
+                                                                               `thenCmp`
+                                                                 ((fromMaybe 9999 $ testPriority b) `compare` (fromMaybe 9999 $ testPriority a)) ) ) tests
 
 formatFullScore :: Maybe Evaluation -> Text
 formatFullScore (Just evaluation) = fromMaybe "???" (T.pack <$> show <$> evaluationScore evaluation)

Handler/ShowChallenge.hs

@@ -314,30 +314,33 @@ checkOrInsertEvaluation repoDir chan out = do
     Nothing -> do
       msg chan $ "Start evaluation..."
       challengeDir <- getRepoDir $ challengePrivateRepo challenge
-      resultOrException <- liftIO $ rawEval challengeDir repoDir (testName test)
+      resultOrException <- liftIO $ rawEval challengeDir (testMetric test) repoDir (testName test)
       case resultOrException of
-        Right (Left parseResult) -> do
+        Right (Left _) -> do
           err chan "Cannot parse options, check the challenge repo"
-        Right (Right (opts, Just result)) -> do
+        Right (Right (_, Just [result])) -> do
           msg chan $ concat [ "Evaluated! Score ", (T.pack $ show result) ]
           time <- liftIO getCurrentTime
-          runDB $ insert $ Evaluation {
+          _ <- runDB $ insert $ Evaluation {
             evaluationTest=outTest out,
             evaluationChecksum=outChecksum out,
             evaluationScore=Just result,
             evaluationErrorMessage=Nothing,
             evaluationStamp=time }
           msg chan "Evaluation done"
+        Right (Right (_, Just _)) -> do
+          err chan "Unexpected multiple results (???)"
         Right (Right (_, Nothing)) -> do
           err chan "Error during the evaluation"
         Left exception -> do
           err chan $ "Evaluation failed: " ++ (T.pack $ show exception)
 
-rawEval :: FilePath -> FilePath -> Text -> IO (Either GEvalException (Either (ParserResult GEvalOptions) (GEvalOptions, Maybe MetricValue)))
+rawEval :: FilePath -> Metric -> FilePath -> Text -> IO (Either GEvalException (Either (ParserResult GEvalOptions) (GEvalOptions, Maybe [MetricValue])))
-rawEval challengeDir repoDir name = Import.try (runGEvalGetOptions [
+rawEval challengeDir metric repoDir name = Import.try (runGEvalGetOptions [
-                                    "--expected-directory", challengeDir,
+                                                          "--metric", (show metric),
-                                    "--out-directory", repoDir,
+                                                          "--expected-directory", challengeDir,
-                                    "--test-name", (T.unpack name)])
+                                                          "--out-directory", repoDir,
+                                                          "--test-name", (T.unpack name)])
 
 getSubmissionRepo :: Key Challenge -> RepoSpec -> Channel -> Handler (Maybe (Key Repo))
 getSubmissionRepo challengeId repoSpec chan = do

config/models

@@ -47,7 +47,8 @@ Test
     commit SHA1
     active Bool default=True
     precision Int Maybe
-    UniqueChallengeNameChecksum challenge name checksum
+    priority Int Maybe
+    UniqueChallengeNameMetricChecksum challenge name metric checksum
 Submission
     repo RepoId
     commit SHA1