82 lines
3.3 KiB
Haskell
82 lines
3.3 KiB
Haskell
import Test.Hspec
|
|
|
|
import GEval.Core
|
|
import GEval.OptionsParser
|
|
import GEval.BLEU
|
|
import Options.Applicative
|
|
import qualified Test.HUnit as HU
|
|
|
|
main :: IO ()
|
|
main = hspec $ do
|
|
describe "root mean square error" $ do
|
|
it "simple test" $ do
|
|
geval (defaultGEvalSpecification {gesExpectedDirectory=Just "test/rmse-simple/rmse-simple", gesOutDirectory="test/rmse-simple/rmse-simple-solution"}) `shouldReturnAlmost` 0.64549722436790
|
|
describe "mean square error" $ do
|
|
it "simple test with arguments" $
|
|
runGEvalTest "mse-simple" `shouldReturnAlmost` 0.4166666666666667
|
|
describe "BLEU" $ do
|
|
it "trivial example from Wikipedia" $
|
|
runGEvalTest "bleu-trivial" `shouldReturnAlmost` 0.0
|
|
it "complex example" $
|
|
runGEvalTest "bleu-complex" `shouldReturnAlmost` 0.6211
|
|
it "perfect translation" $
|
|
runGEvalTest "bleu-perfect" `shouldReturnAlmost` 1.0000
|
|
describe "Accuracy" $ do
|
|
it "simple example" $
|
|
runGEvalTest "accuracy-simple" `shouldReturnAlmost` 0.6
|
|
describe "precision count" $ do
|
|
it "simple test" $ do
|
|
precisionCount [["Alice", "has", "a", "cat" ]] ["Ala", "has", "cat"] `shouldBe` 2
|
|
it "none found" $ do
|
|
precisionCount [["Alice", "has", "a", "cat" ]] ["for", "bar", "baz"] `shouldBe` 0
|
|
it "multiple values" $ do
|
|
precisionCount [["bar", "bar", "bar", "bar", "foo", "xyz", "foo"]] ["foo", "bar", "foo", "baz", "bar", "foo"] `shouldBe` 4
|
|
it "multiple refs" $ do
|
|
precisionCount [["foo", "baz"], ["bar"], ["baz", "xyz"]] ["foo", "bar", "foo"] `shouldBe` 2
|
|
describe "reading options" $ do
|
|
it "can get the metric" $ do
|
|
extractMetric "bleu-complex" `shouldReturn` (Just BLEU)
|
|
describe "error handling" $ do
|
|
it "too few lines are handled" $ do
|
|
runGEvalTest "error-too-few-lines" `shouldThrow` (== TooFewLines)
|
|
it "too many lines are handled" $ do
|
|
runGEvalTest "error-too-many-lines" `shouldThrow` (== TooManyLines)
|
|
it "empty output is handled" $ do
|
|
runGEvalTest "empty-output" `shouldThrow` (== EmptyOutput)
|
|
it "unexpected data is handled" $
|
|
runGEvalTest "unexpected-data" `shouldThrow` (== UnexpectedData "input does not start with a digit")
|
|
it "unwanted data is handled" $
|
|
runGEvalTest "unwanted-data" `shouldThrow` (== UnexpectedData "number expected")
|
|
|
|
|
|
extractVal :: (Either (ParserResult GEvalOptions) (Maybe MetricValue)) -> IO MetricValue
|
|
extractVal (Right (Just val)) = return val
|
|
|
|
runGEvalTest testName = (runGEval [
|
|
"--expected-directory",
|
|
"test/" ++ testName ++ "/" ++ testName,
|
|
"--out-directory",
|
|
"test/" ++ testName ++ "/" ++ testName ++ "-solution"]) >>= extractVal
|
|
|
|
extractMetric :: String -> IO (Maybe Metric)
|
|
extractMetric testName = do
|
|
result <- getOptions ["--expected-directory", "test/" ++ testName ++ "/" ++ testName]
|
|
return $ case result of
|
|
Left _ -> Nothing
|
|
Right opts -> Just $ gesMetric $ geoSpec opts
|
|
|
|
class AEq a where
|
|
(=~) :: a -> a -> Bool
|
|
|
|
instance AEq Double where
|
|
x =~ y = abs ( x - y ) < (1.0e-4 :: Double)
|
|
|
|
(@=~?) :: (Show a, AEq a) => a -> a -> HU.Assertion
|
|
(@=~?) expected actual = expected =~ actual HU.@? assertionMsg
|
|
where
|
|
assertionMsg = "Expected : " ++ show expected ++
|
|
"\nActual : " ++ show actual
|
|
|
|
shouldReturnAlmost :: (AEq a, Show a, Eq a) => IO a -> a -> Expectation
|
|
shouldReturnAlmost action expected = action >>= (@=~? expected)
|