BLEU done

README.md

@@ -0,0 +1,17 @@
+# GEval
+
+GEval is a library (and a stand-alone tool) for evaluating the results
+of solutions to machine learning challenges as defined in the Gonito
+platform.
+
+## Installing
+
+You need [Haskell Stack](https://github.com/commercialhaskell/stack),
+then install with:
+
+    git clone https://github.com/filipg/geval
+    cd geval
+    stack setup
+    stack install
+
+## Directory structure of a Gonito challenge

src/GEval/BLEU.hs

@@ -5,6 +5,10 @@ module GEval.BLEU
 import qualified Data.MultiSet as MS
 import Data.List (minimumBy, zip, zip3, zip4)
 
+import Debug.Trace
+
+
+
 bleuStep :: Ord a => [[a]] -> [a] -> (Int, Int, Int, Int,  Int,  Int, Int, Int, Int)
 bleuStep refs trans = (prec1, prec2, prec3, prec4,  closestLen, len1, len2, len3, len4)
   where prec1 = precisionCountForNgrams id

src/GEval/Core.hs

@@ -131,13 +131,17 @@ gevalCore' :: Metric -> String -> String -> IO (MetricValue)
 gevalCore' MSE = gevalCore'' outParser outParser itemError averageC id
   where outParser = getValue . TR.double
 
-gevalCore' BLEU = gevalCore'' (DLS.splitOn "\t" . unpack) unpack bleuCombine bleuAgg bleuFinal
+gevalCore' BLEU = gevalCore'' (Prelude.map Prelude.words . DLS.splitOn "\t" . unpack) (Prelude.words . unpack) bleuCombine bleuAgg bleuFinal
-  where bleuFinal (p1, p2, p3, p4, cl, l1, l2, l3, l4) = p1 /. l1
+  where bleuFinal (p1, p2, p3, p4, rl, l1, l2, l3, l4) = ((p1 /. l1) * (p2 /. l2) * (p3 /. l3) * (p4 /. l4)) ** 0.25 * (brevityPenalty l1 rl)
         bleuCombine (refs, sen) = bleuStep refs sen
         bleuAgg = CC.foldl bleuFuse (0, 0, 0, 0, 0,  0, 0, 0, 0)
         bleuFuse (a1, a2, a3, a4, a5, a6, a7, a8, a9) (b1, b2, b3, b4, b5, b6, b7, b8, b9) = (a1+b1, a2+b2, a3+b3, a4+b4, a5+b5, a6+b6, a7+b7, a8+b8, a9+b9)
+        brevityPenalty c r
+          | c >= r = 1.0
+          | otherwise = exp (1.0 - (r /. c))
 
 (/.) :: Int -> Int -> Double
+x /. 0 = 1.0
 x /. y = (fromIntegral x) / (fromIntegral y)
 
 gevalCore'' :: (Text -> a) -> (Text -> b) -> ((a, b) -> c) -> (Sink c (ResourceT IO) d) -> (d -> Double ) -> String -> String -> IO (MetricValue)

test/Spec.hs

@@ -23,6 +23,16 @@ main = hspec $ do
                   "test/bleu-trivial/bleu-trivial",
                   "--out-directory",
                   "test/bleu-trivial/bleu-trivial-solution"]) >>= extractVal) `shouldReturnAlmost` 0.0
+    it "complex example" $ do
+      ((runGEval ["--expected-directory",
+                  "test/bleu-complex/bleu-complex",
+                  "--out-directory",
+                  "test/bleu-complex/bleu-complex-solution"]) >>= extractVal) `shouldReturnAlmost` 0.6211
+    it "perfect translation" $ do
+      ((runGEval ["--expected-directory",
+                  "test/bleu-perfect/bleu-perfect",
+                  "--out-directory",
+                  "test/bleu-perfect/bleu-perfect-solution"]) >>= extractVal) `shouldReturnAlmost` 1.0000
   describe "precision count" $ do
     it "simple test" $ do
       precisionCount [["Alice", "has", "a", "cat" ]] ["Ala", "has", "cat"] `shouldBe` 2
@@ -40,7 +50,7 @@ class AEq a where
     (=~) :: a -> a -> Bool
 
 instance AEq Double where
-    x =~ y = abs ( x - y ) < (1.0e-8 :: Double)
+    x =~ y = abs ( x - y ) < (1.0e-4 :: Double)
 
 (@=~?) :: (Show a, AEq a) => a -> a -> HU.Assertion
 (@=~?) expected actual  = expected =~ actual HU.@? assertionMsg

test/bleu-complex/bleu-complex-solution/test-A/out.tsv

@@ -0,0 +1,6 @@
+Ala has a white cat
+It is a trap
+All your base belong to us
+bar
+expected result
+thrash

test/bleu-complex/bleu-complex/config.txt

@@ -0,0 +1 @@
+--metric BLEU

test/bleu-complex/bleu-complex/test-A/expected.tsv

@@ -0,0 +1,6 @@
+Alice has a white cat	Ala has a whitecat
+It is a trap
+All your base are belong to us
+foo	bar	baz
+the expected result	result expected
+foo bar	foo bar baz baq

test/bleu-perfect/bleu-perfect-solution/test-A/out.tsv

@@ -0,0 +1,4 @@
+Litwo, Ojczyzno moja! ty jesteś jak zdrowie;
+Ile cię trzeba cenić, ten tylko się dowie,
+Kto cię stracił. Dziś piękność twą w całej ozdobie
+Widzę i opisuję, bo tęsknię po tobie.

test/bleu-perfect/bleu-perfect/config.txt

@@ -0,0 +1 @@
+--metric BLEU

test/bleu-perfect/bleu-perfect/test-A/expected.tsv

@@ -0,0 +1,4 @@
+Litwo, Ojczyzno moja! ty jesteś jak zdrowie;
+Ile cię trzeba cenić, ten tylko się dowie,
+Kto cię stracił. Dziś piękność twą w całej ozdobie
+Widzę i opisuję, bo tęsknię po tobie.