generating sample CharMatch challenge, CharMatch is F0.5 now
This commit is contained in:
parent
72dbf33b8d
commit
54c899ddfc
@ -5,7 +5,7 @@ module GEval.CharMatch
|
|||||||
import Text.EditDistance
|
import Text.EditDistance
|
||||||
|
|
||||||
charMatchBeta :: Double
|
charMatchBeta :: Double
|
||||||
charMatchBeta = 1.0
|
charMatchBeta = 0.5
|
||||||
|
|
||||||
getCharMatchCount :: String -> String -> String -> (Int, Int, Int)
|
getCharMatchCount :: String -> String -> String -> (Int, Int, Int)
|
||||||
getCharMatchCount input expected output = (correctionsDone, expectedCorrections, distanceToInput)
|
getCharMatchCount input expected output = (correctionsDone, expectedCorrections, distanceToInput)
|
||||||
|
@ -114,6 +114,33 @@ In an input file, left and right contexts (TAB-separated) are given.
|
|||||||
In an expected file, the word to be guessed is given.
|
In an expected file, the word to be guessed is given.
|
||||||
|] ++ (commonReadmeMDContents testName)
|
|] ++ (commonReadmeMDContents testName)
|
||||||
|
|
||||||
|
readmeMDContents CharMatch testName = [i|
|
||||||
|
GEval sample machine challenge for text transformation
|
||||||
|
======================================================
|
||||||
|
|
||||||
|
Transform from British English into American English.
|
||||||
|
|
||||||
|
This is a sample challenge for Gonito framework for CharMarch metric. Replace it with
|
||||||
|
the description of your challenge.
|
||||||
|
|
||||||
|
CharMatch is F0.5-score (precision more important than recall)
|
||||||
|
for expected corrections (i.e. changes between the input text
|
||||||
|
and the expected output).
|
||||||
|
|
||||||
|
Directory structure
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
* `README.md` — this file
|
||||||
|
* `config.txt` — configuration file
|
||||||
|
* `train/` — directory with training data
|
||||||
|
* `train/train.tsv` — just some test in American English
|
||||||
|
* `dev-0/` — directory with dev (test) data
|
||||||
|
* `dev-0/in.tsv` — British input text for the dev set
|
||||||
|
* `dev-0/expected.tsv` — American reference text for the dev set
|
||||||
|
* `${testName}` — directory with test data
|
||||||
|
* `${testName}/in.tsv` — British input data for the test set
|
||||||
|
* `${testName}/expected.tsv` — American reference text for the test set
|
||||||
|
|]
|
||||||
|
|
||||||
readmeMDContents _ testName = [i|
|
readmeMDContents _ testName = [i|
|
||||||
GEval sample challenge
|
GEval sample challenge
|
||||||
@ -188,6 +215,11 @@ Basia ma psa
|
|||||||
Nie kupujemy kota w worku
|
Nie kupujemy kota w worku
|
||||||
Czesia ma kota
|
Czesia ma kota
|
||||||
|]
|
|]
|
||||||
|
trainContents CharMatch = [hereLit|Camptown ladies sing dis song, Doo-dah! doo-dah!
|
||||||
|
Camptown race-track five miles long, Oh, doo-dah day!
|
||||||
|
I come down dah wid my hat caved in, Doo-dah! doo-dah!
|
||||||
|
I go back home wid a pocket full of tin, Oh, doo-dah day!
|
||||||
|
|]
|
||||||
trainContents _ = [hereLit|0.06 0.39 0 0.206
|
trainContents _ = [hereLit|0.06 0.39 0 0.206
|
||||||
1.00 1.00 1 0.017
|
1.00 1.00 1 0.017
|
||||||
317.8 5.20 67 0.048
|
317.8 5.20 67 0.048
|
||||||
@ -208,10 +240,13 @@ When the going gets tough, the tough get going.
|
|||||||
devInContents (FMeasure _) = [hereLit|b b W 29520 779 -28 -32 a 0 0 0 0 0 0 0 0 0 0
|
devInContents (FMeasure _) = [hereLit|b b W 29520 779 -28 -32 a 0 0 0 0 0 0 0 0 0 0
|
||||||
b b W 55200 1259 35 9 a 1 0 1 0 0 0 0 0 4000 4000
|
b b W 55200 1259 35 9 a 1 0 1 0 0 0 0 0 4000 4000
|
||||||
|]
|
|]
|
||||||
|
|
||||||
devInContents (LogLossHashed _) = [hereLit|Nie kupuj w worku
|
devInContents (LogLossHashed _) = [hereLit|Nie kupuj w worku
|
||||||
Ona psa
|
Ona psa
|
||||||
|]
|
|]
|
||||||
|
devInContents CharMatch = [hereLit|honour to organise
|
||||||
|
nothing to change
|
||||||
|
time traveller
|
||||||
|
|]
|
||||||
devInContents _ = [hereLit|0.72 0 0.007
|
devInContents _ = [hereLit|0.72 0 0.007
|
||||||
9.54 62 0.054
|
9.54 62 0.054
|
||||||
|]
|
|]
|
||||||
@ -233,6 +268,10 @@ en
|
|||||||
devExpectedContents (LogLossHashed _) = [hereLit|kota
|
devExpectedContents (LogLossHashed _) = [hereLit|kota
|
||||||
ma
|
ma
|
||||||
|]
|
|]
|
||||||
|
devExpectedContents CharMatch = [hereLit|honor to organize
|
||||||
|
nothing to change
|
||||||
|
time traveler
|
||||||
|
|]
|
||||||
devExpectedContents _ = [hereLit|0.82
|
devExpectedContents _ = [hereLit|0.82
|
||||||
95.2
|
95.2
|
||||||
|]
|
|]
|
||||||
@ -256,6 +295,10 @@ Kwiecień plecień, bo przeplata trochę zimy, trochę lata.
|
|||||||
testInContents (LogLossHashed _) = [hereLit|Ala ma
|
testInContents (LogLossHashed _) = [hereLit|Ala ma
|
||||||
Ona ma kota worku
|
Ona ma kota worku
|
||||||
|]
|
|]
|
||||||
|
testInContents CharMatch = [hereLit|paralysed by practise
|
||||||
|
recognise
|
||||||
|
nothing
|
||||||
|
|]
|
||||||
testInContents _ = [hereLit|1.52 2 0.093
|
testInContents _ = [hereLit|1.52 2 0.093
|
||||||
30.06 14 0.009
|
30.06 14 0.009
|
||||||
|]
|
|]
|
||||||
@ -279,6 +322,10 @@ pl
|
|||||||
testExpectedContents (LogLossHashed _) = [hereLit|ma
|
testExpectedContents (LogLossHashed _) = [hereLit|ma
|
||||||
w
|
w
|
||||||
|]
|
|]
|
||||||
|
testExpectedContents CharMatch = [hereLit|paralyzed by practice
|
||||||
|
recognize
|
||||||
|
nothing
|
||||||
|
|]
|
||||||
testExpectedContents _ = [hereLit|0.11
|
testExpectedContents _ = [hereLit|0.11
|
||||||
17.2
|
17.2
|
||||||
|]
|
|]
|
||||||
|
@ -80,7 +80,7 @@ metricReader = option auto
|
|||||||
<> value defaultMetric
|
<> value defaultMetric
|
||||||
<> showDefault
|
<> showDefault
|
||||||
<> metavar "METRIC"
|
<> metavar "METRIC"
|
||||||
<> help "Metric to be used - RMSE, MSE, Accuracy, F-measure (specify as F1, F2, F0.25, etc.), BLEU, NMI, ClippEU or LogLossHashed" )
|
<> help "Metric to be used - RMSE, MSE, Accuracy, F-measure (specify as F1, F2, F0.25, etc.), BLEU, NMI, ClippEU, LogLossHashed or CharMatch" )
|
||||||
|
|
||||||
runGEval :: [String] -> IO (Either (ParserResult GEvalOptions) (Maybe MetricValue))
|
runGEval :: [String] -> IO (Either (ParserResult GEvalOptions) (Maybe MetricValue))
|
||||||
runGEval args = do
|
runGEval args = do
|
||||||
|
@ -157,11 +157,11 @@ main = hspec $ do
|
|||||||
levenshteinDistance defaultEditCosts "źdźbło" "źdźcło" `shouldBe` 1
|
levenshteinDistance defaultEditCosts "źdźbło" "źdźcło" `shouldBe` 1
|
||||||
describe "CharMatch" $ do
|
describe "CharMatch" $ do
|
||||||
it "simple test" $ do
|
it "simple test" $ do
|
||||||
runGEvalTest "charmatch-simple" `shouldReturnAlmost` 0.4
|
runGEvalTest "charmatch-simple" `shouldReturnAlmost` 0.3571428571428571
|
||||||
it "perfect solution" $ do
|
it "perfect solution" $ do
|
||||||
runGEvalTest "charmatch-perfect" `shouldReturnAlmost` 1.0
|
runGEvalTest "charmatch-perfect" `shouldReturnAlmost` 1.0
|
||||||
it "more complex test" $ do
|
it "more complex test" $ do
|
||||||
runGEvalTest "charmatch-complex" `shouldReturnAlmost` 0.25
|
runGEvalTest "charmatch-complex" `shouldReturnAlmost` 0.1923076923076923
|
||||||
it "broken test without input" $ do
|
it "broken test without input" $ do
|
||||||
runGEvalTest "charmatch-no-input" `shouldThrow` (== NoInputFile "test/charmatch-no-input/charmatch-no-input/test-A/in.tsv")
|
runGEvalTest "charmatch-no-input" `shouldThrow` (== NoInputFile "test/charmatch-no-input/charmatch-no-input/test-A/in.tsv")
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user