filipg/geval
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

incorporate changes

Browse Source
This commit is contained in:
welp 2020-07-13 17:15:01 +02:00
parent 1010902baa
commit d31d4320a9
2 changed files with 38 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/GEval/Core.hs
View File

@ -113,6 +113,7 @@ import GEval.Annotation
import GEval.BlackBoxDebugging import GEval.BlackBoxDebugging
import Data.Conduit.Bootstrap import Data.Conduit.Bootstrap
import GEval.DataSource import GEval.DataSource
import GEval.MatchingSpecification
import qualified Data.HashMap.Strict as M import qualified Data.HashMap.Strict as M
import qualified Data.Vector as V import qualified Data.Vector as V
@ -522,7 +523,7 @@ singleLineAsLineSource (LineInFile sourceSpec lineNo line) itemDecoder preproces
-- some metrics are handled by Bootstrap due to legacy issues, -- some metrics are handled by Bootstrap due to legacy issues,
-- fix on the way -- fix on the way
handleBootstrap :: Metric -> Bool handleBootstrap :: Metric -> Bool
handleBootstrap (Mean (MultiLabelFMeasure _)) = True handleBootstrap (Mean (MultiLabelFMeasure _ _)) = True
handleBootstrap (Mean _) = False handleBootstrap (Mean _) = False
handleBootstrap CharMatch = False handleBootstrap CharMatch = False
handleBootstrap (LogLossHashed _) = False handleBootstrap (LogLossHashed _) = False
@ -567,13 +568,16 @@ gevalBootstrapOnSources :: (MonadIO m, MonadThrow m, MonadUnliftIO m) =>
-> m (MetricOutput) -- ^ metric values for the output against the expected output -> m (MetricOutput) -- ^ metric values for the output against the expected output
-- for the time being hardcoded -- for the time being hardcoded
gevalBootstrapOnSources numberOfSamples (Mean (MultiLabelFMeasure beta)) lsSpec = do gevalBootstrapOnSources numberOfSamples (Mean (MultiLabelFMeasure beta matchingSpec)) lsSpec = do
gevalRunPipeline parserSpec (trans step) finalPipeline context gevalRunPipeline parserSpec (trans step) finalPipeline context
where parserSpec = (ParserSpecWithoutInput (liftOp expParser) (liftOp outParser)) where parserSpec = (ParserSpecWithoutInput (liftOp expParser) (liftOp outParser))
context = fromSpecificationToWithoutInput lsSpec context = fromSpecificationToWithoutInput lsSpec
step = itemStep SAMultiLabelFMeasure step = case toSing matchingSpec of
expParser = expectedParser SAMultiLabelFMeasure SomeSing s -> itemStep (SAMultiLabelFMeasure s)
outParser = outputParser SAMultiLabelFMeasure expParser = case toSing matchingSpec of
SomeSing s -> expectedParser (SAMultiLabelFMeasure s)
outParser = case toSing matchingSpec of
SomeSing s -> outputParser (SAMultiLabelFMeasure s)
finalPipeline = fixer ( finalPipeline = fixer (
CL.map (fMeasureOnCounts beta) CL.map (fMeasureOnCounts beta)
.| (bootstrapC numberOfSamples .| (bootstrapC numberOfSamples
@ -630,10 +634,10 @@ gevalCoreOnSources (LogLossHashed nbOfBits) = helperLogLossHashed nbOfBits id
gevalCoreOnSources (LikelihoodHashed nbOfBits) = helperLogLossHashed nbOfBits logLossToLikehood gevalCoreOnSources (LikelihoodHashed nbOfBits) = helperLogLossHashed nbOfBits logLossToLikehood
gevalCoreOnSources (Mean (MultiLabelFMeasure beta)) gevalCoreOnSources (Mean (MultiLabelFMeasure beta matchingSpec))
= gevalCoreWithoutInputOnItemTargets (Right . intoWords) = gevalCoreWithoutInputOnItemTargets (Right . intoWords)
(Right . getWords) (Right . getWords)
((fMeasureOnCounts beta) . (getCounts (==))) ((fMeasureOnCounts beta) . (getWeightedCounts (getMatchingFunctionForString matchingSpec)))
averageC averageC
id id
noGraph noGraph
@ -661,12 +665,13 @@ gevalCoreOnSources (Mean WER)
gevalCoreOnSources (Mean _) = error $ "Mean/ meta-metric defined only for MultiLabel-F1 and WER for the time being" gevalCoreOnSources (Mean _) = error $ "Mean/ meta-metric defined only for MultiLabel-F1 and WER for the time being"
-- only MultiLabel-F1 handled for JSONs for the time being... -- only MultiLabel-F1 handled for JSONs for the time being...
gevalCoreOnSources (MultiLabelFMeasure beta) = gevalCoreWithoutInputOnItemTargets (Right . intoWords) gevalCoreOnSources (MultiLabelFMeasure beta matchingSpec) =
(Right . getWords) gevalCoreWithoutInputOnItemTargets (Right . intoWords)
(getCounts (==)) (Right . getWords)
countAgg (getWeightedCounts (getMatchingFunctionForString matchingSpec))
(fMeasureOnCounts beta) countAgg
noGraph (fMeasureOnCounts beta)
noGraph
where where
getWords (RawItemTarget t) = Prelude.map unpack $ selectByStandardThreshold $ parseIntoProbList t getWords (RawItemTarget t) = Prelude.map unpack $ selectByStandardThreshold $ parseIntoProbList t
getWords (PartiallyParsedItemTarget ts) = Prelude.map unpack ts getWords (PartiallyParsedItemTarget ts) = Prelude.map unpack ts
@ -848,12 +853,13 @@ gevalRunPipeline' parserSpec itemStep finalPipeline context = do
continueGEvalCalculations :: (MonadIO m) => continueGEvalCalculations :: forall m t . (MonadIO m) =>
SAMetric t SAMetric t
-> Metric -> Metric
-> ConduitT (ItemIntermediateRepresentationType t) Void (ResourceT m) MetricOutput -> ConduitT (ItemIntermediateRepresentationType t) Void (ResourceT m) MetricOutput
continueGEvalCalculations SAMultiLabelFMeasure (MultiLabelFMeasure beta) = defineContinuation countAgg (fMeasureOnCounts beta) noGraph continueGEvalCalculations (SAMultiLabelFMeasure matchingSpec) (MultiLabelFMeasure beta matchingSpec')
= defineContinuation countAgg (fMeasureOnCounts beta) noGraph
continueGEvalCalculations SALikelihood Likelihood = defineContinuation averageC logLossToLikehood noGraph continueGEvalCalculations SALikelihood Likelihood = defineContinuation averageC logLossToLikehood noGraph

33
src/GEval/CreateChallenge.hs
View File

@ -10,6 +10,7 @@ import GEval.EvaluationScheme
import GEval.Common (GEvalException(..), FormattingOptions(..)) import GEval.Common (GEvalException(..), FormattingOptions(..))
import GEval.Core (GEvalSpecification(..), configFileName, gesMainMetric, defaultTestName) import GEval.Core (GEvalSpecification(..), configFileName, gesMainMetric, defaultTestName)
import GEval.Submit (tokenFileName) import GEval.Submit (tokenFileName)
import GEval.MatchingSpecification (MatchingSpecification(ExactMatch))
import qualified System.Directory as D import qualified System.Directory as D
import Control.Conditional (whenM) import Control.Conditional (whenM)
@ -334,8 +335,8 @@ character (inclusively).
|] ++ (commonReadmeMDContents testName) |] ++ (commonReadmeMDContents testName)
readmeMDContents (ProbabilisticMultiLabelFMeasure beta) testName = readmeMDContents (MultiLabelFMeasure beta) testName readmeMDContents (ProbabilisticMultiLabelFMeasure beta) testName = readmeMDContents (MultiLabelFMeasure beta ExactMatch) testName
readmeMDContents (MultiLabelFMeasure beta) testName = [i| readmeMDContents (MultiLabelFMeasure beta _) testName = [i|
Tag names and their component Tag names and their component
============================= =============================
@ -535,8 +536,8 @@ trainContents TokenAccuracy = [hereLit|* V N I like cats
trainContents SegmentAccuracy = [hereLit|Art:1-3 N:5-11 V:12-13 A:15-19 The student's smart trainContents SegmentAccuracy = [hereLit|Art:1-3 N:5-11 V:12-13 A:15-19 The student's smart
N:1-6 N:8-10 V:12-13 A:15-18 Mary's dog is nice N:1-6 N:8-10 V:12-13 A:15-18 Mary's dog is nice
|] |]
trainContents (ProbabilisticMultiLabelFMeasure beta) = trainContents (MultiLabelFMeasure beta) trainContents (ProbabilisticMultiLabelFMeasure beta) = trainContents (MultiLabelFMeasure beta ExactMatch)
trainContents (MultiLabelFMeasure _) = [hereLit|I know Mr John Smith person/3,4,5 first-name/4 surname/5 trainContents (MultiLabelFMeasure _ _) = [hereLit|I know Mr John Smith person/3,4,5 first-name/4 surname/5
Steven bloody Brown person/1,3 first-name/1 surname/3 Steven bloody Brown person/1,3 first-name/1 surname/3
James and James first-name/1 firstname/3 James and James first-name/1 firstname/3
|] |]
@ -608,8 +609,8 @@ Ala has a cat
devInContents SegmentAccuracy = [hereLit|John is smart devInContents SegmentAccuracy = [hereLit|John is smart
Mary's intelligent Mary's intelligent
|] |]
devInContents (ProbabilisticMultiLabelFMeasure beta) = devInContents (MultiLabelFMeasure beta) devInContents (ProbabilisticMultiLabelFMeasure beta) = devInContents (MultiLabelFMeasure beta ExactMatch)
devInContents (MultiLabelFMeasure _) = [hereLit|Jan Kowalski is here devInContents (MultiLabelFMeasure _ _) = [hereLit|Jan Kowalski is here
I see him I see him
Barbara Barbara
|] |]
@ -676,8 +677,8 @@ N V * N
devExpectedContents SegmentAccuracy = [hereLit|N:1-4 V:6-7 A:9-13 devExpectedContents SegmentAccuracy = [hereLit|N:1-4 V:6-7 A:9-13
N:1-4 V:6-7 A:9-19 N:1-4 V:6-7 A:9-19
|] |]
devExpectedContents (ProbabilisticMultiLabelFMeasure beta) = devExpectedContents (MultiLabelFMeasure beta) devExpectedContents (ProbabilisticMultiLabelFMeasure beta) = devExpectedContents (MultiLabelFMeasure beta ExactMatch)
devExpectedContents (MultiLabelFMeasure _) = [hereLit|person/1,2 first-name/1 surname/2 devExpectedContents (MultiLabelFMeasure _ _) = [hereLit|person/1,2 first-name/1 surname/2
first-name/1 first-name/1
|] |]
@ -749,8 +750,8 @@ I know
testInContents SegmentAccuracy = [hereLit|Mary's cat is old testInContents SegmentAccuracy = [hereLit|Mary's cat is old
John is young John is young
|] |]
testInContents (ProbabilisticMultiLabelFMeasure beta) = testInContents (MultiLabelFMeasure beta) testInContents (ProbabilisticMultiLabelFMeasure beta) = testInContents (MultiLabelFMeasure beta ExactMatch)
testInContents (MultiLabelFMeasure _) = [hereLit|John bloody Smith testInContents (MultiLabelFMeasure _ _) = [hereLit|John bloody Smith
Nobody is there Nobody is there
I saw Marketa I saw Marketa
|] |]
@ -818,8 +819,8 @@ testExpectedContents TokenAccuracy = [hereLit|* V N
testExpectedContents SegmentAccuracy = [hereLit|N:1-6 N:8-10 V:12-13 A:15-17 testExpectedContents SegmentAccuracy = [hereLit|N:1-6 N:8-10 V:12-13 A:15-17
N:1-4 V:6-7 A:9-13 N:1-4 V:6-7 A:9-13
|] |]
testExpectedContents (ProbabilisticMultiLabelFMeasure beta) = testExpectedContents (MultiLabelFMeasure beta) testExpectedContents (ProbabilisticMultiLabelFMeasure beta) = testExpectedContents (MultiLabelFMeasure beta ExactMatch)
testExpectedContents (MultiLabelFMeasure _) = [hereLit|person/1,3 first-name/1 surname/3 testExpectedContents (MultiLabelFMeasure _ _) = [hereLit|person/1,3 first-name/1 surname/3
first-name/3 first-name/3
|] |]
@ -877,8 +878,8 @@ inHeaderContents BIOF1Labels = inHeaderContents BIOF1
inHeaderContents BIOF1 = Just ["Text"] inHeaderContents BIOF1 = Just ["Text"]
inHeaderContents TokenAccuracy = Just ["TokenizedText"] inHeaderContents TokenAccuracy = Just ["TokenizedText"]
inHeaderContents SegmentAccuracy = Just ["Segment"] inHeaderContents SegmentAccuracy = Just ["Segment"]
inHeaderContents (ProbabilisticMultiLabelFMeasure beta) = inHeaderContents (MultiLabelFMeasure beta) inHeaderContents (ProbabilisticMultiLabelFMeasure beta) = inHeaderContents (MultiLabelFMeasure beta ExactMatch)
inHeaderContents (MultiLabelFMeasure _) = Just ["Text"] inHeaderContents (MultiLabelFMeasure _ _) = Just ["Text"]
inHeaderContents MultiLabelLikelihood = inHeaderContents MultiLabelLogLoss inHeaderContents MultiLabelLikelihood = inHeaderContents MultiLabelLogLoss
inHeaderContents MultiLabelLogLoss = Just ["Utterance"] inHeaderContents MultiLabelLogLoss = Just ["Utterance"]
inHeaderContents (Soft2DFMeasure _) = inHeaderContents ClippEU inHeaderContents (Soft2DFMeasure _) = inHeaderContents ClippEU
@ -905,8 +906,8 @@ outHeaderContents BIOF1Labels = outHeaderContents BIOF1
outHeaderContents BIOF1 = Just ["BIOOutput"] outHeaderContents BIOF1 = Just ["BIOOutput"]
outHeaderContents TokenAccuracy = Just ["PartsOfSpeech"] outHeaderContents TokenAccuracy = Just ["PartsOfSpeech"]
outHeaderContents SegmentAccuracy = Just ["PartsOfSpeech"] outHeaderContents SegmentAccuracy = Just ["PartsOfSpeech"]
outHeaderContents (ProbabilisticMultiLabelFMeasure beta) = outHeaderContents (MultiLabelFMeasure beta) outHeaderContents (ProbabilisticMultiLabelFMeasure beta) = outHeaderContents (MultiLabelFMeasure beta ExactMatch)
outHeaderContents (MultiLabelFMeasure _) = Just ["Entities"] outHeaderContents (MultiLabelFMeasure _ _) = Just ["Entities"]
outHeaderContents MultiLabelLikelihood = outHeaderContents MultiLabelLogLoss outHeaderContents MultiLabelLikelihood = outHeaderContents MultiLabelLogLoss
outHeaderContents MultiLabelLogLoss = Just ["Emotion"] outHeaderContents MultiLabelLogLoss = Just ["Emotion"]
outHeaderContents (Soft2DFMeasure _) = Just ["Rectangle"] outHeaderContents (Soft2DFMeasure _) = Just ["Rectangle"]