Fix line-by-line mode for MultiLabel
This commit is contained in:
parent
a08ae0cf99
commit
d21cfe64fb
@ -19,6 +19,7 @@ data FormattingOptions = FormattingOptions {
|
|||||||
decimalPlaces :: Maybe Int,
|
decimalPlaces :: Maybe Int,
|
||||||
asPercentage :: Bool
|
asPercentage :: Bool
|
||||||
}
|
}
|
||||||
|
deriving (Show)
|
||||||
|
|
||||||
data MetricResult = SimpleRun MetricValue | BootstrapResampling [MetricValue]
|
data MetricResult = SimpleRun MetricValue | BootstrapResampling [MetricValue]
|
||||||
|
|
||||||
|
@ -56,8 +56,6 @@ module GEval.Core
|
|||||||
fromSpecificationToWithInput
|
fromSpecificationToWithInput
|
||||||
) where
|
) where
|
||||||
|
|
||||||
import Debug.Trace
|
|
||||||
|
|
||||||
import Data.Singletons.TH
|
import Data.Singletons.TH
|
||||||
|
|
||||||
import GEval.Metric
|
import GEval.Metric
|
||||||
@ -152,13 +150,39 @@ hasFiltering (_:ops) = hasFiltering ops
|
|||||||
|
|
||||||
-- | Could output be preprocessable
|
-- | Could output be preprocessable
|
||||||
isPreprocessable :: Metric -> Bool
|
isPreprocessable :: Metric -> Bool
|
||||||
|
isPreprocessable RMSE = False
|
||||||
|
isPreprocessable MSE = False
|
||||||
|
isPreprocessable Pearson = False
|
||||||
|
isPreprocessable Spearman = False
|
||||||
isPreprocessable BLEU = True
|
isPreprocessable BLEU = True
|
||||||
isPreprocessable GLEU = True
|
isPreprocessable GLEU = True
|
||||||
isPreprocessable WER = True
|
isPreprocessable WER = True
|
||||||
isPreprocessable Accuracy = True
|
isPreprocessable Accuracy = True
|
||||||
|
isPreprocessable ClippEU = False
|
||||||
|
isPreprocessable (FMeasure _) = False
|
||||||
|
isPreprocessable (MacroFMeasure _) = False
|
||||||
|
isPreprocessable (SoftFMeasure _) = False
|
||||||
|
isPreprocessable (ProbabilisticMultiLabelFMeasure _) = True
|
||||||
|
isPreprocessable (ProbabilisticSoftFMeasure _) = True
|
||||||
|
isPreprocessable (Soft2DFMeasure _) = False
|
||||||
|
isPreprocessable (FLCFMeasure _) = False
|
||||||
|
isPreprocessable NMI = False
|
||||||
|
isPreprocessable (LogLossHashed _) = False
|
||||||
|
isPreprocessable (LikelihoodHashed _) = False
|
||||||
isPreprocessable CharMatch = True
|
isPreprocessable CharMatch = True
|
||||||
|
isPreprocessable MAP = False
|
||||||
|
isPreprocessable LogLoss = False
|
||||||
|
isPreprocessable Likelihood = False
|
||||||
|
isPreprocessable BIOF1 = False
|
||||||
|
isPreprocessable BIOF1Labels = False
|
||||||
isPreprocessable TokenAccuracy = True
|
isPreprocessable TokenAccuracy = True
|
||||||
isPreprocessable _ = False
|
isPreprocessable SegmentAccuracy = True
|
||||||
|
isPreprocessable MAE = False
|
||||||
|
isPreprocessable SMAPE = False
|
||||||
|
isPreprocessable (MultiLabelFMeasure _ _) = True
|
||||||
|
isPreprocessable MultiLabelLogLoss = False
|
||||||
|
isPreprocessable MultiLabelLikelihood = False
|
||||||
|
isPreprocessable (Mean metric) = isPreprocessable metric
|
||||||
|
|
||||||
defaultOutDirectory = "."
|
defaultOutDirectory = "."
|
||||||
defaultTestName = "test-A"
|
defaultTestName = "test-A"
|
||||||
@ -192,7 +216,7 @@ data GEvalSpecification = GEvalSpecification
|
|||||||
gesBootstrapResampling :: Maybe Int,
|
gesBootstrapResampling :: Maybe Int,
|
||||||
gesInHeader :: Maybe String,
|
gesInHeader :: Maybe String,
|
||||||
gesOutHeader :: Maybe String }
|
gesOutHeader :: Maybe String }
|
||||||
|
deriving (Show)
|
||||||
|
|
||||||
gesMainMetric :: GEvalSpecification -> Metric
|
gesMainMetric :: GEvalSpecification -> Metric
|
||||||
gesMainMetric spec = case gesMetrics spec of
|
gesMainMetric spec = case gesMetrics spec of
|
||||||
@ -560,6 +584,7 @@ isEmptyFileSource _ = return False
|
|||||||
logLossToLikehood logLoss = exp (-logLoss)
|
logLossToLikehood logLoss = exp (-logLoss)
|
||||||
|
|
||||||
data LineInFile = LineInFile SourceSpec Word32 Text
|
data LineInFile = LineInFile SourceSpec Word32 Text
|
||||||
|
deriving Show
|
||||||
|
|
||||||
gevalBootstrapOnSources :: (MonadIO m, MonadThrow m, MonadUnliftIO m) =>
|
gevalBootstrapOnSources :: (MonadIO m, MonadThrow m, MonadUnliftIO m) =>
|
||||||
Int -- ^ number of samples
|
Int -- ^ number of samples
|
||||||
|
15
test/Spec.hs
15
test/Spec.hs
@ -364,7 +364,6 @@ main = hspec $ do
|
|||||||
runGEvalTest "f1-with-preprocessing" `shouldReturnAlmost` 0.57142857142857
|
runGEvalTest "f1-with-preprocessing" `shouldReturnAlmost` 0.57142857142857
|
||||||
it "Regexp substition" $ do
|
it "Regexp substition" $ do
|
||||||
runGEvalTest "accuracy-with-flags" `shouldReturnAlmost` 0.8
|
runGEvalTest "accuracy-with-flags" `shouldReturnAlmost` 0.8
|
||||||
it "In line-by-line mode" $ do
|
|
||||||
let sampleChallenge = GEvalSpecification
|
let sampleChallenge = GEvalSpecification
|
||||||
{ gesOutDirectory = "test/accuracy-flags-line-by-line/accuracy-flags-line-by-line-solution",
|
{ gesOutDirectory = "test/accuracy-flags-line-by-line/accuracy-flags-line-by-line-solution",
|
||||||
gesExpectedDirectory = Just "test/accuracy-flags-line-by-line/accuracy-flags-line-by-line",
|
gesExpectedDirectory = Just "test/accuracy-flags-line-by-line/accuracy-flags-line-by-line",
|
||||||
@ -384,6 +383,7 @@ main = hspec $ do
|
|||||||
gesBootstrapResampling = Nothing,
|
gesBootstrapResampling = Nothing,
|
||||||
gesInHeader = Nothing,
|
gesInHeader = Nothing,
|
||||||
gesOutHeader = Nothing }
|
gesOutHeader = Nothing }
|
||||||
|
it "In line-by-line mode Accuracy" $ do
|
||||||
results <- runLineByLineGeneralized KeepTheOriginalOrder sampleChallenge (const Data.Conduit.List.consume)
|
results <- runLineByLineGeneralized KeepTheOriginalOrder sampleChallenge (const Data.Conduit.List.consume)
|
||||||
results `shouldBe` [
|
results `shouldBe` [
|
||||||
LineRecord "foo"
|
LineRecord "foo"
|
||||||
@ -396,6 +396,19 @@ main = hspec $ do
|
|||||||
"Fox bax 456 bax"
|
"Fox bax 456 bax"
|
||||||
2
|
2
|
||||||
0.0]
|
0.0]
|
||||||
|
it "In line-by-line mode F0" $ do
|
||||||
|
results <- runLineByLineGeneralized KeepTheOriginalOrder sampleChallenge { gesMetrics = [read "MultiLabel-F0:f<in[1]:foo>s<\\d+><>"]} (const Data.Conduit.List.consume)
|
||||||
|
results `shouldBe` [
|
||||||
|
LineRecord "foo"
|
||||||
|
"Ala 123 ma kota."
|
||||||
|
"Ala ma 2 kota ."
|
||||||
|
1
|
||||||
|
1.0,
|
||||||
|
LineRecord "foo"
|
||||||
|
"Foo bar baz"
|
||||||
|
"Fox bax 456 bax"
|
||||||
|
2
|
||||||
|
0.0]
|
||||||
describe "Flag examples" $ do
|
describe "Flag examples" $ do
|
||||||
it "none" $ do
|
it "none" $ do
|
||||||
runGEvalTest "flags-none" `shouldReturnAlmost` 0.2
|
runGEvalTest "flags-none" `shouldReturnAlmost` 0.2
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
foo bar baz
|
foo bar baz
|
||||||
|
|
||||||
foo
|
foo
|
||||||
qqq qqq
|
qqq qqq
|
||||||
|
|
Loading…
Reference in New Issue
Block a user