diff --git a/Gonito/ExtractMetadata.hs b/Gonito/ExtractMetadata.hs index cfcad06..31d8c9a 100644 --- a/Gonito/ExtractMetadata.hs +++ b/Gonito/ExtractMetadata.hs @@ -29,6 +29,8 @@ import Handler.Shared (gitPath) import "Glob" System.FilePath.Glob as G +import PersistSHA1 + data ExtractionOptions = ExtractionOptions { extractionOptionsDescription :: Maybe Text, extractionOptionsTags :: Maybe (S.Set Text), @@ -36,8 +38,9 @@ data ExtractionOptions = ExtractionOptions { extractionOptionsUnwantedParams :: Maybe [Text], extractionOptionsParamFiles :: Maybe [String], extractionOptionsMLRunPath :: Maybe FilePath, - extractionOptionsExternalLinks :: Maybe [Link] - } + extractionOptionsExternalLinks :: Maybe [Link], + extractionOptionsDependencies :: Maybe [SHA1] + } instance FromJSON ExtractionOptions where parseJSON = withObject "ExtractionOptions" $ \v -> ExtractionOptions @@ -48,6 +51,7 @@ instance FromJSON ExtractionOptions where <*> v .:? "param-files" <*> v .:? "mlrun-path" <*> v .:? "links" + <*> fmap (fmap (Import.map fromTextToSHA1)) (v .:? "dependencies") instance Default ExtractionOptions where def = ExtractionOptions { @@ -57,7 +61,8 @@ instance Default ExtractionOptions where extractionOptionsUnwantedParams = Nothing, extractionOptionsParamFiles = Nothing, extractionOptionsMLRunPath = Nothing, - extractionOptionsExternalLinks = Nothing + extractionOptionsExternalLinks = Nothing, + extractionOptionsDependencies = Nothing } data Link = Link { @@ -74,7 +79,8 @@ data GonitoMetadata = GonitoMetadata { gonitoMetadataDescription :: Text, gonitoMetadataTags :: S.Set Text, gonitoMetadataGeneralParams :: M.Map Text Text, - gonitoMetadataExternalLinks :: [Link] + gonitoMetadataExternalLinks :: [Link], + gonitoMetadataDependencies :: [SHA1] } deriving (Eq, Show) @@ -102,7 +108,11 @@ combineExtractionOptions (Just otherOptions) options = ExtractionOptions { extractionOptionsMLRunPath = combineWithF extractionOptionsMLRunPath, extractionOptionsExternalLinks = case extractionOptionsExternalLinks options of Nothing -> extractionOptionsExternalLinks otherOptions - Just links -> Just (links ++ (fromMaybe [] $ extractionOptionsExternalLinks otherOptions)) } + Just links -> Just (links ++ (fromMaybe [] $ extractionOptionsExternalLinks otherOptions)), + extractionOptionsDependencies = case extractionOptionsDependencies options of + Nothing -> extractionOptionsDependencies otherOptions + Just links -> Just (links ++ (fromMaybe [] $ extractionOptionsDependencies otherOptions)) } + where combineWithT fun = case fun options of Nothing -> fun otherOptions Just v -> Just v @@ -146,13 +156,28 @@ extractMetadataFromRepoDir repoDir formExtractionOptions = do `M.union` fromMaybe M.empty (extractionOptionsGeneralParams extractionOptions) + let dependenciesFromYaml = fromMaybe [] $ extractionOptionsDependencies extractionOptions + dependenciesFromGitSubmodules <- extractDependenciesFromGitSubmodules repoDir + pure $ GonitoMetadata { gonitoMetadataDescription = description, gonitoMetadataTags = tagsParsed, gonitoMetadataGeneralParams = params, - gonitoMetadataExternalLinks = fromMaybe [] (extractionOptionsExternalLinks extractionOptions) + gonitoMetadataExternalLinks = fromMaybe [] (extractionOptionsExternalLinks extractionOptions), + gonitoMetadataDependencies = dependenciesFromYaml ++ dependenciesFromGitSubmodules } +extractDependenciesFromGitSubmodules :: FilePath -> IO [SHA1] +extractDependenciesFromGitSubmodules repoDir = do + (exitCode, out) <- runProgram repoDir gitPath ["submodule"] + return $ case exitCode of + ExitSuccess -> Import.map (fromTextToSHA1 + . Data.Text.take sha1Lenght + . Data.Text.drop 1) + $ Data.Text.lines out + ExitFailure _ -> [] + where sha1Lenght = 40 + parseParamFile :: FilePath -> IO (M.Map Text Text) parseParamFile yamlFile = do diff --git a/Handler/ShowChallenge.hs b/Handler/ShowChallenge.hs index 3fa7aa9..78a38a5 100644 --- a/Handler/ShowChallenge.hs +++ b/Handler/ShowChallenge.hs @@ -280,7 +280,8 @@ doCreateSubmission userId challengeId mDescription mTags repoSpec chan = do extractionOptionsUnwantedParams = Nothing, extractionOptionsParamFiles = Nothing, extractionOptionsMLRunPath = Nothing, - extractionOptionsExternalLinks = Nothing }) + extractionOptionsExternalLinks = Nothing, + extractionOptionsDependencies = Nothing }) submissionId <- getSubmission userId repoId @@ -294,6 +295,10 @@ doCreateSubmission userId challengeId mDescription mTags repoSpec chan = do externalLinkTitle = linkTitle l, externalLinkUrl = linkUrl l }) $ gonitoMetadataExternalLinks gonitoMetadata + _ <- runDB $ mapM insert $ map (\s -> Dependency { + dependencySubRepoCommit = s, + dependencySuperRepoCommit = (repoCurrentCommit repo) }) $ gonitoMetadataDependencies gonitoMetadata + outs <- getOuts chan submissionId (gonitoMetadataGeneralParams gonitoMetadata) currentTagIds <- runDB $ selectList [SubmissionTagSubmission ==. submissionId] [] diff --git a/PersistSHA1.hs b/PersistSHA1.hs index f91b97f..8cd4740 100644 --- a/PersistSHA1.hs +++ b/PersistSHA1.hs @@ -9,7 +9,7 @@ import qualified Data.Text as T import Numeric (showHex) data SHA1 = SHA1 ByteString - deriving Show + deriving (Eq, Show) toHex :: ByteString -> ByteString toHex = BC.pack . concat . (map ("\\x"++)) . (map (flip showHex "")) . B.unpack diff --git a/config/models b/config/models index b410937..38239cc 100644 --- a/config/models +++ b/config/models @@ -72,10 +72,23 @@ ExternalLink submission SubmissionId title Text Maybe url Text +-- this represents forks, i.e. when a submission for a given challenge +-- was based (in terms of git history) on another submission for the same challenge +-- NOTE: not implemented yet Fork source SubmissionId target SubmissionId UniqueSourceTarget source target +-- for representing dependencies across challenges; +-- e.g. when a model generated in a submission is used +-- by another submission in another challenge; +-- dependencies are expressed as a relation between +-- commit hashes rather than submissions/repos +-- (can be easily linked to submission via SubmissionCommit link) +Dependency + subRepoCommit SHA1 + superRepoCommit SHA1 + UniqueSubSuperSubmission subRepoCommit superRepoCommit Evaluation test TestId checksum SHA1 diff --git a/test/Gonito/ExtractMetadataSpec.hs b/test/Gonito/ExtractMetadataSpec.hs index 1cf454c..b4be718 100644 --- a/test/Gonito/ExtractMetadataSpec.hs +++ b/test/Gonito/ExtractMetadataSpec.hs @@ -4,6 +4,8 @@ module Gonito.ExtractMetadataSpec (spec) where import Import +import PersistSHA1 + import qualified Data.Set as S import qualified Data.Map.Strict as M @@ -18,7 +20,8 @@ spec = do gonitoMetadataDescription = "Simple solution", gonitoMetadataTags = S.fromList ["foo", "simple-solution", "baz"], gonitoMetadataGeneralParams = M.empty, - gonitoMetadataExternalLinks = [] + gonitoMetadataExternalLinks = [], + gonitoMetadataDependencies = [] } it "simple with some fields from the form" $ do extractMetadataFromRepoDir "test/fake-git-repos/simple/" def { @@ -28,7 +31,8 @@ spec = do gonitoMetadataDescription = "Other solution", gonitoMetadataTags = S.fromList ["foo", "simple-solution", "baz", "other-tag"], gonitoMetadataGeneralParams = M.empty, - gonitoMetadataExternalLinks = [] + gonitoMetadataExternalLinks = [], + gonitoMetadataDependencies = [] } it "with gonito.yaml" $ do extractMetadataFromRepoDir "test/fake-git-repos/with-gonito-yaml/" def `shouldReturn` GonitoMetadata { @@ -41,5 +45,9 @@ spec = do gonitoMetadataExternalLinks = [ Link (Just "gitlab") "https://about.gitlab.com/", Link (Just "Polish Wikipedia") "https://pl.wikipedia.org/wiki/Wikipedia:Strona_g%C5%82%C3%B3wna", - Link Nothing "https://tvtropes.org/" ] + Link Nothing "https://tvtropes.org/" ], + gonitoMetadataDependencies = [ + fromTextToSHA1 "39ae80911874c8f6bac1cdc57771bc2929cf0177", + fromTextToSHA1 "11b33fd677825228412019b289c470260389bea5" + ] } diff --git a/test/fake-git-repos/with-gonito-yaml b/test/fake-git-repos/with-gonito-yaml index 94553a6..9addf5b 160000 --- a/test/fake-git-repos/with-gonito-yaml +++ b/test/fake-git-repos/with-gonito-yaml @@ -1 +1 @@ -Subproject commit 94553a6200cc5e89c195f92a75ae9724694797d1 +Subproject commit 9addf5b00a9f917225934abca20b21f964e6fe92