Add dependency tracking

Gonito/ExtractMetadata.hs

@@ -29,6 +29,8 @@ import Handler.Shared (gitPath)
 
 import "Glob" System.FilePath.Glob as G
 
+import PersistSHA1
+
 data ExtractionOptions = ExtractionOptions {
   extractionOptionsDescription :: Maybe Text,
   extractionOptionsTags :: Maybe (S.Set Text),
@@ -36,8 +38,9 @@ data ExtractionOptions = ExtractionOptions {
   extractionOptionsUnwantedParams :: Maybe [Text],
   extractionOptionsParamFiles :: Maybe [String],
   extractionOptionsMLRunPath :: Maybe FilePath,
-  extractionOptionsExternalLinks :: Maybe [Link]
+  extractionOptionsExternalLinks :: Maybe [Link],
-  }
+  extractionOptionsDependencies :: Maybe [SHA1]
+   }
 
 instance FromJSON ExtractionOptions where
     parseJSON = withObject "ExtractionOptions" $ \v -> ExtractionOptions
@@ -48,6 +51,7 @@ instance FromJSON ExtractionOptions where
         <*> v .:? "param-files"
         <*> v .:? "mlrun-path"
         <*> v .:? "links"
+        <*> fmap (fmap (Import.map fromTextToSHA1)) (v .:? "dependencies")
 
 instance Default ExtractionOptions where
   def = ExtractionOptions {
@@ -57,7 +61,8 @@ instance Default ExtractionOptions where
     extractionOptionsUnwantedParams = Nothing,
     extractionOptionsParamFiles = Nothing,
     extractionOptionsMLRunPath = Nothing,
-    extractionOptionsExternalLinks = Nothing
+    extractionOptionsExternalLinks = Nothing,
+    extractionOptionsDependencies = Nothing
     }
 
 data Link = Link {
@@ -74,7 +79,8 @@ data GonitoMetadata = GonitoMetadata {
   gonitoMetadataDescription :: Text,
   gonitoMetadataTags :: S.Set Text,
   gonitoMetadataGeneralParams :: M.Map Text Text,
-  gonitoMetadataExternalLinks :: [Link]
+  gonitoMetadataExternalLinks :: [Link],
+  gonitoMetadataDependencies :: [SHA1]
   }
   deriving (Eq, Show)
 
@@ -102,7 +108,11 @@ combineExtractionOptions (Just otherOptions) options = ExtractionOptions {
   extractionOptionsMLRunPath = combineWithF extractionOptionsMLRunPath,
   extractionOptionsExternalLinks = case extractionOptionsExternalLinks options of
                                      Nothing -> extractionOptionsExternalLinks otherOptions
-                                     Just links -> Just (links ++ (fromMaybe [] $ extractionOptionsExternalLinks otherOptions)) }
+                                     Just links -> Just (links ++ (fromMaybe [] $ extractionOptionsExternalLinks otherOptions)),
+  extractionOptionsDependencies = case extractionOptionsDependencies options of
+                                     Nothing -> extractionOptionsDependencies otherOptions
+                                     Just links -> Just (links ++ (fromMaybe [] $ extractionOptionsDependencies otherOptions)) }
+
   where combineWithT fun = case fun options of
                              Nothing -> fun otherOptions
                              Just v -> Just v
@@ -146,13 +156,28 @@ extractMetadataFromRepoDir repoDir formExtractionOptions = do
         `M.union`
         fromMaybe M.empty (extractionOptionsGeneralParams extractionOptions)
 
+  let dependenciesFromYaml = fromMaybe [] $ extractionOptionsDependencies extractionOptions
+  dependenciesFromGitSubmodules <- extractDependenciesFromGitSubmodules repoDir
+
   pure $ GonitoMetadata {
     gonitoMetadataDescription = description,
     gonitoMetadataTags = tagsParsed,
     gonitoMetadataGeneralParams = params,
-    gonitoMetadataExternalLinks = fromMaybe [] (extractionOptionsExternalLinks extractionOptions)
+    gonitoMetadataExternalLinks = fromMaybe [] (extractionOptionsExternalLinks extractionOptions),
+    gonitoMetadataDependencies = dependenciesFromYaml ++ dependenciesFromGitSubmodules
   }
 
+extractDependenciesFromGitSubmodules :: FilePath -> IO [SHA1]
+extractDependenciesFromGitSubmodules repoDir = do
+  (exitCode, out) <- runProgram repoDir gitPath ["submodule"]
+  return $ case exitCode of
+             ExitSuccess -> Import.map (fromTextToSHA1
+                                       . Data.Text.take sha1Lenght
+                                       . Data.Text.drop 1)
+                           $ Data.Text.lines out
+             ExitFailure _ -> []
+  where sha1Lenght = 40
+
 
 parseParamFile :: FilePath -> IO (M.Map Text Text)
 parseParamFile yamlFile = do

Handler/ShowChallenge.hs

@@ -280,7 +280,8 @@ doCreateSubmission userId challengeId mDescription mTags repoSpec chan = do
                                                                 extractionOptionsUnwantedParams = Nothing,
                                                                 extractionOptionsParamFiles = Nothing,
                                                                 extractionOptionsMLRunPath = Nothing,
-                                                                extractionOptionsExternalLinks = Nothing })
+                                                                extractionOptionsExternalLinks = Nothing,
+                                                                extractionOptionsDependencies = Nothing })
 
       submissionId <- getSubmission userId
                                    repoId
@@ -294,6 +295,10 @@ doCreateSubmission userId challengeId mDescription mTags repoSpec chan = do
                                         externalLinkTitle = linkTitle l,
                                         externalLinkUrl = linkUrl l }) $ gonitoMetadataExternalLinks gonitoMetadata
 
+      _ <- runDB $ mapM insert $ map (\s -> Dependency {
+                                        dependencySubRepoCommit = s,
+                                        dependencySuperRepoCommit = (repoCurrentCommit repo) }) $ gonitoMetadataDependencies gonitoMetadata
+
       outs <- getOuts chan submissionId (gonitoMetadataGeneralParams gonitoMetadata)
 
       currentTagIds <- runDB $ selectList [SubmissionTagSubmission ==. submissionId] []

PersistSHA1.hs

@@ -9,7 +9,7 @@ import qualified Data.Text as T
 import Numeric (showHex)
 
 data SHA1 = SHA1 ByteString
-            deriving Show
+            deriving (Eq, Show)
 
 toHex :: ByteString -> ByteString
 toHex = BC.pack . concat . (map ("\\x"++)) . (map (flip showHex "")) . B.unpack

config/models

@@ -72,10 +72,23 @@ ExternalLink
     submission SubmissionId
     title Text Maybe
     url Text
+-- this represents forks, i.e. when a submission for a given challenge
+-- was based (in terms of git history) on another submission for the same challenge
+-- NOTE: not implemented yet
 Fork
     source SubmissionId
     target SubmissionId
     UniqueSourceTarget source target
+-- for representing dependencies across challenges;
+-- e.g. when a model generated in a submission is used
+-- by another submission in another challenge;
+-- dependencies are expressed as a relation between
+-- commit hashes rather than submissions/repos
+-- (can be easily linked to submission via SubmissionCommit link)
+Dependency
+    subRepoCommit SHA1
+    superRepoCommit SHA1
+    UniqueSubSuperSubmission subRepoCommit superRepoCommit
 Evaluation
     test TestId
     checksum SHA1

test/Gonito/ExtractMetadataSpec.hs

@@ -4,6 +4,8 @@ module Gonito.ExtractMetadataSpec (spec) where
 
 import Import
 
+import PersistSHA1
+
 import qualified Data.Set as S
 import qualified Data.Map.Strict as M
 
@@ -18,7 +20,8 @@ spec = do
         gonitoMetadataDescription = "Simple solution",
         gonitoMetadataTags = S.fromList ["foo", "simple-solution", "baz"],
         gonitoMetadataGeneralParams = M.empty,
-        gonitoMetadataExternalLinks = []
+        gonitoMetadataExternalLinks = [],
+        gonitoMetadataDependencies = []
         }
     it "simple with some fields from the form" $ do
       extractMetadataFromRepoDir "test/fake-git-repos/simple/" def {
@@ -28,7 +31,8 @@ spec = do
         gonitoMetadataDescription = "Other solution",
         gonitoMetadataTags = S.fromList ["foo", "simple-solution", "baz", "other-tag"],
         gonitoMetadataGeneralParams = M.empty,
-        gonitoMetadataExternalLinks = []
+        gonitoMetadataExternalLinks = [],
+        gonitoMetadataDependencies = []
         }
     it "with gonito.yaml" $ do
       extractMetadataFromRepoDir "test/fake-git-repos/with-gonito-yaml/" def `shouldReturn` GonitoMetadata {
@@ -41,5 +45,9 @@ spec = do
         gonitoMetadataExternalLinks = [
             Link (Just "gitlab") "https://about.gitlab.com/",
             Link (Just "Polish Wikipedia") "https://pl.wikipedia.org/wiki/Wikipedia:Strona_g%C5%82%C3%B3wna",
-            Link Nothing "https://tvtropes.org/" ]
+            Link Nothing "https://tvtropes.org/" ],
+        gonitoMetadataDependencies = [
+            fromTextToSHA1 "39ae80911874c8f6bac1cdc57771bc2929cf0177",
+            fromTextToSHA1 "11b33fd677825228412019b289c470260389bea5"
+            ]
         }

test/fake-git-repos/with-gonito-yaml

@@ -1 +1 @@
-Subproject commit 94553a6200cc5e89c195f92a75ae9724694797d1
+Subproject commit 9addf5b00a9f917225934abca20b21f964e6fe92