From 0c5bbd63aae33000ce830ed785033692064338f3 Mon Sep 17 00:00:00 2001 From: Filip Gralinski Date: Fri, 30 Jul 2021 12:19:27 +0200 Subject: [PATCH] Add API for viewing progress logs --- Foundation.hs | 4 ++ Handler/Shared.hs | 47 +++++++++--- Handler/ShowChallenge.hs | 71 +++++++++++++++++++ Handler/Swagger.hs | 2 + README.md | 4 +- config/routes | 3 + static/test-gonito-as-backend.html | 110 ++++++++++++++++++++++++++++- 7 files changed, 229 insertions(+), 12 deletions(-) diff --git a/Foundation.hs b/Foundation.hs index 4304457..3a7c229 100644 --- a/Foundation.hs +++ b/Foundation.hs @@ -231,11 +231,15 @@ instance Yesod App where isAuthorized CreateTeamR _ = isTrustedAuthorized isAuthorized (TestProgressR _ _) _ = isTrustedAuthorized + isAuthorized (TestProgressJsonR _ _) _ = return Authorized isAuthorized SwaggerR _ = return Authorized isAuthorized (ViewProgressWithWebSocketsR _) _ = isTrustedAuthorized + isAuthorized (ViewProgressWithWebSocketsJsonR _) _ = return Authorized + isAuthorized (ViewProgressLogR _) _ = return Authorized + -- Default to Authorized for now. isAuthorized _ _ = isTrustedAuthorized diff --git a/Handler/Shared.hs b/Handler/Shared.hs index 043d6da..122a60f 100644 --- a/Handler/Shared.hs +++ b/Handler/Shared.hs @@ -1,5 +1,6 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE QuasiQuotes #-} module Handler.Shared where @@ -12,6 +13,8 @@ import Yesod.WebSockets import Handler.Runner import System.Exit +import Handler.JWT + import qualified Data.Text as T import qualified Data.Text.Encoding as DTE @@ -117,7 +120,7 @@ consoleApp jobId = do m <- readTVar jobs case IntMap.lookup jobId m of Nothing -> return Nothing - Just chan -> fmap Just $ dupTChan chan + Just chan -> fmap Just $ cloneTChan chan case mchan of Nothing -> do sendTextData ("CANNOT FIND THE OUTPUT (ALREADY SHOWN??)" :: Text) @@ -135,10 +138,27 @@ consoleApp jobId = do return () -getViewProgressWithWebSocketsR :: Int -> Handler Html -getViewProgressWithWebSocketsR jobId = do +getViewProgressWithWebSocketsJsonR :: Int -> Handler Value +getViewProgressWithWebSocketsJsonR jobId = do + webSockets $ consoleApp jobId + return $ String $ pack $ show jobId + +getViewProgressLogR :: Int -> Handler Html +getViewProgressLogR jobId = do webSockets $ consoleApp jobId - defaultLayout $ do + p <- widgetToPageContent logWidget + hamletToRepHtml [hamlet| + + + + #{pageTitle p} + ^{pageHead p} + <body> + ^{pageBody p} +|] + + +logWidget = do [whamlet| <div #outwindow> <div #output> @@ -196,6 +216,11 @@ getViewProgressWithWebSocketsR jobId = do |] +getViewProgressWithWebSocketsR :: Int -> Handler Html +getViewProgressWithWebSocketsR jobId = do + webSockets $ consoleApp jobId + defaultLayout logWidget + runViewProgressAsynchronously :: (Channel -> Handler ()) -> Handler Value runViewProgressAsynchronously action = runViewProgressGeneralized getJobIdAsJson action -- where getJobIdAsJson jobId = return $ Number (scientific (toInteger jobId) 0) @@ -208,9 +233,11 @@ runViewProgress' route action = runViewProgressGeneralized doRedirection action runViewProgressGeneralized :: (Int -> Handler v) -> (Channel -> Handler ()) -> Handler v runViewProgressGeneralized handler action = do App {..} <- getYesod - jobId <- randomInt + jobId' <- randomInt + -- we don't want negative numbers (so that nobody would be confused) + let jobId = abs jobId' chan <- liftIO $ atom $ do - chan <- newBroadcastTChan + chan <- newTChan m <- readTVar jobs writeTVar jobs $ IntMap.insert jobId chan m return chan @@ -221,10 +248,10 @@ runViewProgressGeneralized handler action = do liftIO $ atom $ do writeTChan chan $ Just "All done\n" writeTChan chan Nothing - m <- readTVar jobs - writeTVar jobs $ IntMap.delete jobId m +-- TODO we don't remove logs, they could clog up the memory +-- m <- readTVar jobs +-- writeTVar jobs $ IntMap.delete jobId m handler jobId - data RepoCloningSpec = RepoCloningSpec { cloningSpecRepo :: RepoSpec, cloningSpecReferenceRepo :: RepoSpec @@ -497,7 +524,7 @@ getViewProgressR jobId = do m <- readTVar jobs case IntMap.lookup jobId m of Nothing -> return Nothing - Just chan -> fmap Just $ dupTChan chan + Just chan -> fmap Just $ cloneTChan chan case mchan of Nothing -> notFound Just chan -> respondSource typePlain $ do diff --git a/Handler/ShowChallenge.hs b/Handler/ShowChallenge.hs index efd570f..938a9e6 100644 --- a/Handler/ShowChallenge.hs +++ b/Handler/ShowChallenge.hs @@ -1610,6 +1610,48 @@ challengeLayout withHeader challenge widget = do getTestProgressR :: Int -> Int -> Handler TypedContent getTestProgressR m d = runViewProgress $ doTestProgress m d +getTestProgressJsonR :: Int -> Int -> Handler Value +getTestProgressJsonR m d = do + _ <- requireAuthPossiblyByToken + runViewProgressAsynchronously $ doTestProgress m d + +declareTestProgressSwagger :: Declare (Definitions Schema) Swagger +declareTestProgressSwagger = do + -- param schemas + let numberSchema = toParamSchema (Proxy :: Proxy Int) + + numberResponse <- declareResponse (Proxy :: Proxy Int) + + return $ mempty + & paths .~ + fromList [ ("/api/test-progress/{num}/{delay}", + mempty & DS.get ?~ (mempty + & parameters .~ [ Inline $ mempty + & name .~ "num" + & description ?~ "The number up to which to count" + & required ?~ True + & schema .~ ParamOther (mempty + & in_ .~ ParamPath + & paramSchema .~ numberSchema), + Inline $ mempty + & name .~ "delay" + & description ?~ "Delay in seconds" + & required ?~ True + & schema .~ ParamOther (mempty + & in_ .~ ParamPath + & paramSchema .~ numberSchema) + ] + & produces ?~ MimeList ["application/json"] + & description ?~ "Counts up to a given number, returns an ID of an asynchronous job. This is just a sample end-point for testing logging of asynchronous jobs." + & at 200 ?~ Inline numberResponse)) + ] + +testProgressApi :: Swagger +testProgressApi = spec & definitions .~ defs + where + (defs, spec) = runDeclare declareTestProgressSwagger mempty + + doTestProgress :: Int -> Int -> Channel -> Handler () doTestProgress m d chan = do _ <- forM [1..m] $ (\i -> do @@ -1617,3 +1659,32 @@ doTestProgress m d chan = do liftIO $ threadDelay (d * 1000000) return ()) return () + + +declareViewProgressWithWebSocketsSwagger :: Declare (Definitions Schema) Swagger +declareViewProgressWithWebSocketsSwagger = do + -- param schemas + let numberSchema = toParamSchema (Proxy :: Proxy Int) + + numberResponse <- declareResponse (Proxy :: Proxy Int) + + return $ mempty + & paths .~ + fromList [ ("/api/view-progress-with-web-sockets/{jobId}", + mempty & DS.get ?~ (mempty + & parameters .~ [ Inline $ mempty + & name .~ "jobId" + & description ?~ "The ID for the job to be shown" + & required ?~ True + & schema .~ ParamOther (mempty + & in_ .~ ParamPath + & paramSchema .~ numberSchema)] + & produces ?~ MimeList ["application/json"] + & description ?~ "Initiates a web socket communication with which progress logs can be read. Returns just the Job ID (the same number as the parameter)" + & at 200 ?~ Inline numberResponse)) + ] + +viewProgressWithWebSockets :: Swagger +viewProgressWithWebSockets = spec & definitions .~ defs + where + (defs, spec) = runDeclare declareViewProgressWithWebSocketsSwagger mempty diff --git a/Handler/Swagger.hs b/Handler/Swagger.hs index 7b0d06e..5858f80 100644 --- a/Handler/Swagger.hs +++ b/Handler/Swagger.hs @@ -31,6 +31,8 @@ apiDescription = generalApi <> myTeamsApi <> challengeImgApi <> challengeRepoApi + <> testProgressApi + <> viewProgressWithWebSockets generalApi :: Swagger generalApi = (mempty :: Swagger) diff --git a/README.md b/README.md index 09902a6..0e68a7e 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,9 @@ application, this feature is on the way). 5. Set _Web Origin_ for the `gonito` client in Keycloak (e.g. simply add `*` there). -6. Set `JSON_WEB_KEY` variable to the content of the JWK key (or `GONITO_JSON_WEB_KEY` when using docker-compose) +6. Add some test user, set up some first/last name for them. + +7. Set `JSON_WEB_KEY` variable to the content of the JWK key (or `GONITO_JSON_WEB_KEY` when using docker-compose) and run Gonito. If you create a new user, you need to run `/api/add-info` GET diff --git a/config/routes b/config/routes index 9aef83f..c865515 100644 --- a/config/routes +++ b/config/routes @@ -10,7 +10,10 @@ /view-progress/#Int ViewProgressR GET /open-view-progress/#Int OpenViewProgressR GET /view-progress-with-web-sockets/#Int ViewProgressWithWebSocketsR GET +/api/view-progress-with-web-sockets/#Int ViewProgressWithWebSocketsJsonR GET +/api/view-progress-log/#Int ViewProgressLogR GET /test-progress/#Int/#Int TestProgressR GET +/api/test-progress/#Int/#Int TestProgressJsonR GET /list-challenges ListChallengesR GET /api/list-challenges ListChallengesJsonR GET diff --git a/static/test-gonito-as-backend.html b/static/test-gonito-as-backend.html index ecd3d9a..1f69e59 100644 --- a/static/test-gonito-as-backend.html +++ b/static/test-gonito-as-backend.html @@ -1,6 +1,39 @@ <html> <head> + <!-- This is an example of how to create a front-end using the Gonito + backend. + The code is ugly, but it is as simple as possible, no front-end + framework was assumed! + --> + + <style type="text/css" media="screen"> + #outwindow { + border: 2px solid black; + margin-bottom: 1em; + color: white; + background-color: black; + padding: 10pt; + } + #outwindow pre { + color: white; + background-color: black; + } + #wait { + animation: blink 1s linear infinite; + } + + @keyframes blink { + 0% { + opacity: 0; + } + 50% { + opacity: .5; + } + 100% { + opacity: 1; + } + </style> <script src="/static/js/keycloak.js"></script> <script> var keycloak; @@ -19,7 +52,7 @@ alert('failed to initialize'); }); - } + } var loadData = function (target) { @@ -82,6 +115,69 @@ xhr.send(); } + + + var getJSON = function(url, callback) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', '/api/' + url, true); + xhr.setRequestHeader('Accept', 'application/json'); + xhr.setRequestHeader('Authorization', 'Bearer ' + keycloak.token); + xhr.responseType = 'json'; + xhr.onload = function() { + callback(xhr.response); + }; + xhr.send(); + }; + + + // This is an example of how to handle logs obtained by an + // asynchronous process. + // As an example, "/api/test-progress/10/2" end-point was used + // (which just counts up to 10 with 2-second delays), it a + // similar way logs from, for example, + // "/api/challenge-submission/..." end-point could be handled + function testLogs() { + getJSON('test-progress/10/2', function(data) { + // the end-point just returns a job id, then we invoke + // the '/api/view-progress-with-web-sockets/' with this + // job Id + url = 'view-progress-with-web-sockets/' + data; + getJSON('view-progress-with-web-sockets/' + data, function(data) { + var output = document.getElementById("output"); + var wait = document.getElementById("wait"); + var seealso = document.getElementById("seealso"); + + wait.appendChild(document.createTextNode('... PLEASE WAIT ...')); + + var parsed_url = new URL(document.URL); + var ws_protocol = 'wss://'; + if (parsed_url.protocol == 'http:') { + ws_protocol = 'ws://'; + } + + msg = "The logs will be also available at " + + parsed_url.protocol + + "//" + + parsed_url.host + + '/api/view-progress-log/' + data; + seealso.appendChild(document.createTextNode(msg)); + + conn = new WebSocket(ws_protocol + parsed_url.host + '/api/' + url); + + conn.onmessage = function(e) { + var p = document.createElement("pre"); + p.appendChild(document.createTextNode(e.data)); + output.appendChild(p); + }; + + conn.onclose = function(e) { + wait.parentNode.removeChild(wait); + }; + + }); + }); + } + </script> </head> <body onload="initKeycloak()"> @@ -101,5 +197,17 @@ <p><button onclick="testCors()">CORS</button></p> + <p><button onclick="testLogs()">Logs</button></p> + + <p id="seealso"></p> + + <div id="outwindow"> + <div id="output"> + </div> + <div id="wait"> + </div> + </div> + + </body> </html>