Compare commits

...

4 Commits

Author SHA1 Message Date
patrycjalazna
931fcdbdb6 doesnt work properly 2021-04-13 11:02:33 +02:00
patrycjalazna
4873d78172 a 2021-04-08 23:06:34 +02:00
patrycjalazna
42ffe501fc geojournals skeleton added 2021-04-08 23:04:49 +02:00
107793cbf1 Change to Curl 2019-03-17 21:52:13 +01:00
4 changed files with 82 additions and 20 deletions

View File

@ -33,7 +33,7 @@ import Data.Tree.NTree.TypeDefs
import Data.Maybe
import Control.Monad.Trans
import Text.XML.HXT.XPath
-- import Text.XML.HXT.Curl
import Text.XML.HXT.Curl
import Text.XML.HXT.HTTP
import Text.Regex.TDFA
@ -64,8 +64,8 @@ downloadDocument = readFromDocument [withParseHTML yes,
withEncodingErrors no,
withPreserveComment yes,
withStrictInput yes,
withHTTP []
-- withCurl [("curl--user-agent","AMU Digital Libraries Indexing Agent")]
-- withHTTP []
withCurl [("curl--user-agent","AMU Digital Libraries Indexing Agent")]
]
downloadDocumentWithEncoding enc = readFromDocument [withParseHTML yes,
@ -73,13 +73,13 @@ downloadDocumentWithEncoding enc = readFromDocument [withParseHTML yes,
withEncodingErrors no,
withPreserveComment yes,
withInputEncoding enc,
withHTTP []]
-- withCurl []]
-- withHTTP []]
withCurl []]
downloadXmlDocument = readFromDocument [withWarnings no,
withEncodingErrors no,
withHTTP []]
-- withCurl [] ]
-- withHTTP []]
withCurl [] ]
data ShadowLibrary = ShadowLibrary { logoUrl :: Maybe String,

View File

@ -4,7 +4,7 @@ import ShadowLibrary.Core
import Text.XML.HXT.Core
import Text.XML.HXT.XPath
-- import Text.XML.HXT.Curl
--import Text.XML.HXT.Curl
import Data.List
import Data.List.Utils (replace)

60
app/geojournals.hs Normal file
View File

@ -0,0 +1,60 @@
{-# LANGUAGE Arrows, NoMonomorphismRestriction #-}
import ShadowLibrary.Core
import Text.XML.HXT.Core
import Text.XML.HXT.XPath
-- import Text.XML.HXT.Curl
import Data.List
import Data.List.Utils (replace)
import Text.Regex.Posix
import Text.Printf
-- extractRecords = extractLinks "//a[@class='action' and text() = 'View Journal']"
-- >>> arr ((++"/issue/archive") . init)
-- >>> extractLinksWithText "//div/h4[contains(@class, 'Vol')]/a"
extractRecords = extractLinksWithText "//a[@class='action' and text() = 'View Journal']" -- pary adres-tytuł
>>> second (arr $ replace "\r\n " " ") -- czyścimy drugi element pary, czyli tytuł z niepotrzebnych białych znaków
>>> first (arr ((++"/issue/archive") . init)) -- modyfikujemy pierwszy element pary, czyli adres URL
>>> first (extractLinksWithText "//div/h4[contains(@class, 'Vol')]/a")
-- >>> first (extractLinksWithText "//div[@class='tocTitle']/a")
-- >>> first (arr ((++"/pdf") . init))
-- >>> second (arr $ replace "2020")
-- ostatecznie wyjdą trójki ((adres URL, tytuł artykułu), tytuł rocznika)
toShadowItem :: ((String, String), String) -> ShadowItem
toShadowItem ((url, articleTitle), yearlyTitle) =
(defaultShadowItem url title) {
originalDate = Just date,
itype = "periodical",
format = Just "pdf",
finalUrl = url
}
where title = "Geo Journal OOO" ++ yearlyTitle ++ " AAAAA" ++ (replace "\r\n" "" (replace "\r\n " "" articleTitle))
date = "202020"
-- getDate url =
-- case url =~~ "/(19[0-9][0-9]|20[0-9][0-9])/" :: Maybe [[String]] of
-- Just [[_, year]] -> year
-- otherwise -> error $ "unexpected url: " ++ url
main = do
let start = "https://geojournals.pgi.gov.pl"
let shadowLibrary = ShadowLibrary {logoUrl=Nothing,
lname="Polish Geological Journals",
abbrev="geojour",
lLevel=0,
webpage=start}
extractItemsStartingFromUrl shadowLibrary start (extractRecords >>> arr toShadowItem)

View File

@ -20,6 +20,7 @@ library
, HTTP
, hxt
, hxt-http
, hxt-curl
, hxt-xpath
, MissingH
, monad-logger
@ -34,18 +35,18 @@ library
, tz
default-language: Haskell2010
-- executable maly-modelarz-exe
-- hs-source-dirs: app
-- main-is: malymodelarz.hs
-- ghc-options: -threaded -rtsopts -with-rtsopts=-N
-- build-depends: base
-- , hxt
-- , hxt-curl
-- , hxt-xpath
-- , MissingH
-- , regex-posix
-- , shadow-library
-- default-language: Haskell2010
executable geojournals
hs-source-dirs: app
main-is: geojournals.hs
ghc-options: -threaded -rtsopts -with-rtsopts=-N
build-depends: base
, hxt
, hxt-curl
, hxt-xpath
, MissingH
, regex-posix
, shadow-library
default-language: Haskell2010
executable almanachmuszyny
hs-source-dirs: app
@ -54,6 +55,7 @@ executable almanachmuszyny
build-depends: base
, hxt
, hxt-xpath
, hxt-curl
, MissingH
, regex-posix
, shadow-library