s434708/twilight-library-s434708
1
0
Fork 0
forked from filipg/twilight-library
Code Issues Pull Requests Releases Wiki Activity

next version

Browse Source
This commit is contained in:
Łukasz Jędyk 2021-04-13 10:25:17 +02:00
parent 974098d4ec
commit c5c01e7d81
1 changed files with 19 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
app/archiwapilsudski.hs
View File

@ -12,14 +12,17 @@ import Text.Regex.Posix
import Text.Printf import Text.Printf
cleanText = arr (replace "\177" "a") cleanText = arr (replace "\261" "a")
>>> arr (replace "\230" "c") >>> arr (replace "\263" "c")
>>> arr (replace "\234" "e") >>> arr (replace "\281" "e")
>>> arr (replace "\179" "l") >>> arr (replace "\322" "l")
>>> arr (replace "\241" "n") >>> arr (replace "\321" "L")
>>> arr (replace "\324" "n")
>>> arr (replace "\243" "o") >>> arr (replace "\243" "o")
>>> arr (replace "\182" "s") >>> arr (replace "\347" "s")
>>> arr (replace "(\188)|(\191)" "z") >>> arr (replace "\346" "S")
>>> arr (replace "(\378)|(\380)" "z")
>>> arr (replace "(\377)|(\379)" "Z")
>>> arr (replace "\160" "") >>> arr (replace "\160" "")
cleanDate = arr (replace "\160" "") cleanDate = arr (replace "\160" "")
@ -38,8 +41,8 @@ getDate = proc doc -> do
returnA -< cleanedDate returnA -< cleanedDate
getUrls = proc doc -> do getUrls = proc doc -> do
xpathTrees <- getXPathTrees "//div[@style='padding: 4px 7px;float:left;']/a" -< doc xpathTrees <- getXPathTrees "//div[@style='padding: 4px 7px;float:left;']/a[contains(@href,'pdf')]" -< doc
urlList <- (getAttrValue "href" >>> arr ("http://archiwa.pilsudski.org/" ++ ) >>> intercalate "//") -< xpathTrees urlList <- (getAttrValue "href" >>> arr ("http://archiwa.pilsudski.org/" ++)) -< xpathTrees
returnA -< urlList returnA -< urlList
extractRecordData = proc recordUrl -> do extractRecordData = proc recordUrl -> do
@ -47,7 +50,7 @@ extractRecordData = proc recordUrl -> do
recordTitle <- getTitle -< doc recordTitle <- getTitle -< doc
recordDate <- getDate -< doc recordDate <- getDate -< doc
recordUrls <- getUrls -< doc recordUrls <- getUrls -< doc
returnA -< recordUrls returnA -< (recordUrls, recordTitle, recordDate)
extractRecordLinks = extractLinks "//table[@class='tabelka']//a[contains(@href,'.php')]" extractRecordLinks = extractLinks "//table[@class='tabelka']//a[contains(@href,'.php')]"
>>> extractLinks "//table[@class='tabelka']//td[@width='68%']/a[contains(@href,'.php')]" >>> extractLinks "//table[@class='tabelka']//td[@width='68%']/a[contains(@href,'.php')]"
@ -55,23 +58,23 @@ extractRecordLinks = extractLinks "//table[@class='tabelka']//a[contains(@href,'
>>> extractRecordData >>> extractRecordData
toShadowItem :: (String, String) -> ShadowItem toShadowItem :: (String, String, String) -> ShadowItem
toShadowItem (url, title) = toShadowItem (url, recordTitle, recordDate) =
(defaultShadowItem url title) { (defaultShadowItem url title) {
originalDate = Just date, originalDate = Just date,
itype = "periodical", itype = "periodical",
format = Just "pdf", format = Just "pdf",
finalUrl = url finalUrl = url
} }
where title = title where title = recordTitle
date = "12-34-5678" date = recordDate
main = do main = do
let start = "https://archiwa.pilsudski.org/dokument.php?nonav=0&nrar=701&nrzesp=4&sygn=4&handle=701.180/16036" let start = "https://archiwa.pilsudski.org/"
let shadowLibrary = ShadowLibrary {logoUrl=Nothing, let shadowLibrary = ShadowLibrary {logoUrl=Nothing,
lname="Archiwa Pilsudski", lname="Archiwa Pilsudski",
abbrev="ArchPil", abbrev="ArchPil",
lLevel=0, lLevel=0,
webpage=start} webpage=start}
extractItemsStartingFromUrl shadowLibrary start (extractRecordData) extractItemsStartingFromUrl shadowLibrary start (extractRecordLinks >>> arr toShadowItem)