s444409/twilight-library
1
0
Fork 0
forked from filipg/twilight-library
Code Issues Pull Requests Projects Releases Wiki Activity

Biblioteka sejmowa crawler

Browse Source
This commit is contained in:
Marcin Kostrzewski 2022-04-19 20:41:35 +02:00
parent 1d5b7e6b73
commit 9d298652a2
1 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
app/bibliotekasejmowa.hs
View File

@ -4,31 +4,31 @@ import ShadowLibrary.Core
import Text.XML.HXT.Core import Text.XML.HXT.Core
import Text.XML.HXT.XPath import Text.XML.HXT.XPath
-- import Text.XML.HXT.Curl import Text.XML.HXT.Curl
import Data.List import Data.List
import Data.List.Utils (replace) import Data.List.Utils (replace)
import Text.Regex.Posix import Text.Regex.Posix
import Text.Printf import Text.Printf
extractAllPDFs = extractLinksWithText "/html/body/table[4]/tbody/tr[7]/td[2]/a"
extractRecords = extractLinksWithText "//a[@class='roczniki']" -- pary adres-tytuł extractRecords = extractLinksWithText "//table[3]//a"
>>> second (arr $ replace "\r\n " " ") -- czyścimy drugi element pary, czyli tytuł z niepotrzebnych białych znaków >>> first (extractLinksWithText "//table[3]//a"
>>> first (arr ((++"tr") . init)) -- modyfikujemy pierwszy element pary, czyli adres URL >>> first (extractLinksWithText "//table[4]//a"
>>> first (extractLinksWithText "//li/a[contains(@href,'.pdf')]") -- pobieramy stronę z adresu URL i wyciągamy linki z tej strony pasujące do wyrażenia XPathowego >>> first (extractAllPDFs
-- ostatecznie wyjdą trójki ((adres URL, tytuł artykułu), tytuł rocznika) >>> first(arr $ (drop 23 ) . init . init . init))))
-- ... a tutaj te trójki przerabiamy do docelowej struktury ShadowItem toShadowItem :: ((((String, String), String), String), String) -> ShadowItem
toShadowItem :: ((String, String), String) -> ShadowItem toShadowItem ((((url, typeName), articleTitle), _), category) =
toShadowItem ((url, articleTitle), yearlyTitle) =
(defaultShadowItem url title) { (defaultShadowItem url title) {
originalDate = Just date, originalDate = Just date,
itype = "periodical", itype = "periodical",
format = Just "pdf", format = Just "pdf",
finalUrl = url finalUrl = url
} }
where title = "Biblioteka Sejmowa " ++ yearlyTitle ++ " " ++ (replace "\r\n" "" (replace "\r\n " "" articleTitle)) where title = "Biblioteka Sejmowa " ++ category ++ " " ++ typeName ++ " " ++ articleTitle
date = getDate url date = getDate articleTitle
getDate url = getDate url =
case url =~~ "/(19[0-9][0-9]|20[0-9][0-9])/" :: Maybe [[String]] of case url =~~ "/(19[0-9][0-9]|20[0-9][0-9])/" :: Maybe [[String]] of
@ -37,7 +37,7 @@ getDate url =
main = do main = do
let start = "https://biblioteka.sejm.gov.pl/zasoby_cyfrowe/" let start = "https://bs.sejm.gov.pl/F/?con_lng=POL&func=file&file_name=find-nowe&local_base=ars01"
let shadowLibrary = ShadowLibrary {logoUrl=Nothing, let shadowLibrary = ShadowLibrary {logoUrl=Nothing,
lname="Biblioteka Sejmowa", lname="Biblioteka Sejmowa",
abbrev="BiblSejm", abbrev="BiblSejm",