s434760/twilight-library
1
0
Fork 0
forked from filipg/twilight-library
Code Issues Pull Requests Releases Wiki Activity

Funkcja do wydobycia roku z url, funkcja do wydobycia url i tekstu

Browse Source
This commit is contained in:
Anna Nowak 2021-04-06 23:25:55 +02:00
parent a7bf508ff4
commit 3e9ad1fa0b
1 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
app/ptd.hs
View File

@ -1,5 +1,7 @@
{-# LANGUAGE UTF #-}
{-# LANGUAGE Arrows, NoMonomorphismRestriction #-} {-# LANGUAGE Arrows, NoMonomorphismRestriction #-}
import ShadowLibrary.Core import ShadowLibrary.Core
import Text.XML.HXT.Core import Text.XML.HXT.Core
@ -10,27 +12,35 @@ import Data.List.Utils (replace)
import Text.Regex.Posix import Text.Regex.Posix
import Text.Printf import Text.Printf
extractNestedLinksWithText xpathCondition = (downloadDocumentWithEncoding "UTF-8" &&& this)
>>> first (getXPathTrees xpathCondition
>>> ((getXPathTrees "//a" >>> getAttrValue "href")
&&& (listA (deep isText >>> getText)
>>> arr (intercalate " "))))
>>> arr rotateSecTh
>>> first expandURIFixed
extractRecords = extractLinksWithText "//div[@class='entry-content']/p[count(a)=1]/a" extractRecords = extractLinksWithText "//div[@class='entry-content']/p/a[contains(@href, 'id')]"
>>> second (arr $ replace "\r\n " " ") >>> first (extractNestedLinksWithText "//div[@class='entry-content']/p[strong[a]] | //div[@class='entry-content']/p[a]")
>>> first (extractLinksWithText "//li/a[contains(@href,'.pdf')]")
-- ... a tutaj te trójki przerabiamy do docelowej struktury ShadowItem
toShadowItem :: ((String, String), String) -> ShadowItem toShadowItem :: ((String, String), String) -> ShadowItem
toShadowItem ((url, articleTitle), yearlyTitle) = toShadowItem ((url, articleTitle), magazineTitle) =
(defaultShadowItem url title) { (defaultShadowItem url title) {
originalDate = Just date, originalDate = Just date,
itype = "periodical", itype = "periodical",
format = Just "pdf", format = Just "pdf",
finalUrl = url finalUrl = url
} }
where title = "Almanach Muszyny " ++ yearlyTitle ++ " " ++ (replace "\r\n" "" (replace "\r\n " "" articleTitle)) where title = magazineTitle ++ " - " ++ (replace " \8211 pdf" "" articleTitle)
date = getDate url date = getYear url
getDate url = getYear :: String -> String
case url =~~ "/(19[0-9][0-9]|20[0-9][0-9])/" :: Maybe [[String]] of getYear url =
case url =~~ "/rocznik[0-9]{2}/" :: Maybe [[String]] of
Just [[_, year]] -> year Just [[_, year]] -> year
otherwise -> error $ "unexpected url: " ++ url otherwise -> case url =~~ "/(19[0-9][0-9]|20[0-9][0-9])/" :: Maybe [[String]] of
Just [[_, year]] -> "19" ++ (replace "rocznik" "" year)
otherwise -> ""
main = do main = do
@ -40,4 +50,4 @@ main = do
abbrev="ptd", abbrev="ptd",
lLevel=0, lLevel=0,
webpage=baseUrl} webpage=baseUrl}
extractItemsStartingFromUrl shadowLibrary baseUrl (extractRecords >>> arr toShadowItem) extractItemsStartingFromUrl shadowLibrary baseUrl (extractRecords >>> arr toShadowItem)