filipg/twilight-library
1
0
Fork 51
Code Issues Pull Requests Releases Wiki Activity

Simplify XPath queries

Browse Source
This commit is contained in:
Norbert Litkowski 2021-04-07 14:33:45 +02:00
parent 4ae00c6660
commit 67344b3d61
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/ZborBielawa.hs
View File

@ -18,7 +18,7 @@ import qualified Data.Text as T
getLinkAndText xpathCondition = proc doc -> do
xpathTrees <- getXPathTrees xpathCondition -< doc
name <- getElemName -< xpathTrees
txt <- (listA (deep isText >>> getText) >>> arr (intercalate " ")) -< xpathTrees
txt <- (getXPathTrees "//a/../text()" >>> listA (deep isText >>> getText) >>> arr (intercalate " ")) -< xpathTrees
href <- (getXPathTrees "//a" >>> getAttrValue "href") -< xpathTrees
returnA -< (href, txt)
@ -31,10 +31,12 @@ extractNestedLinksWithText xpathCondition = proc url -> do
extractRecords = proc startUrl -> do
-- (catUrl, catText) <- extractLinksWithText "(//aside[@class='widget widget_maxmegamenu']//a[@class='mega-menu-link'])[1]" -< startUrl -- pary adres-tytuł podstrony
-- (catUrl, catText) <- extractLinksWithText "(//aside[@class='widget widget_maxmegamenu']//a[@class='mega-menu-link'])[2]" -< startUrl -- pary adres-tytuł podstrony
(catUrl, catText) <- extractLinksWithText "//aside[@class='widget widget_maxmegamenu']//a[@class='mega-menu-link']" -< startUrl -- pary adres-tytuł podstrony
(collUrl, collText) <- (extractLinksWithText "//aside[@class='widget widget_maxmegamenu']//a[@class='mega-menu-link']") -< catUrl -- pobieramy podstronę kategorii i kolejne podstrony z menu
(relUrl, relText) <- (extractNestedLinksWithText "//big[a[contains(@href,'.pdf') and contains(text(), 'Pobierz PDF')]]") -< collUrl -- pobieramy stronę z wydaniami z danej kolekcji i linki do PDFów
(collUrl, collText) <- (extractLinksWithText "(//aside[@class='widget widget_maxmegamenu']//a[@class='mega-menu-link'])[1]") -< catUrl -- pobieramy podstronę kategorii i kolejne podstrony z menu
-- (collUrl, collText) <- (extractLinksWithText "//aside[@class='widget widget_maxmegamenu']//a[@class='mega-menu-link']") -< catUrl -- pobieramy podstronę kategorii i kolejne podstrony z menu
(relUrl, relText) <- (extractNestedLinksWithText "(//big[a[contains(@href,'.pdf')]])[1]") -< collUrl -- pobieramy stronę z wydaniami z danej kolekcji i linki do PDFów
-- (relUrl, relText) <- (extractNestedLinksWithText "//big[a[contains(@href,'.pdf')]]") -< collUrl -- pobieramy stronę z wydaniami z danej kolekcji i linki do PDFów
returnA -< (((relUrl, relText), collText), catText) -- ostatecznie wyjdą krotki (((adres URL PDFa wydania, tytuł wydania), tytuł zbioru), tytuł kategorii)
-- ... a tutaj te krotki przerabiamy do docelowej struktury ShadowItem