final

app/miastobierun.hs

@@ -9,38 +9,32 @@ import Data.List.Utils (replace)
 import Text.Regex.Posix
 import Text.Printf
 
---extractCustomLinks xpathCondition1 = (downloadDocument &&& this)
---                                       >>> first (getXPathTrees xpathCondition1
---                                                  >>> (getAttrValue "href"
---                                                       &&& getAttrValue "title"
---                                                       ))
---                                       >>> arr rotateSecTh
---                                       >>> first expandURIFixed
-
-extractCustomLinks xpathCondition1 xpathCondition2 xpathCondition3= (downloadDocument &&& this)
-                                       >>> first (getXPathTrees xpathCondition1
-                                                  >>> first (getXPathTrees xpathCondition2 >>> (getAttrValue "href" &&& getAttrValue "title"))
-                                                  >>> second (getXPathTrees xpathCondition3 >>> getAttrValue "cell-header")
-                                                 )
-                                       >>> arr rotateSecThX
+extractLinksWithTitleAndText xpathCondition = (downloadDocument &&& this)
+                                       >>> first (getXPathTrees xpathCondition
+                                                  >>> (getAttrValue "href"
+                                                       &&& getAttrValue "title"
+                                                       &&& (listA (deep isText >>> getText)
+                                                           >>> arr (intercalate " "))))
+                                       >>> arr rotateSecTh
                                        >>> first expandURIFixed
-                                       >>> second expandURIFixed
 
-extractRecords = extractCustomLinks "//tr[@class=''] | //tr[@class='_a']" "//td[@cell-header='Nazwa pliku']/a" "//td[@class='size']"
---extractRecords = extractCustomLinks "//td[@cell-header='Nazwa pliku']/a"
+--extractRecords = extractCustomLinks "//tr[@class=''] | //tr[@class='_a']" "//td[@cell-header='Nazwa pliku']/a" "//td[@class='size']"
+extractRecords = extractLinksWithTitleAndText "//td[not(contains(@cell-header,'Nazwa pliku'))]/a[contains(@href,'.pdf') and @target='DownloadWin']"
 
-toShadowItem :: (String, String, String) -> ShadowItem
-toShadowItem (url, size, articleTitle) =
+toShadowItem :: (String, (String, String)) -> ShadowItem
+toShadowItem (url, (info, articleTitle)) =
   (defaultShadowItem url title) {
     originalDate = Just date,
     itype = "periodical",
-    format = Just "pdf",
     finalUrl = url,
-    description = Just desc
+    format = format,
+    description = Just desc,
+    lang = Just "pol"
     }
-  where title = "Miasto Bierun: " ++ " url: " ++ url ++ " title: " ++ articleTitle
-        date = getDate articleTitle
-        desc = getArticleNr articleTitle ++ " size: "
+  where title = replace "\r\n       \r\n        Plik: " "" articleTitle
+        desc = "Article nr: " ++ getArticleNr title ++ " Size: " ++ getFileSize info
+        date = getDate title
+        format = extractFormat url
 
 getDate title =
   case title =~~ "(19[0-9][0-9]|20[0-9][0-9])" :: Maybe [[String]] of
@@ -49,9 +43,13 @@ getDate title =
 
 getArticleNr title =
   case title =~~ "([0-9][0-9]/)" :: Maybe [[String]] of
-    Just [[_, nr]] -> "Article nr: " ++ (replace "/" "" nr)
+    Just [[_, nr]] -> replace "/" "" nr
     otherwise -> "No article nr for: " ++ title
 
+getFileSize title =
+  case title =~~ "(Rozmiar: [0-9]+.[0-9]+[A-Z]+)" :: Maybe [[String]] of
+    Just [[_, size]] -> replace "Rozmiar: " "" size
+    otherwise -> "No file size for: " ++ title
 
 main = do
     let start = "https://www.bierun.pl/mieszkancy/archiwum"