ISI-twilight/ShadowLibrary/Core.hs
Natalia Gawron a5d9f23ca8 my solution
2021-04-02 16:57:47 +02:00

305 lines
12 KiB
Haskell

{-# LANGUAGE QuasiQuotes, TemplateHaskell, TypeFamilies, OverloadedStrings #-}
{-# LANGUAGE GADTs, FlexibleContexts #-}
{-# LANGUAGE Arrows, NoMonomorphismRestriction #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE GeneralizedNewtypeDeriving #-}
module ShadowLibrary.Core where
-- import Database.Persist
-- import Database.Persist.TH
-- import Database.Persist.Postgresql
import Control.Monad.IO.Class (liftIO)
import Control.Monad.Logger (runNoLoggingT)
import Control.Monad.Trans.Resource (runResourceT)
import Control.Monad.Reader
import Data.Time
import Data.Time.Calendar
import Data.Time.LocalTime
import Data.Time.Zones
import Data.Time.Zones.All
import Data.String.Utils (strip)
import Data.List.Utils as DLU
import Text.XML.HXT.Core
import Network.HTTP
import Network.URI
import Control.Monad.Trans.Maybe
import Control.Monad
import Data.Tree.NTree.TypeDefs
import Data.Maybe
import Control.Monad.Trans
import Text.XML.HXT.XPath
import Text.XML.HXT.Curl
import Text.XML.HXT.HTTP
import Text.Regex.TDFA
import Data.List (isInfixOf, intercalate)
import Data.List.Utils (replace)
-- import Network.Curl.Opts
polishTimeZone = TimeZone {
timeZoneMinutes = 120,
timeZoneSummerOnly = True,
timeZoneName = ""}
openUrl :: String -> MaybeT IO String
openUrl url = case parseURI url of
Nothing -> fail ""
Just u -> liftIO (getResponseBody =<< simpleHTTP (mkRequest GET u))
getWebPage :: String -> IO (IOSArrow XmlTree (NTree XNode))
getWebPage url = do
contents <- runMaybeT $ openUrl url
return $ readString [withParseHTML yes, withWarnings no] (fromMaybe "" contents)
downloadDocument = readFromDocument [withParseHTML yes,
withWarnings no,
withEncodingErrors no,
withPreserveComment yes,
withStrictInput yes,
-- withHTTP []
withCurl [("curl--user-agent","AMU Digital Libraries Indexing Agent")]
]
downloadDocumentWithEncoding enc = readFromDocument [withParseHTML yes,
withWarnings no,
withEncodingErrors no,
withPreserveComment yes,
withInputEncoding enc,
-- withHTTP []]
withCurl []]
downloadXmlDocument = readFromDocument [withWarnings no,
withEncodingErrors no,
-- withHTTP []]
withCurl [] ]
data ShadowLibrary = ShadowLibrary { logoUrl :: Maybe String,
lname :: String,
abbrev :: String,
webpage :: String,
lLevel :: Int }
data ShadowItem = ShadowItem {
url :: Maybe String,
title :: String,
itype :: String,
originalDate :: Maybe String,
creator :: Maybe String,
format :: Maybe String,
lang :: Maybe String,
finalUrl :: String,
description :: Maybe String
} deriving (Show)
defaultShadowItem url title = ShadowItem {
url = Just url,
title = title,
itype = "periodical",
originalDate = Nothing,
creator = Nothing,
format = Just "pdf",
lang = Just "pol",
finalUrl = url,
description = Nothing }
getDuration :: Maybe String -> (Maybe UTCTime, Maybe UTCTime)
getDuration Nothing = (Nothing, Nothing)
getDuration (Just date) =
case date =~~ ("^(1[6789]|20)[0-9][0-9]$" :: String) of
Just year -> (Just (yearStart year), Just (yearEnd year))
otherwise ->
case date =~~ ("^((1[789]|20)[0-9][0-9])-(0[1-9]|1[0-2])$" :: String) :: Maybe [[String]] of
Just [[_, year, _, month]] -> (Just (monthStart year month), Just (monthEnd year month))
otherwise ->
case date =~~ ("^((1[789]|20)[0-9][0-9])-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])$" :: String) :: Maybe [[String]] of
Just [[_, year, _, month, day]] -> (Just (dayStart year month day), Just (dayEnd year month day))
otherwise ->
case date =~~ ("^((1[789]|20)[0-9][0-9])-((1[789]|20)[0-9][0-9])$" :: String) :: Maybe [[String]] of
Just [[_, yearb, _, yeare, _]] -> (Just (yearStart yearb), Just (yearEnd yeare))
otherwise -> (Nothing, Nothing)
where
yearAsInteger year = (read year) :: Integer
monthAsInt month = (read month) :: Int
dayAsInt day = (read day) :: Int
yearStart year = localTimeToTimeStamp $ LocalTime {
localDay = fromGregorian (yearAsInteger year) 1 1,
localTimeOfDay = midnight }
yearEnd year = localTimeToTimeStamp $ LocalTime {
localDay = fromGregorian (yearAsInteger year) 12 31,
localTimeOfDay = lastSecond }
monthStart year month = localTimeToTimeStamp $ LocalTime {
localDay = fromGregorian (yearAsInteger year) (monthAsInt month) 1,
localTimeOfDay = midnight }
monthEnd year month = localTimeToTimeStamp $ LocalTime {
localDay = fromGregorian (yearAsInteger year) (monthAsInt month) 31,
localTimeOfDay = lastSecond }
dayStart year month day = localTimeToTimeStamp $ LocalTime {
localDay = fromGregorian (yearAsInteger year) (monthAsInt month) (dayAsInt day),
localTimeOfDay = midnight }
dayEnd year month day = localTimeToTimeStamp $ LocalTime {
localDay = fromGregorian (yearAsInteger year) (monthAsInt month) (dayAsInt day),
localTimeOfDay = lastSecond }
lastSecond = TimeOfDay {todHour = 23, todMin = 59, todSec = 59 }
localTimeToTimeStamp ltime = localTimeToUTCTZ (tzByLabel Europe__Warsaw) ltime
-- zonedTimeToUTC $ ZonedTime {
-- zonedTimeToLocalTime = ltime,
-- zonedTimeZone = timeZoneForUTCTime Europe__Warsaw }
extractLinks xpathCondition = (downloadDocument &&& this)
>>> first (getXPathTrees xpathCondition
>>> getAttrValue "href")
>>> expandURIFixed
extractLinksGeneralized xpathCondition attr = (downloadDocument &&& this)
>>> first (getXPathTrees xpathCondition
>>> getAttrValue attr)
>>> expandURIFixed
rotateSecTh ((a, b), c) = ((a, c), b)
extractLinksWithText xpathCondition = (downloadDocument &&& this)
>>> first (getXPathTrees xpathCondition
>>> (getAttrValue "href"
&&& (listA (deep isText >>> getText)
>>> arr (intercalate " "))))
>>> arr rotateSecTh
>>> first expandURIFixed
extractLinksWithTitle xpathCondition = (downloadDocument &&& this)
>>> first (getXPathTrees xpathCondition
>>> (getAttrValue "href"
&&& getAttrValue "title"))
>>> arr rotateSecTh
>>> first expandURIFixed
urlPreFixer = arr (Data.List.Utils.replace "[" stupidLeftBracketMarker . Data.List.Utils.replace "]" stupidRightBracketMarker)
urlPostFixer = arr (Data.List.Utils.replace stupidLeftBracketMarker "%5B" . Data.List.Utils.replace stupidRightBracketMarker "%5D")
stupidLeftBracketMarker = "ddsfdfdfdfgfgfrrtrtrrtr"
stupidRightBracketMarker = "wqweweerererrtrtrtrtrtr"
expandURIFixed = (urlPreFixer *** urlPreFixer) >>> expandURI >>> urlPostFixer
extractText = (listA (deep isText >>> getText)
>>> arr (intercalate " "))
loopNext extract xpathConditionForNext = initialStep
>>> loopNextCore extract xpathConditionForNext
>>> arr fst
>>> unlistA
initialList :: [(String, a)]
initialList = []
initialStep = arr (const initialList) &&& this
loopNextCore extract xpathConditionForNext = second
(listA extract &&& extractNext xpathConditionForNext)
>>> arr expandList
>>> ifP (nextFound)
(second (arr fromJust >>> (downloadDocument &&& this))
>>> loopNextCore extract xpathConditionForNext)
(this)
nextFound :: ([(String,a)], Maybe String) -> Bool
nextFound (_, Just _) = True
nextFound (_, Nothing) = False
expandList (l, (e, n)) = (l ++ e, n)
extractNext xpathConditionForNext = listA
(first (getXPathTrees ("(" ++ xpathConditionForNext ++ ")[1]")
>>> getAttrValue "href")
>>> expandURIFixed)
>>> arr listToMaybe
extractFormat :: String -> Maybe String
extractFormat finalUrl
| ".gif" `isInfixOf` finalUrl = Just "gif"
| ".jpg" `isInfixOf` finalUrl = Just "jpg"
| ".djvu" `isInfixOf` finalUrl = Just "djvu"
| ".pdf" `isInfixOf` finalUrl = Just "pdf"
| ".doc" `isInfixOf` finalUrl = Just "doc"
| otherwise = Nothing
baseMonthNameToNumber :: String -> Maybe String
baseMonthNameToNumber "styczeń" = Just "01"
baseMonthNameToNumber "styczen" = Just "01"
baseMonthNameToNumber "stycznia" = Just "01"
baseMonthNameToNumber "luty" = Just "02"
baseMonthNameToNumber "lutego" = Just "02"
baseMonthNameToNumber "marzec" = Just "03"
baseMonthNameToNumber "marca" = Just "03"
baseMonthNameToNumber "kwiecień" = Just "04"
baseMonthNameToNumber "kwiecien" = Just "04"
baseMonthNameToNumber "kwietnia" = Just "04"
baseMonthNameToNumber "maj" = Just "05"
baseMonthNameToNumber "maja" = Just "05"
baseMonthNameToNumber "czerwiec" = Just "06"
baseMonthNameToNumber "czeerwiec" = Just "06"
baseMonthNameToNumber "czerwca" = Just "06"
baseMonthNameToNumber "lipiec" = Just "07"
baseMonthNameToNumber "lipca" = Just "07"
baseMonthNameToNumber "sierpień" = Just "08"
baseMonthNameToNumber "sierpien" = Just "08"
baseMonthNameToNumber "sierpnia" = Just "08"
baseMonthNameToNumber "wrzesień" = Just "09"
baseMonthNameToNumber "wrzesien" = Just "09"
baseMonthNameToNumber "września" = Just "09"
baseMonthNameToNumber "wrzesnia" = Just "09"
baseMonthNameToNumber "październik" = Just "10"
baseMonthNameToNumber "pażdziernik" = Just "10"
baseMonthNameToNumber "pazdziernik" = Just "10"
baseMonthNameToNumber "października" = Just "10"
baseMonthNameToNumber "pazdziernika" = Just "10"
baseMonthNameToNumber "listopad" = Just "11"
baseMonthNameToNumber "listopada" = Just "11"
baseMonthNameToNumber "grudzień" = Just "12"
baseMonthNameToNumber "grudzien" = Just "12"
baseMonthNameToNumber "grudnia" = Just "12"
baseMonthNameToNumber "jesien" = Just "10"
baseMonthNameToNumber _ = Nothing
extractYear :: String -> Maybe String
extractYear n =
case n =~~ ("(1[6789]|20)[0-9][0-9]" :: String) of
Just year -> Just year
otherwise -> Nothing
joinAlts :: [String] -> String
joinAlts = intercalate " // "
clean = arr (DLU.replace "\n" "")
>>> arr strip
extractItems shadowLibrary start extractor = do
page <- getWebPage start
items <- runX $ extractor start page
-- insertIntoDatabase shadowLibrary items
putStrLn (show items)
extractItemsStartingFromUrl shadowLibrary start extractor = do
items <- runX $ (arr (const start) >>> setTraceLevel 1 >>> extractor)
-- insertIntoDatabase shadowLibrary items
mapM_ (putStrLn . show) items