Compare commits

..

No commits in common. "master" and "master" have entirely different histories.

6 changed files with 8 additions and 106 deletions

View File

@ -33,7 +33,7 @@ import Data.Tree.NTree.TypeDefs
import Data.Maybe
import Control.Monad.Trans
import Text.XML.HXT.XPath
import Text.XML.HXT.Curl
-- import Text.XML.HXT.Curl
import Text.XML.HXT.HTTP
import Text.Regex.TDFA
@ -64,8 +64,8 @@ downloadDocument = readFromDocument [withParseHTML yes,
withEncodingErrors no,
withPreserveComment yes,
withStrictInput yes,
-- withHTTP []
withCurl [("curl--user-agent","AMU Digital Libraries Indexing Agent")]
withHTTP []
-- withCurl [("curl--user-agent","AMU Digital Libraries Indexing Agent")]
]
downloadDocumentWithEncoding enc = readFromDocument [withParseHTML yes,
@ -73,13 +73,13 @@ downloadDocumentWithEncoding enc = readFromDocument [withParseHTML yes,
withEncodingErrors no,
withPreserveComment yes,
withInputEncoding enc,
-- withHTTP []]
withCurl []]
withHTTP []]
-- withCurl []]
downloadXmlDocument = readFromDocument [withWarnings no,
withEncodingErrors no,
-- withHTTP []]
withCurl [] ]
withHTTP []]
-- withCurl [] ]
data ShadowLibrary = ShadowLibrary { logoUrl :: Maybe String,

View File

@ -4,7 +4,7 @@ import ShadowLibrary.Core
import Text.XML.HXT.Core
import Text.XML.HXT.XPath
--import Text.XML.HXT.Curl
-- import Text.XML.HXT.Curl
import Data.List
import Data.List.Utils (replace)

View File

@ -1,67 +0,0 @@
{-# LANGUAGE Arrows, NoMonomorphismRestriction #-}
import ShadowLibrary.Core
import Text.XML.HXT.Core
import Text.XML.HXT.XPath
import Text.XML.HXT.Curl
import Data.List
import Data.List.Utils (replace)
import Text.Regex.Posix
import Text.Printf
extractH xpathCondition = (downloadDocument &&& this)
>>> first(getXPathTrees xpathCondition
>>> (listA (deep isText >>> getText)
>>> arr (intercalate " ")
))
extractInfo xpathCondition = (downloadDocument &&& this)
>>> first (getXPathTrees xpathCondition
>>> (getAttrValue "href"
&&& (getXPathTrees "//a[@class='msl-box-link']//div[@class='msl-box-title']"
>>> (listA (deep isText >>> getText))
>>> arr (intercalate " ")
)
&&& (getXPathTrees "//a[@class='msl-box-link']//div[@class='msl-box-info']"
>>> (listA (deep isText >>> getText))
>>> arr (intercalate " ")
)
&&& (getXPathTrees "//a[@class='msl-box-link']//div[@class='time']"
>>> (listA (deep isText >>> getText))
>>> arr (intercalate " ")
)
))
extractRecords = extractInfo "//a[@class='msl-box-link']"
toShadow :: ((String, (String, (String, String))), String) -> ShadowItem
toShadow ((url, (title, (author, year))), s) =
ShadowItem {
url = Just url,
title = title,
originalDate = Just year,
creator = Just author,
format = Just s,
itype = "periodical",
lang = Nothing,
finalUrl = url,
description = Nothing
}
main = do
--let start = "https://zasoby.msl.org.pl/mobjects/show"
let start = "https://zasoby.msl.org.pl/mobjects/show?date_range=&otype%5B4%5D=1&otype%5B5%5D=1&otype%5B6%5D=1&otype%5B7%5D=1" --przefiltrowane
let shadowLibrary = ShadowLibrary {logoUrl=Nothing,
lname="Zasoby MSL",
abbrev="MSL",
lLevel=0,
webpage=start}
extractItemsStartingFromUrl shadowLibrary start (extractRecords >>> arr toShadow)

Binary file not shown.

View File

@ -59,18 +59,6 @@ executable almanachmuszyny
, shadow-library
default-language: Haskell2010
executable msl
hs-source-dirs: app
main-is: msl.hs
ghc-options: -threaded -rtsopts -with-rtsopts=-N
build-depends: base
, hxt
, hxt-xpath
, MissingH
, regex-posix
, shadow-library
default-language: Haskell2010
source-repository head
type: git

View File

@ -1,19 +0,0 @@
# This file was autogenerated by Stack.
# You should not edit this file by hand.
# For more information, please see the documentation at:
# https://docs.haskellstack.org/en/stable/lock_files
packages:
- completed:
hackage: hxt-xpath-9.1.2.2@sha256:9cd590ae93a04573db8f90fa4094625ebd97dded45da7667c577ce6b38a42900,1999
pantry-tree:
size: 2225
sha256: aee2f75974e868ff429b8ff349a29667536c60397098f5dfedc968d1951511bb
original:
hackage: hxt-xpath-9.1.2.2
snapshots:
- completed:
size: 507596
url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/11/9.yaml
sha256: 42f472dbf06482da1b3319241f3e3b3593a45bd7d4f537d2789f21386b9b2ad3
original: lts-11.9