39 lines
1.3 KiB
Python
39 lines
1.3 KiB
Python
import scrapy
|
|
from scrapy.spiders import CrawlSpider, Rule
|
|
from scrapy.linkextractors import LinkExtractor
|
|
import re
|
|
from items import MovieItem
|
|
|
|
|
|
class FinaSpider(CrawlSpider):
|
|
name = 'repozytorium.fn.org.pl'
|
|
start_urls = ['http://repozytorium.fn.org.pl/?q=pl/fnsearch/film_index/a']
|
|
|
|
rules = (
|
|
# Extract link from index of titles
|
|
Rule(LinkExtractor(restrict_xpaths='//*[@id="content"]/div/div[2]')),
|
|
|
|
# Extract links with movie titles
|
|
Rule(LinkExtractor(restrict_xpaths='//*[@id="content"]/div/div[4]/div', deny='\?q=pl\/search\/site'), callback='parse_item'),
|
|
|
|
Rule(LinkExtractor(restrict_xpaths='//*[@id="content"]/div/div[4]/div', allow='\?q=pl\/search\/site')),
|
|
|
|
Rule(LinkExtractor(restrict_xpaths='//*[@id="fntiles_page_search_0"]'), callback='parse_item')
|
|
)
|
|
custom_settings = {
|
|
'FEED_EXPORT_ENCODING': 'utf-8'
|
|
}
|
|
|
|
|
|
def parse_item(self, response):
|
|
|
|
def getMp4(text):
|
|
x = re.findall('file: encodeURI\("(.+?\.mp4)"\)',text)
|
|
return x
|
|
|
|
item = MovieItem()
|
|
item['title'] = response.xpath('//*[@id="block-fnfilm-fnfilm"]/div/div[1]/div/div/span/text()').get()
|
|
item['mp4'] = getMp4(response.xpath('//*[@id="block-fnfilm-fnfilm"]/div/div[2]/div/script/text()').get())
|
|
item['url'] = response.url
|
|
return item
|