import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
import re
from items import MovieItem

from fina_scrap.items import MovieItem

class FinaSpider(CrawlSpider):
    name = 'repozytorium.fn.org.pl'
    start_urls = ['http://repozytorium.fn.org.pl/?q=pl/fnsearch/film_index/a']

    rules = (
        # Extract link from index of titles
        Rule(LinkExtractor(restrict_xpaths='//*[@id="content"]/div/div[2]')),

        # Extract links with movie titles
        Rule(LinkExtractor(restrict_xpaths='//*[@id="content"]/div/div[4]/div', deny='\?q=pl\/search\/site'), callback='parse_item'),

        Rule(LinkExtractor(restrict_xpaths='//*[@id="content"]/div/div[4]/div', allow='\?q=pl\/search\/site')),

        Rule(LinkExtractor(restrict_xpaths='//*[@id="fntiles_page_search_0"]'), callback='parse_item')
    )
    custom_settings = {
        'FEED_EXPORT_ENCODING': 'utf-8'
    }

    def parse_item(self, response):

        def getMp4(self, text):
            x = re.findall('file: encodeURI\("(.+?\.mp4)"\)',text)
            return x
        
        item = MovieItem()
        item['title'] = response.xpath('//*[@id="block-fnfilm-fnfilm"]/div/div[1]/div/div/span/text()').get()
        item['mp4'] = self.getMp4(response.xpath('//*[@id="block-fnfilm-fnfilm"]/div/div[2]/div/script/text()').get())
        item['url'] = response.url
        return item