Small changes to annotator.py (to be deleted in near future) Add utils/iterator Add redis to enviroment.yml Rename, adapt and move rule based extractor. Adapt find_hours. Yapify webapp app (probalby nothing more) Rename buttons in index.html
161 lines
5.7 KiB
Python
Executable File
161 lines
5.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from utils import iterator
|
|
from colorama import Fore, Back, Style
|
|
import os
|
|
import jsonlines
|
|
import re
|
|
import pprint
|
|
import sys
|
|
|
|
|
|
class Extractor:
|
|
def __init__(self, page):
|
|
"docstring"
|
|
self.page = page
|
|
self.content = page['content']
|
|
self.header = self.wrap_with_name_group(
|
|
'header',
|
|
'porządek mszy (świętych|św|św\.)|msz[ea][ \n]+([śs]wi[eę]t[ea]|św|św\.)'
|
|
)
|
|
|
|
self.sunday_title = self.wrap_with_name_group(
|
|
'sunday_title',
|
|
'niedziel[a|e][ \n]+i[ \n]+(dni[ \n]+(świąteczne|św|św\.)|święta)'
|
|
'|niedziel[ea]'
|
|
'|porządek świąteczny')
|
|
#'|święta'
|
|
self.sunday_masses = self.wrap_with_name_group(
|
|
'sunday_masses', '.*[^\d]\d{1,2}[^\d].*?')
|
|
self.everyday_title = self.wrap_with_name_group(
|
|
'everyday_title', 'dzień powszedni'
|
|
'|dni powszednie'
|
|
'|w tygodniu'
|
|
'|porządek zwykły'
|
|
'|od poniedziałku do soboty')
|
|
self.everyday_masses = self.wrap_with_name_group(
|
|
'everyday_masses',
|
|
'(.*?[^\d\n]?\d{1,2}[^\d\n]?.*?\n)+') # \n lub koniec stringa
|
|
|
|
def wrap_with_name_group(self, name, pattern):
|
|
return '(?P<{}>{})'.format(name, pattern)
|
|
|
|
def extract(self, search_space=None):
|
|
if search_space == '':
|
|
return None
|
|
if not search_space:
|
|
search_space = self.content
|
|
header_match = re.search(self.header, search_space, re.I)
|
|
if not header_match:
|
|
return None
|
|
search_space = search_space[header_match.end():]
|
|
|
|
sunday_title_match = re.search(self.sunday_title, search_space, re.I)
|
|
if not sunday_title_match:
|
|
return None
|
|
if re.search(self.header, search_space[:sunday_title_match.start()],
|
|
re.I): # found header closer to sunday title
|
|
return self.extract(search_space)
|
|
if sunday_title_match.start(
|
|
) > 50: #sunday_title za daleko header'a wiec szukaj dalej
|
|
try:
|
|
return self.extract(search_space[sunday_title_match.end():])
|
|
except Exception:
|
|
import ipdb
|
|
ipdb.set_trace()
|
|
|
|
everyday_title_match = re.search(self.everyday_title, search_space,
|
|
re.I)
|
|
if not everyday_title_match:
|
|
return None
|
|
sunday_masses_hours = search_space[sunday_title_match.end():
|
|
everyday_title_match.start()]
|
|
if not re.search(self.sunday_masses, sunday_masses_hours,
|
|
re.DOTALL | re.I):
|
|
return None
|
|
if len(sunday_masses_hours) > 500:
|
|
return self.extract(search_space[sunday_title_match.end():])
|
|
everyday_masses_match = re.search(
|
|
self.everyday_masses, search_space[everyday_title_match.end():],
|
|
re.I)
|
|
if not everyday_masses_match:
|
|
return None
|
|
if everyday_masses_match.start() > 150:
|
|
return self.extract(search_space[sunday_title_match.end():])
|
|
|
|
whole_result = header_match.group(
|
|
0) + search_space[:everyday_masses_match.end() +
|
|
everyday_title_match.end()]
|
|
groups = (header_match.group(0), sunday_title_match.group(0),
|
|
sunday_masses_hours, everyday_title_match.group(0),
|
|
everyday_masses_match.group(0))
|
|
# print(whole_result)
|
|
# print(groups)
|
|
# obsłużyć # TODO:
|
|
# w dni powszednie (w roku szkolnym) - górny kościół
|
|
# 6:30, 7:00, 8:00, 18:00
|
|
# w dni powszednie (czas wakacji) - górny kościół
|
|
# 7:00, 8:00, 18:00
|
|
|
|
print('url: {}\ndepth: {}\nbutton: {}'.format(
|
|
self.page['url'], self.page['depth'], self.page['button_text']))
|
|
sys.stdout.flush()
|
|
return whole_result, groups
|
|
|
|
|
|
def process_directory(directory):
|
|
found = 0
|
|
not_found = 0
|
|
for root, dirs, files in os.walk(directory):
|
|
for fname in sorted(files):
|
|
filepath = os.path.join(root, fname)
|
|
if os.path.getsize(filepath) > 0:
|
|
if process_parish(iterator.parish_page_iterator(filepath)):
|
|
print(filepath)
|
|
found += 1
|
|
else:
|
|
not_found += 1
|
|
print('found: {}\nnot_found: {}'.format(found, not_found))
|
|
else:
|
|
pass # empty file
|
|
|
|
|
|
def color_match(whole_match, groups, background, colors, style):
|
|
for i in range(len(groups)):
|
|
whole_match = whole_match.replace(
|
|
groups[i], colors[i] + background + style + groups[i] +
|
|
Style.RESET_ALL + background + style, 1)
|
|
return whole_match + Style.RESET_ALL
|
|
|
|
|
|
def process_parish(reader):
|
|
for page in sorted(reader, key=lambda x: x['depth']): #sort by depth
|
|
extractor = Extractor(page)
|
|
result = extractor.extract()
|
|
if result:
|
|
whole_result, groups = result
|
|
if whole_result not in page['content']:
|
|
pass
|
|
import ipdb
|
|
ipdb.set_trace()
|
|
pretty_text = page['content'].replace(
|
|
whole_result,
|
|
color_match(whole_result, groups, Back.BLACK, [
|
|
Fore.RED, Fore.GREEN, Fore.YELLOW, Fore.MAGENTA, Fore.CYAN
|
|
], Style.BRIGHT))
|
|
# print(pretty_text)
|
|
# print(page['depth'])
|
|
# print(page['url'])
|
|
# print(page['button_text'])
|
|
# import ipdb
|
|
# ipdb.set_trace()
|
|
return True
|
|
return False
|
|
|
|
|
|
def main():
|
|
process_directory('./parishwebsites/data')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|