78 lines
2.4 KiB
Perl
78 lines
2.4 KiB
Perl
|
#!/usr/bin/python
|
||
|
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
import sys, re, codecs, heapq
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
n_largest_param = 10
|
||
|
|
|
||
|
|
html_pattern = r'<\/?(a|abbr|acronym|address|applet|area|b|base|basefont|bdo|big|blockquote|body|br|button|caption|center|cite|code|col|colgroup|dd|del|dir|div|dfn|dl|dt|em|fieldset|font|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|hr|html|i|iframe|img|input|ins|isindex|kbd|label|legend|li|link|map|menu|meta|noframes|noscript|object|ol|optgroup|option|p|param|pre|q|s|samp|script|select|small|span|strike|strong|style|sub|sup|table|tbody|td|textarea|tfoot|th|thead|title|tr|tt|u|ul|var|xmp).*?>'
|
||
|
|
date_pattern = r'[0-9]{1,2}[\.\-/][0-9]{1,2}[\.\-/][0-9]{4}'
|
||
|
|
email_pattern = r'[\w\._\d]+@\w+(\.\w+)*'
|
||
|
|
number_pattern = r'[0-9]+([\.\,][0-9]+)?'
|
||
|
|
|
||
|
|
|
||
|
|
corpus_file = codecs.open(sys.argv[1], encoding='utf-8')
|
||
|
|
|
||
|
|
sentences = []
|
||
|
|
total_sentences = 0
|
||
|
|
empty_sentences = 0
|
||
|
|
|
||
|
|
print "Reading and hashing corpus file..."
|
||
|
|
for line in corpus_file:
|
||
|
|
total_sentences += 1
|
||
|
|
line = re.sub(html_pattern, '', line, re.UNICODE)
|
||
|
|
|
||
|
|
line = re.sub(date_pattern, '[DATE]', line, re.UNICODE)
|
||
|
|
line = re.sub(email_pattern, '[EMAIL]', line, re.UNICODE)
|
||
|
|
line = re.sub(number_pattern, '[NUMBER]', line, re.UNICODE)
|
||
|
|
|
||
|
|
words = re.findall(r'\[[A-Z]+\]|\w+', line, re.UNICODE)
|
||
|
|
if any(re.match(r'\w+', word, re.UNICODE) for word in words):
|
||
|
|
sentences.append(" ".join([word.lower() for word in words]))
|
||
|
|
else:
|
||
|
|
empty_sentences += 1
|
||
|
|
|
||
|
|
if total_sentences % 10000 == 0:
|
||
|
|
print " processed %d sentences" % total_sentences
|
||
|
|
|
||
|
|
|
||
|
|
corpus_file.close()
|
||
|
|
print "Corpus file read."
|
||
|
|
|
||
|
|
|
||
|
|
print "Sorting the corpus"
|
||
|
|
sentences.sort()
|
||
|
|
|
||
|
|
print "Computing unique values..."
|
||
|
|
|
||
|
|
prev_sentence = None
|
||
|
|
curr_count = 1
|
||
|
|
|
||
|
|
ul = []
|
||
|
|
|
||
|
|
for i,sentence in enumerate(sentences):
|
||
|
|
if sentence == prev_sentence:
|
||
|
|
curr_count += 1
|
||
|
|
else:
|
||
|
|
if prev_sentence is not None:
|
||
|
|
ul.append((prev_sentence, curr_count))
|
||
|
|
curr_count = 1
|
||
|
|
prev_sentence = sentence
|
||
|
|
|
||
|
|
#append the last sentence
|
||
|
|
ul.append((prev_sentence, curr_count))
|
||
|
|
|
||
|
|
#print ul
|
||
|
|
print "Unique values computed."
|
||
|
|
|
||
|
|
print "Total sentences %d, empty: %d" % (total_sentences, empty_sentences)
|
||
|
|
print "Among %d non-empty sentences there are %d unique sentences" % (total_sentences - empty_sentences, len(ul))
|
||
|
|
print "%d most common sentences:" % n_largest_param
|
||
|
|
|
||
|
|
for sentence, count in heapq.nlargest(n_largest_param, ul, key = lambda x:x[1]):
|
||
|
|
print "%d occurences of sentence: %s" % (count, sentence)
|
||
|
|
|
||
|
|
|
||
|
|
|