198 lines
3.3 KiB
Perl
Executable File
198 lines
3.3 KiB
Perl
Executable File
#! /usr/bin/env perl
|
|
|
|
use locale;
|
|
#use strict;
|
|
|
|
#
|
|
##################################################
|
|
$linesPerFile = 20000;
|
|
|
|
if (@ARGV < 1) {
|
|
print "usage: prep_user_dict.pl dictionary_file\n";
|
|
exit;
|
|
}
|
|
|
|
$file = shift; # @ARGV;
|
|
|
|
# Przygotowanie etykiet
|
|
|
|
`makeLabels.pl > labels.sym`;
|
|
|
|
`lexmakelab labels`;
|
|
|
|
# Analiza pliku s³ownika
|
|
|
|
|
|
print "Kanonizujê opisy.........................................";
|
|
|
|
`canon.pl <$file >temp1`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Sortujê plik.............................................";
|
|
|
|
`sort -t \\~ -k1,1 -k2,2nr <temp1 > temp2`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Minimalizujê plik s³ownika...............................";
|
|
|
|
#`rmDup.pl < temp2 > temp1`;
|
|
#`rmDup2.pl < temp1 > temp2`;
|
|
|
|
`cp temp1 temp2`;
|
|
|
|
`rm temp1`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Czyszczê pliki...........................................";
|
|
|
|
`sed -r "s/([[:punct:]])/[\\1]/g" < temp2 > temp1`;
|
|
|
|
`cp temp1 temp2`;
|
|
`rm temp1`;
|
|
|
|
print "OK\n";
|
|
|
|
#dzielimy plik na wiele czê¶ci, uruchamiamy lexcomplex dla ka¿dej
|
|
#czê¶ci osobno, nastêpnie ³±czymy to za pomoc± programu fsmunion
|
|
|
|
print "Dzielê s³ownik na mniejsze czê¶ci........................";
|
|
|
|
open(IN, "./temp2");
|
|
|
|
$lineCount = 0;
|
|
$fileCount = 0;
|
|
|
|
`mkdir LemTEMP`;
|
|
|
|
open(FILE, ">LemTEMP/slo_0");
|
|
|
|
while (<IN>) {
|
|
|
|
if (++$lineCount >= $linesPerFile) {
|
|
$fileCount++;
|
|
$lineCount = 0;
|
|
|
|
close(FILE);
|
|
# print "Tworzê nowy plik tymczasowy: slo_".$fileCount."\n";
|
|
open(FILE, ">LemTEMP/slo_".$fileCount);
|
|
}
|
|
|
|
print(FILE $_);
|
|
}
|
|
|
|
print "OK\n";
|
|
|
|
print "Tworzê automaty po¶rednie";
|
|
|
|
#32 kropki, fileCount plikow
|
|
$filesPerDot = $fileCount/32;
|
|
$files=$filesPerDot;
|
|
$dots=0;
|
|
|
|
for ($i=0; $i<=$fileCount; $i++) {
|
|
|
|
if ($files >= $filesPerDot) {
|
|
$files = 0;
|
|
print ".";
|
|
$dots++;
|
|
}
|
|
$files++;
|
|
|
|
$command = "lexcomplex -l labels.lab -S labels.scl < LemTEMP/slo_".$i." > LemTEMP/slownik_".$i.".fsm";
|
|
|
|
`$command`;
|
|
|
|
}
|
|
if ($dots < 32) {
|
|
for ($i=0; $i<32 - $dots; $i++) {
|
|
print ".";
|
|
}
|
|
}
|
|
|
|
print "OK\n";
|
|
|
|
`rm LemTEMP/slo_*`;
|
|
|
|
print "Tworzê automat koñcowy";
|
|
|
|
#35 kropek...
|
|
$filesPerDot = $fileCount/35;
|
|
$files=$filesPerDot;
|
|
$dots=0;
|
|
|
|
`cp LemTEMP/slownik_0.fsm slownik1.fsm`;
|
|
|
|
for ($i=1; $i<=$filecount; $i++) {
|
|
|
|
if ($files >= $filesPerDot) {
|
|
$files = 0;
|
|
print ".";
|
|
$dots++;
|
|
}
|
|
$files++;
|
|
|
|
$command = "fsmunion LemTEMP/slownik_".$i." slownik1.fsm > slownik2.fsm";
|
|
|
|
`$command`;
|
|
|
|
`mv slownik2.fsm slownik1.fsm`;
|
|
}
|
|
|
|
if ($dots < 35) {
|
|
for ($i=0; $i<35 - $dots; $i++) {
|
|
print ".";
|
|
}
|
|
}
|
|
|
|
`fsmunion LemTEMP/* > slownik1.fsm`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Usuwam epsilon-przejscia.................................";
|
|
|
|
`fsmrmepsilon slownik1.fsm > slownik2.fsm`;
|
|
|
|
`rm slownik1.fsm`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Determinizujê automat....................................";
|
|
|
|
`fsmdeterminize slownik2.fsm > slownik1.fsm`;
|
|
|
|
`rm slownik2.fsm`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Minimalizujê automat.....................................";
|
|
|
|
`fsmminimize slownik1.fsm > slownik.fsm`;
|
|
|
|
`rm slownik1.fsm`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Konwertujê automat do formatu fsa........................";
|
|
|
|
`fsmprint -i labels.lab slownik.fsm > slownik.txt`;
|
|
|
|
`../fsm2aut slownik.txt > slownik.aut`;
|
|
|
|
`../aut2fsa < slownik.aut > gue.bin`;
|
|
|
|
print "OK\n";
|
|
|
|
print "Czyszczê pliki pomocnicze................................";
|
|
|
|
`rm LemTEMP/*`;
|
|
`rmdir LemTEMP`;
|
|
`rm temp2`;
|
|
`rm slownik.fsm`;
|
|
`rm slownik.txt`;
|
|
`rm slownik.aut`;
|
|
|
|
print "OK\n";
|