444409
This commit is contained in:
parent
a2e68a1c0b
commit
3f7d5b269e
800
dev-0/out.tsv
Normal file
800
dev-0/out.tsv
Normal file
@ -0,0 +1,800 @@
|
||||
i i i to to the the the the the waste waste waste
|
||||
in the of people people of the in in the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the was of the the the the the the the the the the the the the the the the the and and and and and the the the
|
||||
in nuclear be be be be be plant plant nuclear
|
||||
the the of the the the of and and and and and and and the the that that the the the the the the the the the the of of of of of of of
|
||||
we must not say
|
||||
he was the the the the was the
|
||||
we must to to our our our our our and
|
||||
the the the the the the the the the the the the the the also
|
||||
the representatives of will will to to to next next next the the the
|
||||
the german of the the the the the the the not not not not not not not to the the the the the the the the the the the the the
|
||||
the of of and and and and and also also that that that the the the the the the the and and and and and and and and and and and and
|
||||
in the herzegovina and and and and the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
croatia is in in in is is is is is is is is
|
||||
we cannot be to to them
|
||||
the law of the the the the the the
|
||||
the legislation are the the the
|
||||
in the the the the the commission commission commission commission to to the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the commission will will be to and and and and and and the
|
||||
it is the the the the the the the the the the the the and and and and and and and and and and and and and and and and and and and and and and and and and and and and and and
|
||||
the commission will will commission to to to and and and and
|
||||
the commission has a independent independent the the the authorities authorities
|
||||
the the the the the the the the the the the the the of of of of of of of of of of of of of of of of of of of of of of the the the the the the the the the the the the the the the
|
||||
the the commission will will to to to to of of to education education and and and and and
|
||||
the the will not not the members members members members members members members members members parliament parliament parliament the
|
||||
the of of the the the the the the the the the the the the the the the the the the the the the the the the the in in in in in in in the the the the the the the the the the the women children children children
|
||||
the members of of the the the the the the to to to the the the the of of of of
|
||||
the the of the the of the the the the the the to to to the the the the the the the the
|
||||
the commission the the commission will will will the the the the the the the the majority majority
|
||||
the new will will will will will
|
||||
the will will will will will will to to to the the the the the the the the the the the the
|
||||
according the of the the the to to to to to of of of of of the the the the the the
|
||||
the will be be be the the the or or or or or or
|
||||
the commission is is to to to to to the the the the the the the the the
|
||||
in the the the of the the the the the the the the the the the the the information information
|
||||
the commission will also to to the in the the the of of of or or or or or
|
||||
the first of the the commission commission and and and and the the the the the the the the the
|
||||
the united to to the the of of of
|
||||
the by by by by by by by the the the the the the the the the the the the the the the the the the the the that that that the the the the the the the the the
|
||||
the decision decision the was was is is is political political political
|
||||
the the of the the the the i i i i i to the the the the the the the the the the the the the the the the the the
|
||||
the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the of of of
|
||||
the proposed of the the eur eur eur eur eur eur eur eur eur eur eur eur eur eur eur eur the and and and and and and and and and and and and and and
|
||||
the of of of the the the will will will will will the the the the the the the the the the the and and and and and and and
|
||||
the of of the the the the of of of of of the the the the the the the
|
||||
the was in in in in in in in the the the the the the the the the
|
||||
the the of the will will will be be will the the the the the the the
|
||||
in the the the programme the the the the the
|
||||
the one of of of of of of the the the the to to to and and and and and and
|
||||
as you have the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the of of of of million than than than than than than than than than than is is is which which which
|
||||
the the the that that the the the the the the the the the the the the the the the the the the the the
|
||||
the the of of the of the the the the the the the will will will will the the the the
|
||||
what is a the of the the the the the
|
||||
children and and and and to they to to to to to to
|
||||
the work is now now the work
|
||||
the is a president president the the the the the the the the the must must the the the the
|
||||
in a the the of of the and and and and and and and
|
||||
it of not not have of have have have have to to to to to to the the the the the the the the the the the the the the the the the the the the the the the
|
||||
there are a of of of of the the the the the
|
||||
the is of that that that that that who are are are
|
||||
it is also also and and and and and and and and and violence violence violence
|
||||
this is a of of of is of of the the the the the and is is is is is is is is
|
||||
we are a the of the the
|
||||
however this is is a of of the to to to the of
|
||||
the the of the the the the to to to is
|
||||
you the or the the
|
||||
the the of of the a to to to to to the the the the the that that that that to to
|
||||
children for of of of children children children children and and and
|
||||
how often often often often
|
||||
it is difficult difficult difficult because because because because because because because because is are the the
|
||||
the latest of the the year that two two two two two two two two 000 000 000
|
||||
what can be be be if that that that that be be be be
|
||||
the should be be be the the the the the the the the the the to the the the the the the the the of
|
||||
the has a police police in children children children children children
|
||||
the the police police police in in in in in in in in in in the the in children children children children children
|
||||
the italian of that the the in in in in in in in in
|
||||
the who to the the the the the the the the the the the are are the the
|
||||
in children children children children children parents parents
|
||||
children children children children children and and and and and and are
|
||||
we have also very the the and and and and and and and and the the the the the the the the the the school
|
||||
it is not because because because because they because because are are are are and
|
||||
the was the the the the the has has has to to children children children children children to
|
||||
what children children children and and and
|
||||
however the the of the the the has has the the the the the the the the to to
|
||||
they are for of of of or or or or or
|
||||
many have have have to to to the that that that that
|
||||
we is why the the of the is to
|
||||
china is is a important for
|
||||
it is a a a of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the in in
|
||||
the group of has has has has the the the the the years years years years
|
||||
in the the in in in in in in in in in the to to to to to to to to to to and and and
|
||||
the the the the the the the the the the the the the the the the the the the to to to to
|
||||
we can new new new new new new new new
|
||||
the has to the the the the the the the the the and and and and and and and and
|
||||
he has has to to to to to to the the the and and and
|
||||
de mr president the the of the the in in the the that that that that the the
|
||||
if the the of of the the the the will will will be be
|
||||
the german of the the the the the the the the the the the the the the the the the the the the the the the the the and and and and and and and and and
|
||||
the the of of the of the the the the the the that that the the the the the the the the the the the the the a a a a
|
||||
in the of the the of the the the the the the the the is that is is quality quality quality quality quality quality
|
||||
it is of young young young people people people people young young young to to to to to and and and and and
|
||||
the of of want want want
|
||||
the of of of to to the the the the the the the the the the the
|
||||
the is the the the the the the the the the to to to to the of of of the and and and and and
|
||||
i is very that to to to to the the the the the the the the the the 11 11 11 11 11 11 11 11 11 11
|
||||
the of of the the the the the and and and and and
|
||||
it is the the the the the the the the the in in in in and and and and and and and and and the the the the the the
|
||||
after the the that that that the the the the the the the the the the the the the the the the the
|
||||
the the of was been the the the the the the the the the the the of of
|
||||
the is of is is is one one
|
||||
it
|
||||
there have a a a
|
||||
the has has in in in 2008 2008
|
||||
they is more more more than
|
||||
the also the the the and and and and and and and and and the the the
|
||||
many are have been as as as such such as
|
||||
the mr the the the the
|
||||
we have a few to to have to to to to to to
|
||||
we must that that that have to to to to to and and and and and and
|
||||
it is that that to to to to to is
|
||||
it must be to to and and and to to to
|
||||
as as as as the the the the the the
|
||||
i me to your your
|
||||
i you you you you to to the the the the the the the the the was was was was was was was
|
||||
it is the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
so is therefore to to example example example example example example example as
|
||||
there is of the the of the the the the
|
||||
this is the the the the
|
||||
the of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the aim of the the of the the the and and and and and and and and and and the the the the the the the the the the the the
|
||||
in week last the the the the the the the and and and and and the the the the the
|
||||
we are a our our our our our we we we we we we we we we our our our the the the the the the the and and and and and and and and and and and and and and and and
|
||||
we we to to to to our our our our we we we we we we we we we we we we we we the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
we can be to to the the the the the the the the and and and and and and and and we to to to to to to to the the the the the the the the the
|
||||
the is of a a for for for for for for the the the the the
|
||||
the the of the will will to to the the the the the the the and and and and
|
||||
the writing of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
in the the the the the of the the the the the the the the the the the the and and and and and and and and and
|
||||
the will will be or
|
||||
the the that and and and to to to to to to the the
|
||||
the the of the of the the the the the the the the the the the
|
||||
it is is for
|
||||
you can be be the the the or or or or or or or or or
|
||||
they is not not to to money to
|
||||
the is is a serious problem the the
|
||||
he has also about about
|
||||
the is a a the
|
||||
there only only only only only only only that
|
||||
the prime of of the the the the the the the of
|
||||
why is no for to to to to not not to
|
||||
we are us us us to
|
||||
i think that is is that that that that that that that
|
||||
we us not to this problem this
|
||||
the the of of of of of the the to to
|
||||
as was was was was was was was to to to
|
||||
it is to to the the the the the
|
||||
the the of the the the the the the the the
|
||||
at the the the the the the the the the the the the the
|
||||
in thursday the the of the the of and and and and
|
||||
mr week we to to to the the the the and and and
|
||||
the and and and and will be be be the
|
||||
the of of of have to to
|
||||
in year this to to to to to to to to to the the in in in in in in in in in in in in
|
||||
according the the the the the the the the the the of 000 000 000 000
|
||||
the who be be be to the the the the the the
|
||||
it also also that that the who who who the the the the the the the the the the the the the the the the the the the the
|
||||
in the the the the and and the the the the the the the the that that the the the the the
|
||||
the was was was was of of of of of of
|
||||
the children of children to to to to to to the the
|
||||
in have the the of of of of the the in and and and and and and in in in in in in in
|
||||
then then then to to and
|
||||
since a a a in in in in in in in in in in in in in in in in
|
||||
the was been to to the the the the the the and and and and and and and the the the the the the the the the the the the
|
||||
the month later the a the the the the and and and and and and the the the the the and and and and
|
||||
if the the the the the the and and and to the the
|
||||
no schulz
|
||||
i will not not i i i
|
||||
the was of the the the and and and and and and and and and and and and and and
|
||||
when it the the the school school school school
|
||||
the has a to the the 30 30 30 30 30 and and and
|
||||
the was was and and and and and and and and and the the the the the the the the the the the the the the the in in
|
||||
the will will the the will will will will
|
||||
the report calls the the the of the the the the of of of of
|
||||
the president of the the of of and and and and and and and and and and and and and the the of of of of of of of of the the the the of of of of of of of of of
|
||||
the was was to to the the the the the the the the the the the the and and and and and and and and
|
||||
the of of of of of the the the the the the the the the the the the the the
|
||||
this is this this this that that that the the the the the the the the the the the the the the the
|
||||
the the in in and and in in in in in in in in in in in in in in in in in in the the the the the the the the the the the the the the the the the the the the the the media media media media
|
||||
as the the the the the the the the the the the the the the the the the the the the the the the the the the the the and
|
||||
however it is that that that that that the the the the to to the the the the the the of of of
|
||||
the progress progress more more more more more even even even even even more and and
|
||||
if a a of the the the the would would a a a a a a a a to to to the the the the the the the the
|
||||
the the of the the the the the the the the the the the the the the the the the the the the the the
|
||||
he was was the the the in in
|
||||
mr was a a the the the
|
||||
the 2 2 2 2 the the the the the the the and and and and and the the the the
|
||||
however the the of of the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the was was was was the the the the the the the the the the the the the
|
||||
the the and and and and are are to to to the the the the the the the the the
|
||||
this example example example example example example the the the the and and the
|
||||
however this is a the the the the the the the the the the the the the the the the the the the the the
|
||||
in the the of the the was was was the the the the the the and and the the the the the the the the the
|
||||
the the of of of the the the the the the and and and and and and and and and and and and and and and and
|
||||
the second point the the the the the the the the the the the the a a a the the the the the the the the the the the
|
||||
mr minister was was was the a the the the the the the the the to to to to to to
|
||||
at the the of of of the the the the that that be be be
|
||||
he has has the the the the the the the the the
|
||||
he was was was the the the the was was was
|
||||
the of of of not not not the the the the the the the the the the the the the the the the the the the the
|
||||
the secretary of the the the the the the the the the the the the the the the the the the the the the
|
||||
the report report the report the the the the the the the the the the the the the the the the the the the the the
|
||||
the is is is the is of the the the the the the the of of of of of of the the the the
|
||||
as the the i the the the the not
|
||||
as you who who who who who who who who who he he he he he he he he the the and and and and and and and and and
|
||||
the is is the the that that that that that that that the the the the the the the the the the the the
|
||||
he was been in in in in the was was was was was was the
|
||||
the is to to to the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the has has has that that that that to to the the the the the the the the the the the
|
||||
the has has the the of the political
|
||||
the
|
||||
yesterday some have have have have to to the the the the the the of of of of
|
||||
after the of of the the the the the
|
||||
in the of of a a the of of
|
||||
the the of the the of of of of the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
some few later later the the the the the in in in in in in in in in
|
||||
the of of the the of the the
|
||||
the 30 30 30 30 30 and and and and and and the the the the the the
|
||||
in the the the the the to to to to to to to to to the the
|
||||
the of of of of of of of of of
|
||||
the was of the the the the the the
|
||||
the has has that that has been years years the and and and and and and the europe europe europe europe
|
||||
the also also also to to to
|
||||
the 10 of of the the the the the the the the the
|
||||
all all of of the the in the the
|
||||
the are a 3 10 000 000 000 were
|
||||
the south of has a a of of of
|
||||
in the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the have on the the the the the the in in
|
||||
the court has has the the the the the the the the the the
|
||||
the the that the the the the the the the the
|
||||
as the the the the the the the the the the the the
|
||||
at the the the the the the the the that that that that the the the and and and
|
||||
it is that that the in in in in the the
|
||||
the the the of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
unfortunately one of of of and and and and and and and the the
|
||||
mr president the the the the the to the the the the the the the the the and and and and and and and and and and and
|
||||
we are to eur eur billion usd usd the usd the
|
||||
the has to to the the the the the the the the the the the the the the the the the the the the the the
|
||||
the commission of the the the the the the the the the to to to to to medicines
|
||||
in the the the the the the the the the of of of of of of of of of of of
|
||||
in president it has has that the the the the the the the the
|
||||
in the the the the the that that that that not not not not not
|
||||
is is is the
|
||||
what is is is is
|
||||
the is of of of of of
|
||||
this is a that to to and and and and and difficult
|
||||
we have
|
||||
we we we heard hear that
|
||||
why it is the and and and
|
||||
the of of the the the the
|
||||
when a the ago the was the the the the the
|
||||
no is no or the the and to to to
|
||||
they are not that that who children children children the the the the the the
|
||||
it it not not to to
|
||||
however after many of of of the the the the the the the the the
|
||||
we has to to the the the
|
||||
today is is a of of and and and the the the the the many
|
||||
the fact that that that that has that
|
||||
it is difficult difficult difficult to to
|
||||
we are a the of
|
||||
we are us that that this
|
||||
if we know that is is is and and and and and and and and is is is is is is is
|
||||
he is a the two two two two
|
||||
the has has has that that that that that to to to to the the the the the the the the the and and and and and and and and
|
||||
the years years the the the and and and years years years
|
||||
the has has in in in in in in in in in in in in in in in in in in in in in
|
||||
however the of however been the the the the the the the the the the the the the
|
||||
the is of of of the is are not
|
||||
the has of of the the the the the of of of
|
||||
the two two two two two two two two two and and and of of of
|
||||
the the the the have have the the the the the the
|
||||
the of of of the the the two two and and
|
||||
four of of of the the the four four four four and
|
||||
the the of to to to to to 24 24
|
||||
all all of are than than than than
|
||||
the the of the the the i i i i to to to to to to to the the the
|
||||
the people who the to to to
|
||||
subject of of
|
||||
the to to to the of and and and and and to to to to
|
||||
i am sure that that is is be be to to to to to to to to
|
||||
this will will will will will to to to the the the to to to
|
||||
the was was be
|
||||
what the is
|
||||
i is the the i i i i i to to to to the the the
|
||||
the reason is is is is the the the the
|
||||
one of of of of of is is is is the the the the the the
|
||||
it are well aware the the the and and and and
|
||||
it is the and and and and and and and and
|
||||
as you the the the the the the the the
|
||||
we have to that that the the the the the the the the the the the the the
|
||||
in
|
||||
in have the the the the the the
|
||||
the 5 programme a 5 5 5 5 5 5 and
|
||||
the of of is not not not to
|
||||
the have been a problems problems problems problems problems problems problems problems
|
||||
it has also the economic economic economic economic economic the the and and and and and not not
|
||||
climate world world world world world
|
||||
the of of of young young in in in in in in the in in in in
|
||||
the was to to to to to to the the the the climate climate climate climate climate climate climate climate
|
||||
the have to the the the the the the the the the the the the the the
|
||||
we
|
||||
the
|
||||
we we we
|
||||
now now
|
||||
the is to the the the the the the the the the the the the the
|
||||
the police police authorities authorities be be to to the the the the the the the the the the the the the the the the the
|
||||
the has has the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
it is of how how to to the to to to to to the
|
||||
it is not that that that that been
|
||||
now is therefore to to to to to this
|
||||
the has has the the the the the the the the the and and and
|
||||
this was a a the the the the the the the the the the the the the the the the the the the police police police police
|
||||
it is is to to to in in the the the the
|
||||
as said said said the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the is that that that the the the the years years years years years years years years years years the the the the the the the the the the the the the the the the the the the the the
|
||||
in the the the the the the the the the the the the of 000 000 000 000
|
||||
the irish are have to to the the the procedure the the the the
|
||||
the german german the the the the the the the the the the and and and that the the the
|
||||
however as the the of the the the the the in in in in in in in in in in
|
||||
the of of the the minister the the the the
|
||||
the decision of the the the the the the the the the the the the the the
|
||||
according the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the has has no been a the the the the and and and and and and to to to to to to to to to to to a the the the the the the the
|
||||
without the the the society not not not not to political political
|
||||
as the ago ago ago have have have have that to the the the the the the the the the the the the the the the
|
||||
the were of the the the the the the the the the the the the the the the the the the the the the the was was was
|
||||
the of of the the the of of the of of the
|
||||
the of of of the the the the the the the the the the the the the the the the and and and and and and the
|
||||
the the of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
it was a a the the the the to to the
|
||||
the amendment of the the that to to to to the the the the
|
||||
the has in in in in in in in in in in in in in in in the
|
||||
the of of of has not not not the the the the or or or or or or or or to
|
||||
i was the the the the the the the the in in in in in in in in in in in the the the the the the the of
|
||||
the is the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the response to to to the the the the not not
|
||||
the was was the the the the the the the and and and the
|
||||
the is of the and the the the the the and and and and the the the the the the the
|
||||
the the of the has has the the the the years years years years years in in
|
||||
the the of of the the the the the a the the and and and and and and the the
|
||||
one is a of of of the the the the the the the the
|
||||
the the the the the the the the the the the the the the the the the and and and and and and
|
||||
the was the the the the the the not not not not
|
||||
the 10 10 more more more more of
|
||||
the is is a a of of of of the the the of of the the
|
||||
the is of to to to and and and and and and or
|
||||
in more more more more more more more more
|
||||
in is is is is is the the the
|
||||
the is is is at when when when the the
|
||||
we is the the the a a a a
|
||||
we was been what the said said
|
||||
in writing it the a the the the the the the the the of of
|
||||
the the of the to to to to to to to to to to and and and
|
||||
the mr of for that that that to in in in in in
|
||||
he was been years was was was was was was was and
|
||||
he was been and and and
|
||||
however the of of of of the the the the years years years years and and and and the the the the
|
||||
in was and and and and was was was was was was
|
||||
i i said said said said i i i i i that to to to to to to own own own
|
||||
i my words i i i i i i i i i not not
|
||||
as the has the the the the the the own own own own own the the the the the the the the the the the the the the the the
|
||||
the future of the to to to to to the the the the of
|
||||
one is of is is is
|
||||
the is a the of of people people people and and and the the the the the the the
|
||||
the have have have have have the the the the the the the the the the the in
|
||||
in in in in in in in in in in the the the the the the
|
||||
in the and and in in in in in and and and and and and and and and and and a a
|
||||
you have the the the the in in to to to to
|
||||
the the the the the the the the the that that the the the the the the the the the the the the the the the the the the the
|
||||
in the that to to to to to the the
|
||||
the the of the the the the the the the the the the the the the that that that that that that that the the the the the the the the the the the the
|
||||
the was a a the the the the that the that that not
|
||||
he was was been and and and and and and
|
||||
the has of the the the the the the the the the the the the the the the the the the the the the the of of of of of
|
||||
the german and and and and and have to to to to the the
|
||||
it is the the of the the the the of of of the the the the the the the the the the of of of and and and and and and and and and and
|
||||
the has also the and and and and and the the the the the the the
|
||||
the who not not not to to the the the the the the
|
||||
the the not not not not to the the the the the the
|
||||
the the the the the the the was that the the the the the the was was was was was was the the the the the
|
||||
in the the the the the is is the the the the the the the the the the the the the the the the the the the
|
||||
it has that that that to to to to and and and and the the the the the
|
||||
according the the the the the the the the to to the the the the the and the the the the the the the the the
|
||||
the the of that that that that that that that the the the the the the the the the the the the
|
||||
the was a the the the to to to the and and and and and to to to to
|
||||
no has not been any
|
||||
in the to the the the to to to to to to to to to the the the the the the the
|
||||
as the was was that that that the the the the the the the but but but but the the the
|
||||
you want you to you you what the
|
||||
i will to to
|
||||
the the the the the not not not a the the on information information information information information
|
||||
the first of the new new new
|
||||
in will will be be to to the the the the the the the the the the the and the the a a a a
|
||||
as said said mr the the the the the the the the the the the the the the the the the the the the the the the the the to
|
||||
in than more more more more more more than than than than than
|
||||
the 1 1 1
|
||||
the and and and and and and and and and and and and and and and and and and
|
||||
in is the the the the and and and and and and and and and and and
|
||||
the is way the the the the the and and and and and and and and and and and
|
||||
the is of the the the of the the
|
||||
the is of to to to to to the the the the the the and and been already
|
||||
the new of of new new new new new new of of of of the the the the the the the the the the
|
||||
the cost of of of of of of of
|
||||
the quality of quality the be be be be the the 10
|
||||
the 3 3 3 3 3 3 the of of of of of of of of which the the the the the the the the the and and and and and and
|
||||
the the of of the the the the the the the the the the the the the the the and and and and and and and and and the the the the the the the the the the the
|
||||
the main of of is is is 5 5 5 5
|
||||
the is is the the the the the the the the the the the and and and
|
||||
the 1 more more 1 1 billion billion billion
|
||||
in this of of of of of of of of and and and and of of of of
|
||||
in the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the the the the the the of of of of of the the the the the the the the of of of of of of
|
||||
our our is our
|
||||
in president the the the the the the the the the the
|
||||
i want to to to to to to to to to the the the the the the the the the the
|
||||
the is of the the the the the the the the the the the
|
||||
in more more more more more more more more more more more more
|
||||
it is be be that that that people people people people people people people people people people people who who who who to
|
||||
the recent of the the the programme programme programme programme has years years years years years
|
||||
we have that we we we we
|
||||
the is of and and and and and that that that that that not not not not not not
|
||||
we have told that that that money money not not have have have have have have have
|
||||
the is also about the the the and and and and and and and also also also also also
|
||||
in addition the the of of of the the the the and and and and and
|
||||
the money not not be be only the the the but but but the the the the and and and and and and and and and and the the the
|
||||
i us to the the the
|
||||
mr president the mr mr mr his his his his his his his his he he he he he he he he he
|
||||
mr was the the of the the the the the the
|
||||
the prime the minister minister minister minister minister was in the in in in in
|
||||
all of all of the of the
|
||||
the mr as as as as as as as
|
||||
the years years years the the have have the the the have
|
||||
it it and it it it it it or or or or or all
|
||||
this was this for the the the
|
||||
it is a great us us our our our our and and and and
|
||||
the has has that the the the the the the the the the the the the the the the the peace peace peace peace peace peace peace peace peace peace peace
|
||||
the the the the the the the the the the the the the
|
||||
in writing the palestinian palestinian of of the
|
||||
mr will will be to a the the the the the will will will will to to to to
|
||||
the czech of the the to to to to to to to to the the the the who who who the the the the the the
|
||||
he was been the of the the the the in in in in
|
||||
the president the the the of
|
||||
the will in be be the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the of the the the the the the the the the the and and and and and and
|
||||
the of of the the the the
|
||||
the march 24 the the the the the the the the
|
||||
the main of is eur eur eur eur million million million 000 000 000 000 000 000 the the to to to to to to to and and and and and and and and and and
|
||||
the measures to measures to to to to to to the the the the the the the the the the the the the and and and
|
||||
the has has the the the the the and and and and and and
|
||||
in the the the the the will will the the the the the the the the the the the will will will will will will will will will will
|
||||
the the of the the the the to to to to the the the the the the the the
|
||||
in president of the of of the the the the the the the the the the the the the the the the the the the and and and and and and and and and and
|
||||
the is the of the the the the to to
|
||||
in the the the the of the the the the the the the to to the of of of the the the and and and to to to to to to
|
||||
it will be be be and and and and and the the the the the the the the the in in in in in in in in in in
|
||||
the new new of the will will will will will will in
|
||||
the the of of of of of the the the the the the the the the the the the the public public public public the the the the the to to to
|
||||
the of of of of of many many many many the the the health health health health health the
|
||||
the is of that that to to to to to the quality quality quality and and and and and and
|
||||
the is that to to to the the and and and be be be be be
|
||||
it has that that that that companies companies to to to to to to and and and and and and and and to to
|
||||
it was that that it it it to to to to to to to to to to the the the the the the the the the the the the the the the the the
|
||||
as the case case the be be be be be be be or or or and and and and and and
|
||||
the have of the the of the the the the the the but we we we we and and and and and
|
||||
the a the the the the the the is has has has has has the the the and and and and and
|
||||
we believe that is is is is is is is to to to to to to to experience experience or or or or or or or
|
||||
we want to to to to to
|
||||
in the not the the the to to the the the the the the the the the the the the the the the the the the the
|
||||
the 1 is a a the
|
||||
the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
it was that that the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
we call to to to to to to to to to the the the the the the the the the
|
||||
the is of the the the the the
|
||||
this this this the will will will will to to to to the the the the the the the the the
|
||||
the the the the the the the to to to to to to to to to to to to the the the the the the the the the the the the the the
|
||||
in the the the the the the the the the the the the the the the the the the waste waste waste
|
||||
the of of of of of of of of of of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
in the of the the the the the the the of of of of of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the of of of of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the of of the the the the the
|
||||
i have that the the the the the the the the the the the the the the the the the the the the the
|
||||
in the of of of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
according the the the the the is is the the the in in the the the to to
|
||||
approximately 000 000 000 in in in in
|
||||
in 8 8 8 8 of people of in the the the and and the the the
|
||||
the than two two two two two two two two two two two two two
|
||||
the have the the the the the
|
||||
according the the the the the the the the the the the the the the the the the the the of of of of the the the 9 9 5
|
||||
the have the the the the the was a a a the
|
||||
the the of of the the the the the the the the the the the the the the the the
|
||||
the is has that that that the the the of the the and and and and the
|
||||
it is the the the of the the the the the the the the
|
||||
it would be to the the the
|
||||
he has to in in the the the the the the the the the
|
||||
the again has mr the that that that is is is a a
|
||||
in the the the the the the the in in in and and
|
||||
it has also the the the the the of of
|
||||
let us us to those who who who
|
||||
that
|
||||
in years years years years years years the the the the the
|
||||
i have
|
||||
on thursday april the of of of of the the the the the the the the the the the the the the the the the the the
|
||||
the article of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the proposal is the the the the the the the the the the
|
||||
the the of of the the the the the the the the the the the the the the which which which the
|
||||
the october march the the on the the the and and and and and
|
||||
this this these the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the new new new new
|
||||
the the of the the the the the the the the the
|
||||
the was was a the the the the the the the in in in
|
||||
the al of the the the the was was was the the the and and and and and and in in in in in in in
|
||||
the was of the was to the the the the the in in in in in in in in in in in in in in
|
||||
he were were a
|
||||
the few days days days days was was was was was was was was was was was was was and and and and and and and was
|
||||
the was was was the the the the the the the the the the the the was he he he he he the the
|
||||
if he it be be be the the the if
|
||||
the the of of the the the the that that that that that that be be
|
||||
that is
|
||||
this was my my my my was was
|
||||
the has has the the the and and and and and and and the the the the the the the the the the the the the the the the of of
|
||||
they are have they their their their as or or or or or or or or their their their their their their their their their their
|
||||
the are often the the the
|
||||
in the the of the the the and and and and and to to to to to to the
|
||||
the years weeks the the the parliament parliament parliament parliament the the the the the the the the the
|
||||
however the is is be be a a a a a that that that that that that that be be be be be and and
|
||||
in the the the the the the the the the the the the the not not not
|
||||
however it however that that that that to to to the and and and and and
|
||||
the attacks of of by by by the the the the the the and and and and and and and and and and and to to
|
||||
one one one of is is to to to to to and and and and and
|
||||
it has been been and many many many many many many many in in in in in in in in in in in in in in a a a a to to to
|
||||
the rights of rights in in the in in in in in in in in in in in in in in and women women women women
|
||||
the is
|
||||
on have we we in in in in in in in in
|
||||
the 10 20 10 10
|
||||
they will be in in in and and and
|
||||
the is of the the the the of
|
||||
the are a the the the the the the the the the the the the the the the the the the the the the the the the the the the programme programme
|
||||
the 10 10 10 10 10
|
||||
the the of the the the the the the the the the the
|
||||
the is of and the the the the the
|
||||
the group is the the of of the and and
|
||||
the three three three three three three three
|
||||
the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the of of of of of of of of of of of of
|
||||
in the the the the the the the the the the the the the the the the the the the of of of of of of of of 000 000 000
|
||||
the un of nations that that the the the the the the the the the the the the the the to to the the the the the the the
|
||||
the the of the the the the the the the that the the the the the the the the the the the the the the the the the the
|
||||
the increase of in in and and the the the the the the the the the the of of of
|
||||
this is a of and and and and
|
||||
we have the the the the the the the the the the the and and and and and and and and and and and and
|
||||
the to of the the in in police police police police police police police police
|
||||
however the is is the the the the the the the the the the the the the the in in in
|
||||
there did not been
|
||||
we have to to to to
|
||||
the of of the the the in in
|
||||
in the the the the the the the the that that that that that to to to to to to the the the the the the the the the the the the the
|
||||
the was a new new the the the the of of children children children children children children children
|
||||
the was a the the the the the the the the the the the the the the the and and and and
|
||||
the the of the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
no have not to
|
||||
in the the the the the the the the the the the the the and and the the the the in in in in in in in in in
|
||||
the the that that that the the the the the the the the the the the the the the
|
||||
the the and and and and and are are that that that united united united united united united and and
|
||||
in was was a to to of the the
|
||||
in the the the the the the the the the the the the the the the the the the the the
|
||||
at the two two two the the the the the the the the the the the the the the the the the the the the the the the the the to to
|
||||
the is the the the the the the the the the the the the the the the the the the the the the the
|
||||
the new the the the the the the system system system system a a a a a the the the the the the the the the the
|
||||
the eu has have the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
you are two two two with the the the the the the the the the the the the the the the the the
|
||||
it will be the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
we never never to a in in in in in in
|
||||
the minister minister now to to to and and and that it it to
|
||||
the is not not clear that that that plan plan plan
|
||||
we cannot not to the many many many
|
||||
the the of minister the to to to to to to to the the the the the the the the the the not not
|
||||
mr minister minister i i to to to to to to to to the the the the the the the the
|
||||
they are to to to and and and and and and to the the
|
||||
if we we to to to the the the end end end end the the the the the the
|
||||
mrs mr who the the the not not not
|
||||
the was the the the the minister minister minister said said said that to to to to the union
|
||||
however we do not see that brussels brussels brussels brussels brussels brussels
|
||||
we the of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the presidency has has that has has the the the and and and and and and and to to to to to to to
|
||||
however the says also that that that that the the the the the in in in in in in in in in in
|
||||
it is is that that that the the the the the the the the the the the the the the the to to to to to to to
|
||||
the march on brussels parliament brussels brussels brussels brussels brussels brussels brussels brussels brussels the the the the the the
|
||||
we will a a a the the the the the the the the the the and and
|
||||
no and and and
|
||||
the
|
||||
the has
|
||||
however there has to
|
||||
it was been the the the in in
|
||||
the was was was the the the
|
||||
the is
|
||||
i have seen the the the the the i i i have have have have have
|
||||
the was has to to to to to to the the not not
|
||||
however there still not
|
||||
i would to to to to to to that to to to to to to
|
||||
it was a and and and
|
||||
in the the of have to to
|
||||
it is
|
||||
it is it it is is it it it it it it that is is is is is is is is is
|
||||
the the of of been the the the
|
||||
it is more more more 000 000
|
||||
mr has has been that that that that never never never never never never to
|
||||
according the the the the the the the the the the the the the the not not not not not
|
||||
the was of the the the the the the the the the the the the the the the the the
|
||||
the the ago ago was was
|
||||
i i have in the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
i have to the the the the the the the the and and and and and and and the the in in in
|
||||
i have not to to the the the the the the the the the and and and and
|
||||
i am not that that i i i in in the
|
||||
i would i i i i i to
|
||||
when the was was was was the the the in in in in in in in in in in in in in
|
||||
then then then to and and and and and and and the the the the the the the
|
||||
the next of the the programme the the the the the the the the the the the and and and and and
|
||||
he said that that was was been the the and and and and and work work work
|
||||
they and be fair
|
||||
the and and and and and and and and and
|
||||
as the the the the the the the the the the the was was was was was
|
||||
i am not that that the of of of of of of of and and and the the the the the the the the the children children children children children children
|
||||
one example of example example the the the the was was was was was was was was was was was the the the the the in in
|
||||
the was was and and and and and and and and and the the the the the
|
||||
it was it was and and and and and and and and and in in and and and and and and and
|
||||
we must to to because because because because because because not are and and and and and and and and and
|
||||
i am all the of
|
||||
the next the the the the the the the the the the the the the the the 30 30 30 30 30 30 30
|
||||
the were have have a
|
||||
the the of the the the president president president president president the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
in week last last the the the the the the the the the the the the the the the the the the the the
|
||||
however the of of of the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
you has have that that that to to to to the the the the the the the the the the the the
|
||||
the military between between police police and and and and and the the the the the the the the
|
||||
the was has that the the the the the the the the the the the the the the the the the the the the the the and and
|
||||
the the of the the the the the the the the the was was was was was was was was was was was was to to to to to to to to to to the the the
|
||||
the is is more more more more more than than than than than more the the the the the the the the the the the to to to to to to to
|
||||
the the of the the the the the the the the the the the the the the
|
||||
this is to to to to the the the the the the the the the the the the the the the the the
|
||||
however there was a a a a the the but to to to to to to not not not not
|
||||
it is not that that that that the the the the the the the the the the the the the the the the the the the the the the
|
||||
it was that that that that the the the the the the the the the the the the the
|
||||
in was that the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the to to to to to to to to to to
|
||||
this is a in of as and and this this a
|
||||
he has has that that that has has was was was was was the the that that that to to to to to
|
||||
i said that that the to to to the the the the the the the the the
|
||||
the heads government government the that to to to to to to to to the the the the the the
|
||||
at the time time the has to to to to to and and and and and and and the the the the the the the
|
||||
the of and and and and the the
|
||||
the the of the the the the we we we we we we we we we we the the the the the the the the the the the the the the the the the the the the
|
||||
the who authorities that that the the military military military military the the the the the the the the the not not not not not not not not not
|
||||
the would to to to to to to to to to to to to to to to the the the the the the the the to to to to
|
||||
furthermore addition the the the the the the the the the the in
|
||||
the the the but but but but but but but but but but the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the in in in in
|
||||
it is not not and and and and and and not not not not not not
|
||||
europe is a to and and the the the policy policy
|
||||
it is a strong the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the government government the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
there has a a in in in in and and and and many
|
||||
the is is that that is is and and and and and and and and and and world world world world world world
|
||||
the authorities authorities authorities the problems the problems the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
the the of also also the the the the the the the the
|
||||
the fact that that that that china to to to to the foreign foreign foreign to to to to to to or or
|
||||
the chinese not not to to the other the the the the the the
|
||||
in the events events and and and and and and the and and and and and the the the the the the the the the the the the the the the the the the
|
||||
it is a a situation the the the the the the the the the the the the the the the the the the the the the the the the the the the the the and and and and
|
||||
the is is that that that not not the the the the the the to to to to to
|
||||
in the of the the the the the the the the the the the trade trade and and and and and and to to to
|
||||
it is a economy economy the economy the the the the the the the the the and and and economic economic economic economic
|
||||
the has was has has been been
|
||||
they are are that to to to to the the the the the new new new new new new new new
|
||||
the is states states the to to to china china china china china china china china china china not not not not not the the the the the united united united united united united united
|
||||
the united states china china china china china china and and that that the to to to
|
||||
the the and and other other other other other other and the the the the the the the the the the the the the
|
||||
the world of is the
|
||||
this is a a the the the of of of of of of of and and and and and
|
||||
this the the the the the the the and and and and and and and and and and and and and and and and and and and and and and and
|
||||
the of of the the the the the the the the the the the the the the the the century century century century century century
|
||||
the of of of the the the the the the the the the the the the the the the the and and the the the
|
||||
many of of the the the the the the and and and and
|
||||
this is a of of the is is
|
||||
it is is is in
|
||||
however are are are are are they to to to the the their their of
|
||||
as the the the the and and the the the the
|
||||
the has of the the the the the the the the the the constitution constitution
|
||||
in all it more more more more than than in world world world world
|
||||
but a a a of and is is is is europe europe europe europe europe is is but but but is a quality
|
||||
in president the the in in in in in in own
|
||||
what is not not not not
|
||||
the the the the the the the the the the the the the the
|
||||
the is of that not not not not not not
|
||||
the will to the the of of of of of of of of of the
|
||||
in example in example example in in in in in in in in in in in in in the
|
||||
this is one of of of of of of of in the the the the the
|
||||
we have have a a good good and and and to to to to to to to and and and
|
||||
this will never be be be to that that that that a the the the the the the the
|
||||
the of of of the international international international is is is is is is
|
||||
the the the of his the on the in in the to to
|
||||
the has has the the the has the the the the the the the the the the the the the the and and and and and
|
||||
the the the the the the the the the the the the the the the the the the the the the the the and and and and and
|
||||
the is is that that to to to the the and it it it it it it it to
|
||||
it is now years and and and and that that that that that that that
|
||||
the my the of of of of of of i i i that that that that that that is
|
||||
the of of is is is is my
|
||||
in example the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
in to to the of of of of of of of of of of of are are are
|
||||
perhaps years a a a a years years
|
||||
the is the the the the the
|
||||
the has the the the the the the the the the the the the
|
||||
the of of be be the the the the the the the war
|
||||
i the the
|
||||
i am like the the
|
||||
i i i i speak to speak the the the the the the the
|
||||
europe must must must to to to to to the the
|
||||
in the last last the the the the the the the the the the the the the the and and and
|
||||
the of of the has in in the the
|
||||
mr president first mr first first his his he he he he to to to to to to to
|
||||
the was a to the the the the the the the the and and and
|
||||
last four four four four weeks to in
|
||||
the 000 000 000 and the the the the
|
||||
the of of the the of the the the the the the the
|
||||
we us the the of the the the the the
|
||||
there is not the the the the the
|
||||
the was to the the the the to the and and and and
|
||||
he has is
|
||||
the will will to to to to
|
||||
it is it and and and the the the the the
|
||||
they were have their their their
|
||||
we have not that the children and and and and and and and the the the
|
||||
we have to the the the the the the the the the
|
||||
there the a of the the the the the the the the the the the the the the the the the the the the the the
|
||||
on the the the and and and and and and and and the the the the the the the the the
|
||||
the year of that was was was that that that that in in in in in in
|
||||
the year of the the the the the the the the the the the the the the the the the the the the the the
|
||||
one is one one one of the the the the the
|
||||
in fact the the the was was was was was was was was was was
|
||||
the the of the the and and and and and and women that that that that that that that that that that that that
|
||||
the the of of of of of of
|
||||
the a a the the the million million that that that the the the the the the the the the
|
||||
it is a of of of
|
||||
the year ago the was was was the the the the the the the the the
|
||||
the has has been the the the the the the the the the the the the to to to
|
||||
this is a the the the my my my
|
||||
it is be be be be
|
||||
it can be to to of of of
|
||||
this is a the the
|
||||
the has was been been
|
||||
this is a a
|
||||
it idea idea idea idea idea idea the the the the the the the
|
||||
it is a and and and and and and
|
||||
the has years years years years years
|
||||
the the of the the the the the
|
||||
it of of of the all
|
||||
in are are of of of the the the
|
||||
this is a the the the the the the
|
||||
there is is a a
|
||||
this was the the to to to to to to the the united united united united united united
|
||||
i was not to to to to to to to
|
||||
it is a
|
||||
the of of of of
|
||||
the was the the the the the the the the the the the the of of of
|
||||
the shows shows the the of of of
|
||||
the 1 of of the the the the and and and and and and and and the the the the the the the the the the the the the the the the the the
|
||||
however the however is is is
|
||||
the of of one one one one one of of of of of of of the the the the which the the the the the the the
|
||||
the figures that that that the the the of of of of one one one one one one
|
||||
the the the the the the the the the the the the the the the the and and and and and and and and and and and and and and and and and and and and and and and and and and
|
||||
however the of of of of our our our of of of
|
||||
this is the the of the the the the the the the the the the the the the the the the programme the the the the the the in in in
|
||||
this is only one but but one one one of the the the the the the the the the
|
||||
the is that that that the year year more more more more more more and and and and the the the the the
|
||||
the is the the is is is is
|
||||
two of are of of
|
||||
however as the case of of of of the the of of of of of of of of the the the the the the the
|
||||
i said i i i the the the the the the the the the it it it that that that that that that the the the the the the the the the the the the the the the
|
||||
i think that if more more and and and that that be be be
|
||||
this is a than than than we we we we the the of of
|
||||
the other of are are are least least least least the
|
||||
if the the and and and and and that that that that that that that and and the the the the the
|
||||
the the of and and and and and and and the the that that that and and and and and and and and and and and and and and and and and and and
|
||||
we must be start the
|
||||
we need to to the the the we to to to more more more and and and and and that that that that that the the
|
||||
if we we the we we we we we we the the the the the the the the the the
|
||||
the 30 30 30 and and and and our
|
||||
we we can to the the and and and and and and and is
|
||||
if we our our our we our our
|
||||
the the of the the to to to to to the the the the the the the and and and the the the the the the the the the the the the the the the the the the the the the the the the the
|
||||
|
164
run.py
Normal file
164
run.py
Normal file
@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
import lzma
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from keras import optimizers
|
||||
from keras.callbacks import ModelCheckpoint
|
||||
from keras.layers import Dense, LSTM, Embedding, RepeatVector
|
||||
from keras.models import Sequential, load_model
|
||||
from keras.preprocessing.text import Tokenizer
|
||||
from keras.utils import pad_sequences
|
||||
from keras.backend import clear_session
|
||||
get_ipython().run_line_magic('matplotlib', 'inline')
|
||||
pd.set_option('display.max_colwidth', 200)
|
||||
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
physical_devices = tf.config.list_physical_devices('GPU')
|
||||
tf.config.experimental.set_memory_growth(physical_devices[0], enable=True)
|
||||
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
train_folder = 'wmt-2020-pl-en/train'
|
||||
|
||||
|
||||
def read_xz_file(fname):
|
||||
with lzma.open(fname, mode='rt', encoding='utf-8') as f:
|
||||
return [line.strip() for line in f.readlines()]
|
||||
|
||||
|
||||
train_data = read_xz_file(f'{train_folder}/in.tsv.xz')
|
||||
train_labels = read_xz_file(f'{train_folder}/expected.tsv.xz')
|
||||
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
dev_folder = 'wmt-2020-pl-en/dev-0'
|
||||
|
||||
|
||||
def read_file(fname):
|
||||
with open(fname, mode='rt', encoding='utf-8') as f:
|
||||
return [line.strip() for line in f.readlines()]
|
||||
|
||||
|
||||
dev_data = read_file(f'{dev_folder}/in.tsv')
|
||||
dev_labels = read_file(f'{dev_folder}/in.tsv')
|
||||
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
def tokenize(lines):
|
||||
tokenizer = Tokenizer()
|
||||
tokenizer.fit_on_texts(lines)
|
||||
return tokenizer
|
||||
|
||||
|
||||
src_tokenizer = tokenize(train_data)
|
||||
src_vocab_size = len(src_tokenizer.word_index) + 1
|
||||
dst_tokenizer = tokenize(train_labels)
|
||||
dst_vocab_size = len(dst_tokenizer.word_index) + 1
|
||||
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
def encode_sequences(tokenizer, length, lines):
|
||||
seq = tokenizer.texts_to_sequences(lines)
|
||||
# pad sequences with 0 values
|
||||
seq = pad_sequences(seq, maxlen=length, padding='post', value=0)
|
||||
return seq
|
||||
|
||||
|
||||
seq_len = 50
|
||||
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
train_x = encode_sequences(src_tokenizer, seq_len, train_data)
|
||||
train_y = encode_sequences(dst_tokenizer, seq_len, train_labels)
|
||||
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
hidden_size = 256
|
||||
|
||||
model = Sequential()
|
||||
model.add(Embedding(src_vocab_size, hidden_size, input_length=seq_len, mask_zero=True))
|
||||
model.add(LSTM(hidden_size))
|
||||
model.add(RepeatVector(seq_len))
|
||||
model.add(LSTM(hidden_size, return_sequences=True))
|
||||
model.add(Dense(dst_vocab_size, activation='softmax'))
|
||||
|
||||
rms = optimizers.RMSprop(learning_rate=0.001)
|
||||
model.compile(optimizer=rms, loss='sparse_categorical_crossentropy')
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
filename = 'model'
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
checkpoint = ModelCheckpoint(filename, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
|
||||
|
||||
model.fit(
|
||||
train_x,
|
||||
train_y.reshape(train_y.shape[0], train_y.shape[1], 1),
|
||||
epochs=10,
|
||||
batch_size=256,
|
||||
validation_split=0.2,
|
||||
callbacks=[checkpoint]
|
||||
)
|
||||
|
||||
|
||||
# In[28]:
|
||||
|
||||
|
||||
dev_x = encode_sequences(src_tokenizer, seq_len, dev_data)
|
||||
dev_y = encode_sequences(dst_tokenizer, seq_len, dev_labels)
|
||||
|
||||
model = load_model(filename)
|
||||
dev_preds = np.argmax(model.predict(dev_x), axis=-1)
|
||||
dev_preds_texts = dst_tokenizer.sequences_to_texts(dev_preds)
|
||||
with open(f'{dev_folder}/out.tsv', 'w') as f:
|
||||
f.writelines(dev_preds_texts)
|
||||
|
||||
|
||||
# In[27]:
|
||||
|
||||
|
||||
dst_tokenizer.sequences_to_texts([dev_preds[0]])
|
||||
|
||||
|
||||
# In[ ]:
|
||||
|
||||
|
||||
test_folder = 'wmt-2020-pl-en/test-A'
|
||||
|
||||
test_data = read_file(f'{test_folder}/in.tsv')
|
||||
|
||||
test_x = encode_sequences(src_tokenizer, seq_len, test_data)
|
||||
test_preds = np.argmax(model.predict(test_x), axis=-1)
|
||||
test_preds_texts = dst_tokenizer.sequences_to_texts(test_preds)
|
||||
with open(f'{test_folder}/out.tsv', 'w') as f:
|
||||
f.writelines(test_preds_texts)
|
||||
|
||||
1200
test-A/out.tsv
Normal file
1200
test-A/out.tsv
Normal file
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user