PCQRSCANER/venv/Lib/site-packages/nltk/test/unit/lm/test_models.py
2019-12-22 21:51:47 +01:00

447 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# -*- coding: utf-8 -*-
# Natural Language Toolkit: Language Model Unit Tests
#
# Copyright (C) 2001-2019 NLTK Project
# Author: Ilia Kurenkov <ilia.kurenkov@gmail.com>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT
from __future__ import division
import math
import sys
import unittest
from six import add_metaclass
from nltk.lm import (
Vocabulary,
MLE,
Lidstone,
Laplace,
WittenBellInterpolated,
KneserNeyInterpolated,
)
from nltk.lm.preprocessing import padded_everygrams
def _prepare_test_data(ngram_order):
return (
Vocabulary(["a", "b", "c", "d", "z", "<s>", "</s>"], unk_cutoff=1),
[
list(padded_everygrams(ngram_order, sent))
for sent in (list("abcd"), list("egadbe"))
],
)
class ParametrizeTestsMeta(type):
"""Metaclass for generating parametrized tests."""
def __new__(cls, name, bases, dct):
contexts = (
("a",),
("c",),
(u"<s>",),
("b",),
(u"<UNK>",),
("d",),
("e",),
("r",),
("w",),
)
for i, c in enumerate(contexts):
dct["test_sumto1_{0}".format(i)] = cls.add_sum_to_1_test(c)
scores = dct.get("score_tests", [])
for i, (word, context, expected_score) in enumerate(scores):
dct["test_score_{0}".format(i)] = cls.add_score_test(
word, context, expected_score
)
return super(ParametrizeTestsMeta, cls).__new__(cls, name, bases, dct)
@classmethod
def add_score_test(cls, word, context, expected_score):
if sys.version_info > (3, 5):
message = "word='{word}', context={context}"
else:
# Python 2 doesn't report the mismatched values if we pass a custom
# message, so we have to report them manually.
message = (
"{score} != {expected_score} within 4 places, "
"word='{word}', context={context}"
)
def test_method(self):
score = self.model.score(word, context)
self.assertAlmostEqual(
score, expected_score, msg=message.format(**locals()), places=4
)
return test_method
@classmethod
def add_sum_to_1_test(cls, context):
def test(self):
s = sum(self.model.score(w, context) for w in self.model.vocab)
self.assertAlmostEqual(s, 1.0, msg="The context is {}".format(context))
return test
@add_metaclass(ParametrizeTestsMeta)
class MleBigramTests(unittest.TestCase):
"""unit tests for MLENgramModel class"""
score_tests = [
("d", ["c"], 1),
# Unseen ngrams should yield 0
("d", ["e"], 0),
# Unigrams should also be 0
("z", None, 0),
# N unigrams = 14
# count('a') = 2
("a", None, 2.0 / 14),
# count('y') = 3
("y", None, 3.0 / 14),
]
def setUp(self):
vocab, training_text = _prepare_test_data(2)
self.model = MLE(2, vocabulary=vocab)
self.model.fit(training_text)
def test_logscore_zero_score(self):
# logscore of unseen ngrams should be -inf
logscore = self.model.logscore("d", ["e"])
self.assertTrue(math.isinf(logscore))
def test_entropy_perplexity_seen(self):
# ngrams seen during training
trained = [
("<s>", "a"),
("a", "b"),
("b", "<UNK>"),
("<UNK>", "a"),
("a", "d"),
("d", "</s>"),
]
# Ngram = Log score
# <s>, a = -1
# a, b = -1
# b, UNK = -1
# UNK, a = -1.585
# a, d = -1
# d, </s> = -1
# TOTAL logscores = -6.585
# - AVG logscores = 1.0975
H = 1.0975
perplexity = 2.1398
self.assertAlmostEqual(H, self.model.entropy(trained), places=4)
self.assertAlmostEqual(perplexity, self.model.perplexity(trained), places=4)
def test_entropy_perplexity_unseen(self):
# In MLE, even one unseen ngram should make entropy and perplexity infinite
untrained = [("<s>", "a"), ("a", "c"), ("c", "d"), ("d", "</s>")]
self.assertTrue(math.isinf(self.model.entropy(untrained)))
self.assertTrue(math.isinf(self.model.perplexity(untrained)))
def test_entropy_perplexity_unigrams(self):
# word = score, log score
# <s> = 0.1429, -2.8074
# a = 0.1429, -2.8074
# c = 0.0714, -3.8073
# UNK = 0.2143, -2.2224
# d = 0.1429, -2.8074
# c = 0.0714, -3.8073
# </s> = 0.1429, -2.8074
# TOTAL logscores = -21.6243
# - AVG logscores = 3.0095
H = 3.0095
perplexity = 8.0529
text = [("<s>",), ("a",), ("c",), ("-",), ("d",), ("c",), ("</s>",)]
self.assertAlmostEqual(H, self.model.entropy(text), places=4)
self.assertAlmostEqual(perplexity, self.model.perplexity(text), places=4)
@add_metaclass(ParametrizeTestsMeta)
class MleTrigramTests(unittest.TestCase):
"""MLE trigram model tests"""
score_tests = [
# count(d | b, c) = 1
# count(b, c) = 1
("d", ("b", "c"), 1),
# count(d | c) = 1
# count(c) = 1
("d", ["c"], 1),
# total number of tokens is 18, of which "a" occured 2 times
("a", None, 2.0 / 18),
# in vocabulary but unseen
("z", None, 0),
# out of vocabulary should use "UNK" score
("y", None, 3.0 / 18),
]
def setUp(self):
vocab, training_text = _prepare_test_data(3)
self.model = MLE(3, vocabulary=vocab)
self.model.fit(training_text)
@add_metaclass(ParametrizeTestsMeta)
class LidstoneBigramTests(unittest.TestCase):
"""unit tests for Lidstone class"""
score_tests = [
# count(d | c) = 1
# *count(d | c) = 1.1
# Count(w | c for w in vocab) = 1
# *Count(w | c for w in vocab) = 1.8
("d", ["c"], 1.1 / 1.8),
# Total unigrams: 14
# Vocab size: 8
# Denominator: 14 + 0.8 = 14.8
# count("a") = 2
# *count("a") = 2.1
("a", None, 2.1 / 14.8),
# in vocabulary but unseen
# count("z") = 0
# *count("z") = 0.1
("z", None, 0.1 / 14.8),
# out of vocabulary should use "UNK" score
# count("<UNK>") = 3
# *count("<UNK>") = 3.1
("y", None, 3.1 / 14.8),
]
def setUp(self):
vocab, training_text = _prepare_test_data(2)
self.model = Lidstone(0.1, 2, vocabulary=vocab)
self.model.fit(training_text)
def test_gamma(self):
self.assertEqual(0.1, self.model.gamma)
def test_entropy_perplexity(self):
text = [
("<s>", "a"),
("a", "c"),
("c", "<UNK>"),
("<UNK>", "d"),
("d", "c"),
("c", "</s>"),
]
# Unlike MLE this should be able to handle completely novel ngrams
# Ngram = score, log score
# <s>, a = 0.3929, -1.3479
# a, c = 0.0357, -4.8074
# c, UNK = 0.0(5), -4.1699
# UNK, d = 0.0263, -5.2479
# d, c = 0.0357, -4.8074
# c, </s> = 0.0(5), -4.1699
# TOTAL logscore: 24.5504
# - AVG logscore: 4.0917
H = 4.0917
perplexity = 17.0504
self.assertAlmostEqual(H, self.model.entropy(text), places=4)
self.assertAlmostEqual(perplexity, self.model.perplexity(text), places=4)
@add_metaclass(ParametrizeTestsMeta)
class LidstoneTrigramTests(unittest.TestCase):
score_tests = [
# Logic behind this is the same as for bigram model
("d", ["c"], 1.1 / 1.8),
# if we choose a word that hasn't appeared after (b, c)
("e", ["c"], 0.1 / 1.8),
# Trigram score now
("d", ["b", "c"], 1.1 / 1.8),
("e", ["b", "c"], 0.1 / 1.8),
]
def setUp(self):
vocab, training_text = _prepare_test_data(3)
self.model = Lidstone(0.1, 3, vocabulary=vocab)
self.model.fit(training_text)
@add_metaclass(ParametrizeTestsMeta)
class LaplaceBigramTests(unittest.TestCase):
"""unit tests for Laplace class"""
score_tests = [
# basic sanity-check:
# count(d | c) = 1
# *count(d | c) = 2
# Count(w | c for w in vocab) = 1
# *Count(w | c for w in vocab) = 9
("d", ["c"], 2.0 / 9),
# Total unigrams: 14
# Vocab size: 8
# Denominator: 14 + 8 = 22
# count("a") = 2
# *count("a") = 3
("a", None, 3.0 / 22),
# in vocabulary but unseen
# count("z") = 0
# *count("z") = 1
("z", None, 1.0 / 22),
# out of vocabulary should use "UNK" score
# count("<UNK>") = 3
# *count("<UNK>") = 4
("y", None, 4.0 / 22),
]
def setUp(self):
vocab, training_text = _prepare_test_data(2)
self.model = Laplace(2, vocabulary=vocab)
self.model.fit(training_text)
def test_gamma(self):
# Make sure the gamma is set to 1
self.assertEqual(1, self.model.gamma)
def test_entropy_perplexity(self):
text = [
("<s>", "a"),
("a", "c"),
("c", "<UNK>"),
("<UNK>", "d"),
("d", "c"),
("c", "</s>"),
]
# Unlike MLE this should be able to handle completely novel ngrams
# Ngram = score, log score
# <s>, a = 0.2, -2.3219
# a, c = 0.1, -3.3219
# c, UNK = 0.(1), -3.1699
# UNK, d = 0.(09), 3.4594
# d, c = 0.1 -3.3219
# c, </s> = 0.(1), -3.1699
# Total logscores: 18.7651
# - AVG logscores: 3.1275
H = 3.1275
perplexity = 8.7393
self.assertAlmostEqual(H, self.model.entropy(text), places=4)
self.assertAlmostEqual(perplexity, self.model.perplexity(text), places=4)
@add_metaclass(ParametrizeTestsMeta)
class WittenBellInterpolatedTrigramTests(unittest.TestCase):
def setUp(self):
vocab, training_text = _prepare_test_data(3)
self.model = WittenBellInterpolated(3, vocabulary=vocab)
self.model.fit(training_text)
score_tests = [
# For unigram scores by default revert to MLE
# Total unigrams: 18
# count('c'): 1
("c", None, 1.0 / 18),
# in vocabulary but unseen
# count("z") = 0
("z", None, 0.0 / 18),
# out of vocabulary should use "UNK" score
# count("<UNK>") = 3
("y", None, 3.0 / 18),
# gamma(['b']) = 0.1111
# mle.score('c', ['b']) = 0.5
# (1 - gamma) * mle + gamma * mle('c') ~= 0.45 + .3 / 18
("c", ["b"], (1 - 0.1111) * 0.5 + 0.1111 * 1 / 18),
# building on that, let's try 'a b c' as the trigram
# gamma(['a', 'b']) = 0.0667
# mle("c", ["a", "b"]) = 1
("c", ["a", "b"], (1 - 0.0667) + 0.0667 * ((1 - 0.1111) * 0.5 + 0.1111 / 18)),
]
@add_metaclass(ParametrizeTestsMeta)
class KneserNeyInterpolatedTrigramTests(unittest.TestCase):
def setUp(self):
vocab, training_text = _prepare_test_data(3)
self.model = KneserNeyInterpolated(3, vocabulary=vocab)
self.model.fit(training_text)
score_tests = [
# For unigram scores revert to uniform
# Vocab size: 8
# count('c'): 1
("c", None, 1.0 / 8),
# in vocabulary but unseen, still uses uniform
("z", None, 1 / 8),
# out of vocabulary should use "UNK" score, i.e. again uniform
("y", None, 1.0 / 8),
# alpha = count('bc') - discount = 1 - 0.1 = 0.9
# gamma(['b']) = discount * number of unique words that follow ['b'] = 0.1 * 2
# normalizer = total number of bigrams with this context = 2
# the final should be: (alpha + gamma * unigram_score("c"))
("c", ["b"], (0.9 + 0.2 * (1 / 8)) / 2),
# building on that, let's try 'a b c' as the trigram
# alpha = count('abc') - discount = 1 - 0.1 = 0.9
# gamma(['a', 'b']) = 0.1 * 1
# normalizer = total number of trigrams with prefix "ab" = 1 => we can ignore it!
("c", ["a", "b"], 0.9 + 0.1 * ((0.9 + 0.2 * (1 / 8)) / 2)),
]
class NgramModelTextGenerationTests(unittest.TestCase):
"""Using MLE estimator, generate some text."""
def setUp(self):
vocab, training_text = _prepare_test_data(3)
self.model = MLE(3, vocabulary=vocab)
self.model.fit(training_text)
def test_generate_one_no_context(self):
self.assertEqual(self.model.generate(random_seed=3), "<UNK>")
def test_generate_one_limiting_context(self):
# We don't need random_seed for contexts with only one continuation
self.assertEqual(self.model.generate(text_seed=["c"]), "d")
self.assertEqual(self.model.generate(text_seed=["b", "c"]), "d")
self.assertEqual(self.model.generate(text_seed=["a", "c"]), "d")
def test_generate_one_varied_context(self):
# When context doesn't limit our options enough, seed the random choice
self.assertEqual(
self.model.generate(text_seed=("a", "<s>"), random_seed=2), "a"
)
def test_generate_cycle(self):
# Add a cycle to the model: bd -> b, db -> d
more_training_text = [list(padded_everygrams(self.model.order, list("bdbdbd")))]
self.model.fit(more_training_text)
# Test that we can escape the cycle
self.assertEqual(
self.model.generate(7, text_seed=("b", "d"), random_seed=5),
["b", "d", "b", "d", "b", "d", "</s>"],
)
def test_generate_with_text_seed(self):
self.assertEqual(
self.model.generate(5, text_seed=("<s>", "e"), random_seed=3),
["<UNK>", "a", "d", "b", "<UNK>"],
)
def test_generate_oov_text_seed(self):
self.assertEqual(
self.model.generate(text_seed=("aliens",), random_seed=3),
self.model.generate(text_seed=("<UNK>",), random_seed=3),
)
def test_generate_None_text_seed(self):
# should crash with type error when we try to look it up in vocabulary
with self.assertRaises(TypeError):
self.model.generate(text_seed=(None,))
# This will work
self.assertEqual(
self.model.generate(text_seed=None, random_seed=3),
self.model.generate(random_seed=3),
)