2483 lines
89 KiB
Python
2483 lines
89 KiB
Python
|
|
|
|
|
|
|
|
|
|
import hypothesis.strategies as st
|
|
import numpy as np
|
|
import numpy.testing as npt
|
|
from hypothesis import given, settings
|
|
|
|
import caffe2.python.hypothesis_test_util as hu
|
|
|
|
from caffe2.python import (
|
|
layer_model_instantiator,
|
|
core,
|
|
schema,
|
|
workspace,
|
|
)
|
|
from caffe2.python.layers.layers import (
|
|
AccessedFeatures,
|
|
almost_equal_schemas,
|
|
get_key,
|
|
IdList,
|
|
IdScoreList,
|
|
InstantiationContext,
|
|
is_request_only_scalar,
|
|
set_request_only,
|
|
)
|
|
from caffe2.python.layers.tags import Tags
|
|
from caffe2.python.layer_test_util import (
|
|
LayersTestCase,
|
|
OpSpec,
|
|
)
|
|
import logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class TestLayers(LayersTestCase):
|
|
def testSparseDropoutWithReplacement(self):
|
|
input_record = schema.NewRecord(self.model.net, IdList)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
lengths_blob = input_record.field_blobs()[0]
|
|
values_blob = input_record.field_blobs()[1]
|
|
lengths = np.array([1] * 10).astype(np.int32)
|
|
values = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).astype(np.int64)
|
|
workspace.FeedBlob(lengths_blob, lengths)
|
|
workspace.FeedBlob(values_blob, values)
|
|
|
|
out = self.model.SparseDropoutWithReplacement(
|
|
input_record, 0.0, 0.5, 1.0, -1, output_names_or_num=1)
|
|
self.assertEqual(schema.List(schema.Scalar(np.int64,)), out)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
eval_net = self.get_eval_net()
|
|
predict_net = self.get_predict_net()
|
|
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
out_values = workspace.FetchBlob(out.items())
|
|
out_lengths = workspace.FetchBlob(out.lengths())
|
|
self.assertBlobsEqual(out_values, values)
|
|
self.assertBlobsEqual(out_lengths, lengths)
|
|
|
|
workspace.RunNetOnce(eval_net)
|
|
|
|
workspace.RunNetOnce(predict_net)
|
|
predict_values = workspace.FetchBlob("values_auto_0")
|
|
predict_lengths = workspace.FetchBlob("lengths_auto_0")
|
|
self.assertBlobsEqual(predict_values, np.array([-1] * 10).astype(np.int64))
|
|
self.assertBlobsEqual(predict_lengths, lengths)
|
|
|
|
def testAddLoss(self):
|
|
input_record_LR = self.new_record(
|
|
schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1, )))),
|
|
('logit', schema.Scalar((np.float32, (2, )))),
|
|
('weight', schema.Scalar((np.float64, (1, ))))
|
|
)
|
|
)
|
|
loss_LR = self.model.BatchLRLoss(input_record_LR)
|
|
|
|
self.model.add_loss(loss_LR)
|
|
assert 'unnamed' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, tuple())), self.model.loss.unnamed
|
|
)
|
|
self.assertEqual(loss_LR, self.model.loss.unnamed)
|
|
|
|
self.model.add_loss(loss_LR, 'addLoss')
|
|
assert 'addLoss' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, tuple())), self.model.loss.addLoss
|
|
)
|
|
self.assertEqual(loss_LR, self.model.loss.addLoss)
|
|
|
|
self.model.add_loss(
|
|
schema.Scalar(
|
|
dtype=np.float32, blob=core.BlobReference('loss_blob_1')
|
|
), 'addLoss'
|
|
)
|
|
assert 'addLoss_auto_0' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, tuple())), self.model.loss.addLoss_auto_0
|
|
)
|
|
assert core.BlobReference('loss_blob_1') in self.model.loss.field_blobs()
|
|
|
|
self.model.add_loss(
|
|
schema.Struct(
|
|
(
|
|
'structName', schema.Scalar(
|
|
dtype=np.float32,
|
|
blob=core.BlobReference('loss_blob_2')
|
|
)
|
|
)
|
|
), 'addLoss'
|
|
)
|
|
assert 'addLoss_auto_1' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Struct(('structName', schema.Scalar((np.float32, tuple())))),
|
|
self.model.loss.addLoss_auto_1
|
|
)
|
|
assert core.BlobReference('loss_blob_2') in self.model.loss.field_blobs()
|
|
|
|
loss_in_tuple_0 = schema.Scalar(
|
|
dtype=np.float32, blob=core.BlobReference('loss_blob_in_tuple_0')
|
|
)
|
|
|
|
loss_in_tuple_1 = schema.Scalar(
|
|
dtype=np.float32, blob=core.BlobReference('loss_blob_in_tuple_1')
|
|
)
|
|
|
|
loss_tuple = schema.NamedTuple(
|
|
'loss_in_tuple', * [loss_in_tuple_0, loss_in_tuple_1]
|
|
)
|
|
self.model.add_loss(loss_tuple, 'addLoss')
|
|
assert 'addLoss_auto_2' in self.model.loss
|
|
self.assertEqual(
|
|
schema.Struct(
|
|
('loss_in_tuple_0', schema.Scalar((np.float32, tuple()))),
|
|
('loss_in_tuple_1', schema.Scalar((np.float32, tuple())))
|
|
), self.model.loss.addLoss_auto_2
|
|
)
|
|
assert core.BlobReference('loss_blob_in_tuple_0')\
|
|
in self.model.loss.field_blobs()
|
|
assert core.BlobReference('loss_blob_in_tuple_1')\
|
|
in self.model.loss.field_blobs()
|
|
|
|
def testFilterMetricSchema(self):
|
|
self.model.add_metric_field("a:b", schema.Scalar())
|
|
self.model.add_metric_field("a:c", schema.Scalar())
|
|
self.model.add_metric_field("d", schema.Scalar())
|
|
|
|
self.assertEqual(
|
|
self.model.metrics_schema,
|
|
schema.Struct(
|
|
("a", schema.Struct(
|
|
("b", schema.Scalar()),
|
|
("c", schema.Scalar()),
|
|
)),
|
|
("d", schema.Scalar()),
|
|
))
|
|
|
|
self.model.filter_metrics_schema({"a:b", "d"})
|
|
self.assertEqual(
|
|
self.model.metrics_schema,
|
|
schema.Struct(
|
|
("a", schema.Struct(
|
|
("b", schema.Scalar()),
|
|
)),
|
|
("d", schema.Scalar()),
|
|
))
|
|
|
|
def testAddOutputSchema(self):
|
|
# add the first field
|
|
self.model.add_output_schema('struct', schema.Struct())
|
|
expected_output_schema = schema.Struct(('struct', schema.Struct()))
|
|
self.assertEqual(
|
|
self.model.output_schema,
|
|
expected_output_schema,
|
|
)
|
|
|
|
# add the second field
|
|
self.model.add_output_schema('scalar', schema.Scalar(np.float64))
|
|
expected_output_schema = schema.Struct(
|
|
('struct', schema.Struct()),
|
|
('scalar', schema.Scalar(np.float64)),
|
|
)
|
|
self.assertEqual(
|
|
self.model.output_schema,
|
|
expected_output_schema,
|
|
)
|
|
|
|
# overwrite a field should raise
|
|
with self.assertRaises(AssertionError):
|
|
self.model.add_output_schema('scalar', schema.Struct())
|
|
|
|
def _test_net(self, net, ops_list):
|
|
'''
|
|
Helper function to assert the net contains some set of operations and
|
|
then to run the net.
|
|
|
|
Inputs:
|
|
net -- the network to test and run
|
|
ops_list -- the list of operation specifications to check for
|
|
in the net
|
|
'''
|
|
ops_output = self.assertNetContainOps(net, ops_list)
|
|
workspace.RunNetOnce(net)
|
|
return ops_output
|
|
|
|
def testFCWithoutBias(self):
|
|
output_dims = 2
|
|
fc_without_bias = self.model.FCWithoutBias(
|
|
self.model.input_feature_schema.float_features, output_dims)
|
|
self.model.output_schema = fc_without_bias
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
fc_without_bias
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
)
|
|
|
|
mat_mul_spec = OpSpec(
|
|
"MatMul",
|
|
[
|
|
self.model.input_feature_schema.float_features(),
|
|
init_ops[0].output[0],
|
|
],
|
|
fc_without_bias.field_blobs()
|
|
)
|
|
|
|
self.assertNetContainOps(train_net, [mat_mul_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [mat_mul_spec])
|
|
|
|
def testFCWithBootstrap(self):
|
|
output_dims = 1
|
|
fc_with_bootstrap = self.model.FCWithBootstrap(
|
|
self.model.input_feature_schema.float_features,
|
|
output_dims=output_dims,
|
|
num_bootstrap=2,
|
|
max_fc_size=-1
|
|
)
|
|
self.model.output_schema = fc_with_bootstrap
|
|
|
|
|
|
self.assertEqual(len(fc_with_bootstrap), 4)
|
|
|
|
# must be in this order
|
|
assert (
|
|
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_0/indices") == fc_with_bootstrap[0].field_blobs()[0]
|
|
)
|
|
assert (
|
|
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_0/preds") == fc_with_bootstrap[1].field_blobs()[0]
|
|
)
|
|
assert (
|
|
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_1/indices") == fc_with_bootstrap[2].field_blobs()[0]
|
|
)
|
|
assert (
|
|
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_1/preds") == fc_with_bootstrap[3].field_blobs()[0]
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
predict_net = layer_model_instantiator.generate_predict_net(self.model)
|
|
|
|
train_proto = train_net.Proto()
|
|
eval_proto = predict_net.Proto()
|
|
|
|
train_ops = train_proto.op
|
|
eval_ops = eval_proto.op
|
|
|
|
master_train_ops = [
|
|
"Shape",
|
|
"GivenTensorInt64Fill",
|
|
"Gather",
|
|
"GivenTensorIntFill",
|
|
"GivenTensorIntFill",
|
|
"Cast",
|
|
"Sub",
|
|
"UniformIntFill",
|
|
"Gather",
|
|
"FC",
|
|
"UniformIntFill",
|
|
"Gather",
|
|
"FC",
|
|
]
|
|
|
|
master_eval_ops = [
|
|
"Shape",
|
|
"GivenTensorInt64Fill",
|
|
"Gather",
|
|
"GivenTensorIntFill",
|
|
"GivenTensorIntFill",
|
|
"Cast",
|
|
"Sub",
|
|
"UniformIntFill",
|
|
"FC",
|
|
"UniformIntFill",
|
|
"FC",
|
|
]
|
|
|
|
assert len(train_ops) == len(master_train_ops)
|
|
assert len(eval_ops) == len(master_eval_ops)
|
|
|
|
assert train_proto.external_input == eval_proto.external_input
|
|
assert train_proto.external_output == list()
|
|
|
|
# make sure all the ops are present and unchanged for train_net and eval_net
|
|
for idx, op in enumerate(master_train_ops):
|
|
assert train_ops[idx].type == op
|
|
|
|
for idx, op in enumerate(master_eval_ops):
|
|
assert eval_ops[idx].type == op
|
|
|
|
|
|
def testFCwithAxis2(self):
|
|
input_dim = 10
|
|
output_dim = 30
|
|
max_length = 20
|
|
input_record = self.new_record(
|
|
schema.Struct(
|
|
('history_sequence', schema.Scalar((np.float32, (max_length,
|
|
input_dim)))),
|
|
)
|
|
)
|
|
fc_out = self.model.FC(
|
|
input_record.history_sequence, output_dim,
|
|
axis=2)
|
|
self.model.output_schema = fc_out
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (max_length, output_dim))),
|
|
fc_out
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
def testFCTransposed(self):
|
|
input_dim = 10
|
|
output_dim = 30
|
|
max_length = 20
|
|
input_record = self.new_record(
|
|
schema.Struct(
|
|
('history_sequence', schema.Scalar((np.float32, (max_length,
|
|
input_dim)))),
|
|
)
|
|
)
|
|
fc_transposed_out = self.model.FC(
|
|
input_record.history_sequence, output_dim,
|
|
axis=2, transposed=True)
|
|
self.model.output_schema = fc_transposed_out
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (max_length, output_dim))),
|
|
fc_transposed_out
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
def testFCTransposedWithMaxFCSize(self):
|
|
input_dim = 10
|
|
output_dim = 30
|
|
max_length = 20
|
|
input_record = self.new_record(
|
|
schema.Struct(
|
|
('history_sequence', schema.Scalar((np.float32, (max_length,
|
|
input_dim)))),
|
|
)
|
|
)
|
|
fc_transposed_out = self.model.FC(
|
|
input_record.history_sequence, output_dim,
|
|
max_fc_size=input_dim * output_dim // 2,
|
|
axis=2, transposed=True)
|
|
self.model.output_schema = fc_transposed_out
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (max_length, output_dim))),
|
|
fc_transposed_out
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
def testSparseLookupSumPoolingWithEviction(self):
|
|
# Create test embedding table of 1 row
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.ListWithEvicted(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1)),)),)),
|
|
))
|
|
embedding_dim = 8
|
|
lengths_blob = record.sparse.sparse_feature_0.lengths.get()
|
|
values_blob = record.sparse.sparse_feature_0.items.get()
|
|
evicted_values_blob = record.sparse.sparse_feature_0._evicted_values.get()
|
|
lengths = np.array([1]).astype(np.int32)
|
|
values = np.array([0]).astype(np.int64)
|
|
# Need to reset row 0
|
|
evicted_values = np.array([0]).astype(np.int64)
|
|
workspace.FeedBlob(lengths_blob, lengths)
|
|
workspace.FeedBlob(values_blob, values)
|
|
workspace.FeedBlob(evicted_values_blob, evicted_values)
|
|
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'Sum', weight_init=("ConstantFill", {"value": 1.0}))
|
|
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
train_init_net, train_net = self.get_training_nets()
|
|
workspace.RunNetOnce(train_init_net)
|
|
embedding_after_init = workspace.FetchBlob("sparse_lookup/w")
|
|
# Change row 0's value before reset
|
|
new_values = np.array([[2, 2, 2, 2, 2, 2, 2, 2]]).astype(np.float32)
|
|
workspace.FeedBlob("sparse_lookup/w", new_values)
|
|
workspace.RunNetOnce(train_net.Proto())
|
|
embedding_after_training = workspace.FetchBlob("sparse_lookup/w")
|
|
# Verify row 0's value does not change after reset
|
|
self.assertEquals(embedding_after_training.all(), embedding_after_init.all())
|
|
|
|
|
|
def testSparseLookupSumPooling(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'Sum')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
sparse_lookup_op_spec = OpSpec(
|
|
'SparseLengthsSum',
|
|
[
|
|
init_ops[0].output[0],
|
|
record.sparse.sparse_feature_0.items(),
|
|
record.sparse.sparse_feature_0.lengths(),
|
|
],
|
|
[embedding_after_pooling()]
|
|
)
|
|
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
|
|
|
|
@given(
|
|
use_hashing=st.booleans(),
|
|
modulo=st.integers(min_value=100, max_value=200),
|
|
use_divide_mod=st.booleans(),
|
|
divisor=st.integers(min_value=10, max_value=20),
|
|
)
|
|
def testSparseFeatureHashIdList(self, use_hashing, modulo, use_divide_mod, divisor):
|
|
record = schema.NewRecord(
|
|
self.model.net,
|
|
schema.List(schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(categorical_limit=60000)
|
|
))
|
|
)
|
|
use_divide_mod = use_divide_mod if use_hashing is False else False
|
|
output_schema = self.model.SparseFeatureHash(
|
|
record,
|
|
modulo=modulo,
|
|
use_hashing=use_hashing,
|
|
use_divide_mod=use_divide_mod,
|
|
divisor=divisor,
|
|
)
|
|
|
|
self.model.output_schema = output_schema
|
|
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
self.assertEqual(output_schema._items.metadata.categorical_limit,
|
|
modulo)
|
|
train_init_net, train_net = self.get_training_nets()
|
|
if use_divide_mod:
|
|
self.assertEqual(len(train_net.Proto().op), 3)
|
|
else:
|
|
self.assertEqual(len(train_net.Proto().op), 2)
|
|
|
|
@given(
|
|
use_hashing=st.booleans(),
|
|
modulo=st.integers(min_value=100, max_value=200),
|
|
)
|
|
def testSparseFeatureHashIdScoreList(self, use_hashing, modulo):
|
|
record = schema.NewRecord(self.model.net,
|
|
schema.Map(schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(
|
|
categorical_limit=60000)),
|
|
np.float32))
|
|
|
|
output_schema = self.model.SparseFeatureHash(
|
|
record,
|
|
modulo=modulo,
|
|
use_hashing=use_hashing)
|
|
|
|
self.model.output_schema = output_schema
|
|
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
self.assertEqual(output_schema._items.keys.metadata.categorical_limit,
|
|
modulo)
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
def testSparseLookupIncorrectPositionWeightedOnIdList(self):
|
|
'''
|
|
Currently the implementation of SparseLookup assumed input is id_score_list
|
|
when use PositionWeighted.
|
|
'''
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
with self.assertRaises(AssertionError):
|
|
self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'PositionWeighted')
|
|
|
|
def testSparseLookupPositionWeightedOnIdList(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
|
|
# convert id_list to id_score_list with PositionWeighted layer
|
|
sparse_segment = record.sparse.sparse_feature_0
|
|
pos_w_layer = self.model.PositionWeighted(sparse_segment)
|
|
|
|
sparse_segment = schema.Map(
|
|
keys=get_key(sparse_segment),
|
|
values=pos_w_layer.position_weights,
|
|
lengths_blob=sparse_segment.lengths
|
|
)
|
|
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
sparse_segment, [embedding_dim], 'PositionWeighted')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("ConstantFill", None, None), # position_weights/pos_w
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("LengthsRangeFill", None, None),
|
|
OpSpec("Gather", None, None),
|
|
OpSpec("SparseLengthsWeightedSum", None, None),
|
|
])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [
|
|
OpSpec("LengthsRangeFill", None, None),
|
|
OpSpec("Gather", None, None),
|
|
OpSpec("SparseLengthsWeightedSum", None, None),
|
|
])
|
|
|
|
def testSparseLookupPositionWeightedOnIdScoreList(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('id_score_list_0', schema.Map(
|
|
schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(
|
|
categorical_limit=1000
|
|
),
|
|
),
|
|
np.float32
|
|
)),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.id_score_list_0, [embedding_dim], 'PositionWeighted')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
sparse_lookup_op_spec = OpSpec(
|
|
'SparseLengthsWeightedSum',
|
|
[
|
|
init_ops[0].output[0],
|
|
record.sparse.id_score_list_0.values(),
|
|
record.sparse.id_score_list_0.keys(),
|
|
record.sparse.id_score_list_0.lengths(),
|
|
],
|
|
[embedding_after_pooling()]
|
|
)
|
|
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
|
|
|
|
def testSparseLookupIncorrectRecencyWeightedOnIdList(self):
|
|
'''
|
|
Currently the implementation of SparseLookup assumed input is id_score_list
|
|
when use RecencyWeighted.
|
|
'''
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('sparse_feature_0', schema.List(
|
|
schema.Scalar(np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)))),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
with self.assertRaises(AssertionError):
|
|
self.model.SparseLookup(
|
|
record.sparse.sparse_feature_0, [embedding_dim], 'RecencyWeighted')
|
|
|
|
def testSparseLookupRecencyWeightedOnIdScoreList(self):
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('sparse', schema.Struct(
|
|
('id_score_list_0', schema.Map(
|
|
schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(
|
|
categorical_limit=1000
|
|
),
|
|
),
|
|
np.float32
|
|
)),
|
|
)),
|
|
))
|
|
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record.sparse.id_score_list_0, [embedding_dim], 'RecencyWeighted')
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim, ))),
|
|
embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
sparse_lookup_op_spec = OpSpec(
|
|
'SparseLengthsWeightedSum',
|
|
[
|
|
init_ops[0].output[0],
|
|
record.sparse.id_score_list_0.values(),
|
|
record.sparse.id_score_list_0.keys(),
|
|
record.sparse.id_score_list_0.lengths(),
|
|
],
|
|
[embedding_after_pooling()]
|
|
)
|
|
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
|
|
|
|
def testPairwiseSimilarityWithAllEmbeddings(self):
|
|
embedding_dim = 64
|
|
N = 5
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('all_embeddings', schema.Scalar(
|
|
((np.float32, (N, embedding_dim)))
|
|
)),
|
|
))
|
|
current = self.model.PairwiseSimilarity(
|
|
record, N * N)
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (N * N, ))),
|
|
current
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
self.assertNetContainOps(train_init_net, [])
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("BatchMatMul", None, None),
|
|
OpSpec("Flatten", None, None),
|
|
])
|
|
|
|
def testPairwiseSimilarityWithXandYEmbeddings(self):
|
|
embedding_dim = 64
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('x_embeddings', schema.Scalar(
|
|
((np.float32, (5, embedding_dim)))
|
|
)),
|
|
('y_embeddings', schema.Scalar(
|
|
((np.float32, (6, embedding_dim)))
|
|
)),
|
|
))
|
|
current = self.model.PairwiseSimilarity(
|
|
record, 5 * 6)
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (5 * 6, ))),
|
|
current
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
self.assertNetContainOps(train_init_net, [])
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("BatchMatMul", None, None),
|
|
OpSpec("Flatten", None, None),
|
|
])
|
|
|
|
def testPairwiseSimilarityWithXandYEmbeddingsAndGather(self):
|
|
embedding_dim = 64
|
|
|
|
output_idx = [1, 3, 5]
|
|
output_idx_blob = self.model.add_global_constant(
|
|
str(self.model.net.NextScopedBlob('pairwise_dot_product_gather')),
|
|
output_idx,
|
|
dtype=np.int32,
|
|
)
|
|
indices_to_gather = schema.Scalar(
|
|
(np.int32, len(output_idx)),
|
|
output_idx_blob,
|
|
)
|
|
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('x_embeddings', schema.Scalar(
|
|
((np.float32, (5, embedding_dim)))
|
|
)),
|
|
('y_embeddings', schema.Scalar(
|
|
((np.float32, (6, embedding_dim)))
|
|
)),
|
|
('indices_to_gather', indices_to_gather),
|
|
))
|
|
current = self.model.PairwiseSimilarity(
|
|
record, len(output_idx))
|
|
|
|
# This assert is not necessary,
|
|
# output size is passed into PairwiseSimilarity
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (len(output_idx), ))),
|
|
current
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
self.assertNetContainOps(train_init_net, [])
|
|
self.assertNetContainOps(train_net, [
|
|
OpSpec("BatchMatMul", None, None),
|
|
OpSpec("Flatten", None, None),
|
|
OpSpec("BatchGather", None, None),
|
|
])
|
|
|
|
def testPairwiseSimilarityIncorrectInput(self):
|
|
embedding_dim = 64
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('x_embeddings', schema.Scalar(
|
|
((np.float32, (5, embedding_dim)))
|
|
)),
|
|
))
|
|
with self.assertRaises(AssertionError):
|
|
self.model.PairwiseSimilarity(
|
|
record, 25)
|
|
|
|
record = schema.NewRecord(self.model.net, schema.Struct(
|
|
('all_embeddings', schema.List(np.float32))
|
|
))
|
|
with self.assertRaises(AssertionError):
|
|
self.model.PairwiseSimilarity(
|
|
record, 25)
|
|
|
|
def testConcat(self):
|
|
embedding_dim = 64
|
|
input_record = self.new_record(schema.Struct(
|
|
('input1', schema.Scalar((np.float32, (embedding_dim, )))),
|
|
('input2', schema.Scalar((np.float32, (embedding_dim, )))),
|
|
('input3', schema.Scalar((np.float32, (embedding_dim, )))),
|
|
))
|
|
|
|
output = self.model.Concat(input_record)
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, ((len(input_record.fields) * embedding_dim, )))),
|
|
output
|
|
)
|
|
|
|
# Note that in Concat layer we assume first dimension is batch.
|
|
# so input is B * embedding_dim
|
|
# add_axis=1 make it B * 1 * embedding_dim
|
|
# concat on axis=1 make it B * N * embedding_dim
|
|
output = self.model.Concat(input_record, axis=1, add_axis=1)
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, ((len(input_record.fields), embedding_dim)))),
|
|
output
|
|
)
|
|
|
|
def testSamplingTrain(self):
|
|
output_dims = 1000
|
|
|
|
indices = self.new_record(schema.Scalar((np.int32, (10,))))
|
|
sampling_prob = self.new_record(schema.Scalar((np.float32, (10, ))))
|
|
|
|
sampled_fc = self.model.SamplingTrain(
|
|
schema.Struct(
|
|
('input', self.model.input_feature_schema.float_features),
|
|
('indices', indices),
|
|
('sampling_prob', sampling_prob),
|
|
),
|
|
"FC",
|
|
output_dims,
|
|
)
|
|
self.model.output_schema = sampled_fc
|
|
|
|
# Check that we don't add prediction layer into the model
|
|
self.assertEqual(1, len(self.model.layers))
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
sampled_fc
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
)
|
|
|
|
sampled_fc_layer = self.model.layers[0]
|
|
|
|
gather_w_spec = OpSpec(
|
|
"Gather",
|
|
[
|
|
init_ops[0].output[0],
|
|
indices(),
|
|
],
|
|
[
|
|
sampled_fc_layer._prediction_layer.train_param_blobs[0]
|
|
]
|
|
)
|
|
gather_b_spec = OpSpec(
|
|
"Gather",
|
|
[
|
|
init_ops[1].output[0],
|
|
indices(),
|
|
],
|
|
[
|
|
sampled_fc_layer._prediction_layer.train_param_blobs[1]
|
|
]
|
|
)
|
|
train_fc_spec = OpSpec(
|
|
"FC",
|
|
[
|
|
self.model.input_feature_schema.float_features(),
|
|
] + sampled_fc_layer._prediction_layer.train_param_blobs,
|
|
sampled_fc.field_blobs()
|
|
)
|
|
log_spec = OpSpec("Log", [sampling_prob()], [None])
|
|
sub_spec = OpSpec(
|
|
"Sub",
|
|
[sampled_fc.field_blobs()[0], None],
|
|
sampled_fc.field_blobs()
|
|
)
|
|
|
|
train_ops = self.assertNetContainOps(
|
|
train_net,
|
|
[gather_w_spec, gather_b_spec, train_fc_spec, log_spec, sub_spec])
|
|
|
|
self.assertEqual(train_ops[3].output[0], train_ops[4].input[1])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[
|
|
OpSpec(
|
|
"FC",
|
|
[
|
|
self.model.input_feature_schema.float_features(),
|
|
init_ops[0].output[0],
|
|
init_ops[1].output[0],
|
|
],
|
|
sampled_fc.field_blobs()
|
|
)
|
|
]
|
|
)
|
|
|
|
def testBatchLRLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1,)))),
|
|
('logit', schema.Scalar((np.float32, (2,)))),
|
|
('weight', schema.Scalar((np.float64, (1,))))
|
|
))
|
|
loss = self.model.BatchLRLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchLRLossWithUncertainty(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1,)))),
|
|
('logit', schema.Scalar((np.float32, (2,)))),
|
|
('weight', schema.Scalar((np.float64, (1,)))),
|
|
('log_variance', schema.Scalar((np.float64, (1,)))),
|
|
))
|
|
loss = self.model.BatchLRLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testMarginRankLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('pos_prediction', schema.Scalar((np.float32, (1,)))),
|
|
('neg_prediction', schema.List(np.float32)),
|
|
))
|
|
pos_items = np.array([0.1, 0.2, 0.3], dtype=np.float32)
|
|
neg_lengths = np.array([1, 2, 3], dtype=np.int32)
|
|
neg_items = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6], dtype=np.float32)
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[pos_items, neg_lengths, neg_items]
|
|
)
|
|
loss = self.model.MarginRankLoss(input_record)
|
|
self.run_train_net_forward_only()
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBPRLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('pos_prediction', schema.Scalar((np.float32, (1,)))),
|
|
('neg_prediction', schema.List(np.float32)),
|
|
))
|
|
pos_items = np.array([0.8, 0.9], dtype=np.float32)
|
|
neg_lengths = np.array([1, 2], dtype=np.int32)
|
|
neg_items = np.array([0.1, 0.2, 0.3], dtype=np.float32)
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[pos_items, neg_lengths, neg_items]
|
|
)
|
|
loss = self.model.BPRLoss(input_record)
|
|
self.run_train_net_forward_only()
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
result = workspace.FetchBlob('bpr_loss/output')
|
|
np.testing.assert_array_almost_equal(np.array(1.24386, dtype=np.float32), result)
|
|
|
|
def testBatchMSELoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float64, (1,)))),
|
|
('prediction', schema.Scalar((np.float32, (2,)))),
|
|
))
|
|
loss = self.model.BatchMSELoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchHuberLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, (1,)))),
|
|
('prediction', schema.Scalar((np.float32, (2,)))),
|
|
))
|
|
loss = self.model.BatchHuberLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchSigmoidCrossEntropyLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, (32,)))),
|
|
('prediction', schema.Scalar((np.float32, (32,))))
|
|
))
|
|
loss = self.model.BatchSigmoidCrossEntropyLoss(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
|
|
|
|
def testBatchSoftmaxLoss(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, tuple()))),
|
|
('prediction', schema.Scalar((np.float32, (32,))))
|
|
))
|
|
loss = self.model.BatchSoftmaxLoss(input_record)
|
|
self.assertEqual(schema.Struct(
|
|
('softmax', schema.Scalar((np.float32, (32,)))),
|
|
('loss', schema.Scalar(np.float32)),
|
|
), loss)
|
|
|
|
def testBatchSoftmaxLossWeight(self):
|
|
input_record = self.new_record(schema.Struct(
|
|
('label', schema.Scalar((np.float32, tuple()))),
|
|
('prediction', schema.Scalar((np.float32, (32,)))),
|
|
('weight', schema.Scalar((np.float64, (1,))))
|
|
))
|
|
loss = self.model.BatchSoftmaxLoss(input_record)
|
|
self.assertEqual(schema.Struct(
|
|
('softmax', schema.Scalar((np.float32, (32,)))),
|
|
('loss', schema.Scalar(np.float32)),
|
|
), loss)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[2, 5]),
|
|
)
|
|
def testBatchNormalization(self, X):
|
|
input_record = self.new_record(schema.Scalar((np.float32, (5,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
bn_output = self.model.BatchNormalization(input_record)
|
|
self.assertEqual(schema.Scalar((np.float32, (5,))), bn_output)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("ConstantFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
|
|
input_blob = input_record.field_blobs()[0]
|
|
output_blob = bn_output.field_blobs()[0]
|
|
|
|
expand_dims_spec = OpSpec(
|
|
"ExpandDims",
|
|
[input_blob],
|
|
None,
|
|
)
|
|
|
|
train_bn_spec = OpSpec(
|
|
"SpatialBN",
|
|
[None, init_ops[0].output[0], init_ops[1].output[0],
|
|
init_ops[2].output[0], init_ops[3].output[0]],
|
|
[output_blob, init_ops[2].output[0], init_ops[3].output[0], None, None],
|
|
{'is_test': 0, 'order': 'NCHW', 'momentum': 0.9},
|
|
)
|
|
|
|
test_bn_spec = OpSpec(
|
|
"SpatialBN",
|
|
[None, init_ops[0].output[0], init_ops[1].output[0],
|
|
init_ops[2].output[0], init_ops[3].output[0]],
|
|
[output_blob],
|
|
{'is_test': 1, 'order': 'NCHW', 'momentum': 0.9},
|
|
)
|
|
|
|
squeeze_spec = OpSpec(
|
|
"Squeeze",
|
|
[output_blob],
|
|
[output_blob],
|
|
)
|
|
|
|
self.assertNetContainOps(
|
|
train_net,
|
|
[expand_dims_spec, train_bn_spec, squeeze_spec]
|
|
)
|
|
|
|
eval_net = self.get_eval_net()
|
|
|
|
self.assertNetContainOps(
|
|
eval_net,
|
|
[expand_dims_spec, test_bn_spec, squeeze_spec]
|
|
)
|
|
|
|
predict_net = self.get_predict_net()
|
|
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[expand_dims_spec, test_bn_spec, squeeze_spec]
|
|
)
|
|
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(eval_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(predict_net)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[2, 5, 6]),
|
|
use_layer_norm_op=st.booleans(),
|
|
)
|
|
def testLayerNormalization(self, X, use_layer_norm_op):
|
|
expect = (5, 6,)
|
|
if not use_layer_norm_op:
|
|
X = X.reshape(10, 6)
|
|
expect = (6,)
|
|
input_record = self.new_record(schema.Scalar((np.float32, expect)))
|
|
schema.FeedRecord(input_record, [X])
|
|
ln_output = self.model.LayerNormalization(
|
|
input_record, use_layer_norm_op=use_layer_norm_op
|
|
)
|
|
self.assertEqual(schema.Scalar((np.float32, expect)), ln_output)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets(add_constants=True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[5, 2]),
|
|
num_to_collect=st.integers(min_value=1, max_value=10),
|
|
)
|
|
def testLastNWindowCollector(self, X, num_to_collect):
|
|
input_record = self.new_record(schema.Scalar(np.float32))
|
|
schema.FeedRecord(input_record, [X])
|
|
last_n = self.model.LastNWindowCollector(input_record, num_to_collect)
|
|
self.run_train_net_forward_only()
|
|
output_record = schema.FetchRecord(last_n.last_n)
|
|
start = max(0, 5 - num_to_collect)
|
|
npt.assert_array_equal(X[start:], output_record())
|
|
num_visited = schema.FetchRecord(last_n.num_visited)
|
|
npt.assert_array_equal([5], num_visited())
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[5, 2]),
|
|
num_to_collect=st.integers(min_value=3, max_value=3),
|
|
)
|
|
@settings(deadline=1000)
|
|
def testReservoirSamplingWithID(self, X, num_to_collect):
|
|
ID = np.array([1, 2, 3, 1, 2], dtype=np.int64)
|
|
input_record = self.new_record(
|
|
schema.Struct(
|
|
('record', schema.Struct(
|
|
('dense', schema.Scalar()),
|
|
)),
|
|
('object_id', schema.Scalar(np.int64)),
|
|
)
|
|
)
|
|
schema.FeedRecord(input_record, [X, ID])
|
|
packed_record = self.model.PackRecords(
|
|
input_record.record, 1, fields=input_record.record.field_names())
|
|
reservoir_input = schema.Struct(
|
|
('data', packed_record),
|
|
('object_id', input_record.object_id),
|
|
)
|
|
reservoir = self.model.ReservoirSampling(
|
|
reservoir_input, num_to_collect)
|
|
self.model.output_schema = schema.Struct()
|
|
train_init_net, train_net = \
|
|
layer_model_instantiator.generate_training_nets_forward_only(
|
|
self.model)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.CreateNet(train_net)
|
|
workspace.RunNet(train_net.Proto().name, num_iter=2)
|
|
num_visited = schema.FetchRecord(reservoir.num_visited)
|
|
npt.assert_array_equal([3], num_visited())
|
|
for param in self.model.params:
|
|
serialized = workspace.SerializeBlob(str(param))
|
|
workspace.DeserializeBlob(str(param), serialized)
|
|
ID = np.array([3, 5, 3, 3, 5], dtype=np.int64)
|
|
schema.FeedRecord(input_record.object_id, [ID])
|
|
workspace.RunNet(train_net.Proto().name, num_iter=2)
|
|
num_visited = schema.FetchRecord(reservoir.num_visited)
|
|
npt.assert_array_equal([2], num_visited())
|
|
|
|
def testUniformSampling(self):
|
|
input_record = self.new_record(schema.Scalar(np.int32))
|
|
input_array = np.array([3, 10, 11, 15, 20, 99], dtype=np.int32)
|
|
schema.FeedRecord(input_record, [input_array])
|
|
num_samples = 20
|
|
num_elements = 100
|
|
uniform_sampling_output = self.model.UniformSampling(
|
|
input_record, num_samples, num_elements)
|
|
self.model.loss = uniform_sampling_output
|
|
self.run_train_net()
|
|
samples = workspace.FetchBlob(uniform_sampling_output.samples())
|
|
sampling_prob = workspace.FetchBlob(
|
|
uniform_sampling_output.sampling_prob())
|
|
self.assertEqual(num_samples, len(samples))
|
|
np.testing.assert_array_equal(input_array, samples[:len(input_array)])
|
|
np.testing.assert_almost_equal(
|
|
np.array([float(num_samples) / num_elements] * num_samples,
|
|
dtype=np.float32),
|
|
sampling_prob
|
|
)
|
|
|
|
def testUniformSamplingWithIncorrectSampleSize(self):
|
|
input_record = self.new_record(schema.Scalar(np.int32))
|
|
num_samples = 200
|
|
num_elements = 100
|
|
with self.assertRaises(AssertionError):
|
|
self.model.UniformSampling(input_record, num_samples, num_elements)
|
|
|
|
def testGatherRecord(self):
|
|
indices = np.array([1, 3, 4], dtype=np.int32)
|
|
dense = np.array(list(range(20)), dtype=np.float32).reshape(10, 2)
|
|
lengths = np.array(list(range(10)), dtype=np.int32)
|
|
items = np.array(list(range(lengths.sum())), dtype=np.int64)
|
|
items_lengths = np.array(list(range(lengths.sum())), dtype=np.int32)
|
|
items_items = np.array(list(range(items_lengths.sum())), dtype=np.int64)
|
|
record = self.new_record(schema.Struct(
|
|
('dense', schema.Scalar(np.float32)),
|
|
('sparse', schema.Struct(
|
|
('list', schema.List(np.int64)),
|
|
('list_of_list', schema.List(schema.List(np.int64))),
|
|
)),
|
|
('empty_struct', schema.Struct())
|
|
))
|
|
indices_record = self.new_record(schema.Scalar(np.int32))
|
|
input_record = schema.Struct(
|
|
('indices', indices_record),
|
|
('record', record),
|
|
)
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[indices, dense, lengths, items, lengths, items_lengths,
|
|
items_items])
|
|
gathered_record = self.model.GatherRecord(input_record)
|
|
self.assertTrue(schema.equal_schemas(gathered_record, record))
|
|
|
|
self.run_train_net_forward_only()
|
|
gathered_dense = workspace.FetchBlob(gathered_record.dense())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([dense[i:i + 1] for i in indices]), gathered_dense)
|
|
gathered_lengths = workspace.FetchBlob(
|
|
gathered_record.sparse.list.lengths())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([lengths[i:i + 1] for i in indices]),
|
|
gathered_lengths)
|
|
gathered_items = workspace.FetchBlob(
|
|
gathered_record.sparse.list.items())
|
|
offsets = lengths.cumsum() - lengths
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([
|
|
items[offsets[i]: offsets[i] + lengths[i]]
|
|
for i in indices
|
|
]), gathered_items)
|
|
|
|
gathered_items_lengths = workspace.FetchBlob(
|
|
gathered_record.sparse.list_of_list.items.lengths())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([
|
|
items_lengths[offsets[i]: offsets[i] + lengths[i]]
|
|
for i in indices
|
|
]),
|
|
gathered_items_lengths
|
|
)
|
|
|
|
nested_offsets = []
|
|
nested_lengths = []
|
|
nested_offset = 0
|
|
j = 0
|
|
for l in lengths:
|
|
nested_offsets.append(nested_offset)
|
|
nested_length = 0
|
|
for _i in range(l):
|
|
nested_offset += items_lengths[j]
|
|
nested_length += items_lengths[j]
|
|
j += 1
|
|
nested_lengths.append(nested_length)
|
|
|
|
gathered_items_items = workspace.FetchBlob(
|
|
gathered_record.sparse.list_of_list.items.items())
|
|
np.testing.assert_array_equal(
|
|
np.concatenate([
|
|
items_items[nested_offsets[i]:
|
|
nested_offsets[i] + nested_lengths[i]]
|
|
for i in indices
|
|
]),
|
|
gathered_items_items
|
|
)
|
|
|
|
def testMapToRange(self):
|
|
input_record = self.new_record(schema.Scalar(np.int32))
|
|
indices_blob = self.model.MapToRange(input_record,
|
|
max_index=100).indices
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 20, 99, 15, 11, 3, 11], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 3, 4, 5, 6, 2, 6], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 23, 35, 60, 15, 10, 15], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(train_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 7, 8, 9, 5, 1, 5], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
eval_net = self.get_eval_net()
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 23, 35, 60, 15, 200], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(eval_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 7, 8, 9, 5, 0], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([10, 3, 23, 15, 101, 115], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(eval_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([1, 2, 7, 5, 0, 0], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
predict_net = self.get_predict_net()
|
|
|
|
schema.FeedRecord(
|
|
input_record,
|
|
[np.array([3, 3, 20, 23, 151, 35, 60, 15, 200], dtype=np.int32)]
|
|
)
|
|
workspace.RunNetOnce(predict_net)
|
|
indices = workspace.FetchBlob(indices_blob())
|
|
np.testing.assert_array_equal(
|
|
np.array([2, 2, 3, 7, 0, 8, 9, 5, 0], dtype=np.int32),
|
|
indices
|
|
)
|
|
|
|
def testSelectRecordByContext(self):
|
|
float_features = self.model.input_feature_schema.float_features
|
|
|
|
float_array = np.array([1.0, 2.0], dtype=np.float32)
|
|
|
|
schema.FeedRecord(float_features, [float_array])
|
|
|
|
with Tags(Tags.EXCLUDE_FROM_PREDICTION):
|
|
log_float_features = self.model.Log(float_features, 1)
|
|
joined = self.model.SelectRecordByContext(
|
|
schema.Struct(
|
|
(InstantiationContext.PREDICTION, float_features),
|
|
(InstantiationContext.TRAINING, log_float_features),
|
|
# TODO: TRAIN_ONLY layers are also generated in eval
|
|
(InstantiationContext.EVAL, log_float_features),
|
|
)
|
|
)
|
|
|
|
# model.output_schema has to a struct
|
|
self.model.output_schema = schema.Struct((
|
|
'joined', joined
|
|
))
|
|
predict_net = layer_model_instantiator.generate_predict_net(self.model)
|
|
workspace.RunNetOnce(predict_net)
|
|
predict_output = schema.FetchRecord(predict_net.output_record())
|
|
npt.assert_array_equal(float_array,
|
|
predict_output['joined']())
|
|
eval_net = layer_model_instantiator.generate_eval_net(self.model)
|
|
workspace.RunNetOnce(eval_net)
|
|
eval_output = schema.FetchRecord(eval_net.output_record())
|
|
npt.assert_array_equal(np.log(float_array),
|
|
eval_output['joined']())
|
|
_, train_net = (
|
|
layer_model_instantiator.generate_training_nets_forward_only(
|
|
self.model
|
|
)
|
|
)
|
|
workspace.RunNetOnce(train_net)
|
|
train_output = schema.FetchRecord(train_net.output_record())
|
|
npt.assert_array_equal(np.log(float_array),
|
|
train_output['joined']())
|
|
|
|
def testFunctionalLayer(self):
|
|
def normalize(net, in_record, out_record):
|
|
mean = net.ReduceFrontMean(in_record(), 1)
|
|
net.Sub(
|
|
[in_record(), mean],
|
|
out_record(),
|
|
broadcast=1)
|
|
normalized = self.model.Functional(
|
|
self.model.input_feature_schema.float_features, 1,
|
|
normalize, name="normalizer")
|
|
|
|
# Attach metadata to one of the outputs and use it in FC
|
|
normalized.set_type((np.float32, 32))
|
|
self.model.output_schema = self.model.FC(normalized, 2)
|
|
|
|
predict_net = layer_model_instantiator.generate_predict_net(
|
|
self.model)
|
|
ops = predict_net.Proto().op
|
|
assert len(ops) == 3
|
|
assert ops[0].type == "ReduceFrontMean"
|
|
assert ops[1].type == "Sub"
|
|
assert ops[2].type == "FC"
|
|
assert len(ops[0].input) == 1
|
|
assert ops[0].input[0] ==\
|
|
self.model.input_feature_schema.float_features()
|
|
assert len(ops[1].output) == 1
|
|
assert ops[1].output[0] in ops[2].input
|
|
|
|
def testFunctionalLayerHelper(self):
|
|
mean = self.model.ReduceFrontMean(
|
|
self.model.input_feature_schema.float_features, 1)
|
|
normalized = self.model.Sub(
|
|
schema.Tuple(
|
|
self.model.input_feature_schema.float_features, mean),
|
|
1, broadcast=1)
|
|
# Attach metadata to one of the outputs and use it in FC
|
|
normalized.set_type((np.float32, (32,)))
|
|
self.model.output_schema = self.model.FC(normalized, 2)
|
|
|
|
predict_net = layer_model_instantiator.generate_predict_net(
|
|
self.model)
|
|
ops = predict_net.Proto().op
|
|
assert len(ops) == 3
|
|
assert ops[0].type == "ReduceFrontMean"
|
|
assert ops[1].type == "Sub"
|
|
assert ops[2].type == "FC"
|
|
assert len(ops[0].input) == 1
|
|
assert ops[0].input[0] ==\
|
|
self.model.input_feature_schema.float_features()
|
|
assert len(ops[1].output) == 1
|
|
assert ops[1].output[0] in ops[2].input
|
|
|
|
def testFunctionalLayerHelperAutoInference(self):
|
|
softsign = self.model.Softsign(
|
|
schema.Tuple(self.model.input_feature_schema.float_features),
|
|
1)
|
|
assert softsign.field_type().base == np.float32
|
|
assert softsign.field_type().shape == (32,)
|
|
self.model.output_schema = self.model.FC(softsign, 2)
|
|
|
|
predict_net = layer_model_instantiator.generate_predict_net(
|
|
self.model)
|
|
ops = predict_net.Proto().op
|
|
assert len(ops) == 2
|
|
assert ops[0].type == "Softsign"
|
|
assert ops[1].type == "FC"
|
|
assert len(ops[0].input) == 1
|
|
assert ops[0].input[0] ==\
|
|
self.model.input_feature_schema.float_features()
|
|
assert len(ops[0].output) == 1
|
|
assert ops[0].output[0] in ops[1].input
|
|
|
|
def testHalfToFloatTypeInference(self):
|
|
input = self.new_record(schema.Scalar((np.float32, (32,))))
|
|
|
|
output = self.model.FloatToHalf(input, 1)
|
|
assert output.field_type().base == np.float16
|
|
assert output.field_type().shape == (32, )
|
|
|
|
output = self.model.HalfToFloat(output, 1)
|
|
assert output.field_type().base == np.float32
|
|
assert output.field_type().shape == (32, )
|
|
|
|
def testFunctionalLayerHelperAutoInferenceScalar(self):
|
|
loss = self.model.AveragedLoss(self.model.input_feature_schema, 1)
|
|
self.assertEqual(1, len(loss.field_types()))
|
|
self.assertEqual(np.float32, loss.field_types()[0].base)
|
|
self.assertEqual(tuple(), loss.field_types()[0].shape)
|
|
|
|
def testFunctionalLayerInputCoercion(self):
|
|
one = self.model.global_constants['ONE']
|
|
two = self.model.Add([one, one], 1)
|
|
self.model.loss = two
|
|
self.run_train_net()
|
|
data = workspace.FetchBlob(two.field_blobs()[0])
|
|
np.testing.assert_array_equal([2.0], data)
|
|
|
|
def testFunctionalLayerWithOutputNames(self):
|
|
k = 3
|
|
topk = self.model.TopK(
|
|
self.model.input_feature_schema,
|
|
output_names_or_num=['values', 'indices'],
|
|
k=k,
|
|
)
|
|
self.assertEqual(2, len(topk.field_types()))
|
|
self.assertEqual(np.float32, topk.field_types()[0].base)
|
|
self.assertEqual((k,), topk.field_types()[0].shape)
|
|
self.assertEqual(np.int32, topk.field_types()[1].base)
|
|
self.assertEqual((k,), topk.field_types()[1].shape)
|
|
self.assertEqual(['TopK/values', 'TopK/indices'], topk.field_blobs())
|
|
|
|
def testFunctionalLayerSameOperatorOutputNames(self):
|
|
Con1 = self.model.ConstantFill([], 1, value=1)
|
|
Con2 = self.model.ConstantFill([], 1, value=2)
|
|
self.assertNotEqual(str(Con1), str(Con2))
|
|
|
|
def testFunctionalLayerWithOutputDtypes(self):
|
|
loss = self.model.AveragedLoss(
|
|
self.model.input_feature_schema,
|
|
1,
|
|
output_dtypes=(np.float32, (1,)),
|
|
)
|
|
self.assertEqual(1, len(loss.field_types()))
|
|
self.assertEqual(np.float32, loss.field_types()[0].base)
|
|
self.assertEqual((1,), loss.field_types()[0].shape)
|
|
|
|
def testPropagateRequestOnly(self):
|
|
# test case when output is request only
|
|
input_record = self.new_record(schema.Struct(
|
|
('input1', schema.Scalar((np.float32, (32, )))),
|
|
('input2', schema.Scalar((np.float32, (64, )))),
|
|
('input3', schema.Scalar((np.float32, (16, )))),
|
|
))
|
|
|
|
set_request_only(input_record)
|
|
concat_output = self.model.Concat(input_record)
|
|
self.assertEqual(is_request_only_scalar(concat_output), True)
|
|
|
|
# test case when output is not request only
|
|
input_record2 = self.new_record(schema.Struct(
|
|
('input4', schema.Scalar((np.float32, (100, ))))
|
|
)) + input_record
|
|
|
|
concat_output2 = self.model.Concat(input_record2)
|
|
self.assertEqual(is_request_only_scalar(concat_output2), False)
|
|
|
|
def testSetRequestOnly(self):
|
|
input_record = schema.Scalar(np.int64)
|
|
schema.attach_metadata_to_scalars(
|
|
input_record,
|
|
schema.Metadata(
|
|
categorical_limit=100000000,
|
|
expected_value=99,
|
|
feature_specs=schema.FeatureSpec(
|
|
feature_ids=[1, 100, 1001]
|
|
)
|
|
)
|
|
)
|
|
|
|
set_request_only(input_record)
|
|
self.assertEqual(input_record.metadata.categorical_limit, 100000000)
|
|
self.assertEqual(input_record.metadata.expected_value, 99)
|
|
self.assertEqual(
|
|
input_record.metadata.feature_specs.feature_ids,
|
|
[1, 100, 1001]
|
|
)
|
|
|
|
@given(
|
|
X=hu.arrays(dims=[5, 5]), # Shape of X is irrelevant
|
|
dropout_for_eval=st.booleans(),
|
|
)
|
|
def testDropout(self, X, dropout_for_eval):
|
|
input_record = self.new_record(schema.Scalar((np.float32, (1,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
d_output = self.model.Dropout(
|
|
input_record,
|
|
dropout_for_eval=dropout_for_eval
|
|
)
|
|
self.assertEqual(schema.Scalar((np.float32, (1,))), d_output)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
input_blob = input_record.field_blobs()[0]
|
|
output_blob = d_output.field_blobs()[0]
|
|
|
|
with_d_spec = OpSpec(
|
|
"Dropout",
|
|
[input_blob],
|
|
[output_blob, None],
|
|
{'is_test': 0, 'ratio': 0.5}
|
|
)
|
|
|
|
without_d_spec = OpSpec(
|
|
"Dropout",
|
|
[input_blob],
|
|
[output_blob, None],
|
|
{'is_test': 1, 'ratio': 0.5}
|
|
)
|
|
|
|
self.assertNetContainOps(
|
|
train_net,
|
|
[with_d_spec]
|
|
)
|
|
|
|
eval_net = self.get_eval_net()
|
|
predict_net = self.get_predict_net()
|
|
|
|
if dropout_for_eval:
|
|
self.assertNetContainOps(
|
|
eval_net,
|
|
[with_d_spec]
|
|
)
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[with_d_spec]
|
|
)
|
|
else:
|
|
self.assertNetContainOps(
|
|
eval_net,
|
|
[without_d_spec]
|
|
)
|
|
self.assertNetContainOps(
|
|
predict_net,
|
|
[without_d_spec]
|
|
)
|
|
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(eval_net)
|
|
|
|
schema.FeedRecord(input_record, [X])
|
|
workspace.RunNetOnce(predict_net)
|
|
|
|
@given(
|
|
num_inputs=st.integers(1, 3),
|
|
batch_size=st.integers(5, 10)
|
|
)
|
|
def testMergeIdListsLayer(self, num_inputs, batch_size):
|
|
inputs = []
|
|
for _ in range(num_inputs):
|
|
lengths = np.random.randint(5, size=batch_size).astype(np.int32)
|
|
size = lengths.sum()
|
|
values = np.random.randint(1, 10, size=size).astype(np.int64)
|
|
inputs.append(lengths)
|
|
inputs.append(values)
|
|
input_schema = schema.Tuple(
|
|
*[schema.List(
|
|
schema.Scalar(dtype=np.int64, metadata=schema.Metadata(
|
|
categorical_limit=20
|
|
))) for _ in range(num_inputs)]
|
|
)
|
|
|
|
input_record = schema.NewRecord(self.model.net, input_schema)
|
|
schema.FeedRecord(input_record, inputs)
|
|
output_schema = self.model.MergeIdLists(input_record)
|
|
assert schema.equal_schemas(
|
|
output_schema, IdList,
|
|
check_field_names=False)
|
|
|
|
@given(
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dims=st.integers(min_value=5, max_value=10),
|
|
output_dims=st.integers(min_value=5, max_value=10),
|
|
bandwidth=st.floats(min_value=0.1, max_value=5),
|
|
)
|
|
def testRandomFourierFeatures(self, batch_size, input_dims, output_dims, bandwidth):
|
|
|
|
def _rff_hypothesis_test(rff_output, X, W, b, scale):
|
|
'''
|
|
Runs hypothesis test for Semi Random Features layer.
|
|
|
|
Inputs:
|
|
rff_output -- output of net after running random fourier features layer
|
|
X -- input data
|
|
W -- weight parameter from train_init_net
|
|
b -- bias parameter from train_init_net
|
|
scale -- value by which to scale the output vector
|
|
'''
|
|
output = workspace.FetchBlob(rff_output)
|
|
output_ref = scale * np.cos(np.dot(X, np.transpose(W)) + b)
|
|
npt.assert_allclose(output, output_ref, rtol=1e-3, atol=1e-3)
|
|
|
|
X = np.random.random((batch_size, input_dims)).astype(np.float32)
|
|
scale = np.sqrt(2.0 / output_dims)
|
|
input_record = self.new_record(schema.Scalar((np.float32, (input_dims,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
input_blob = input_record.field_blobs()[0]
|
|
rff_output = self.model.RandomFourierFeatures(input_record,
|
|
output_dims,
|
|
bandwidth)
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
rff_output
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
# Init net assertions
|
|
init_ops_list = [
|
|
OpSpec("GaussianFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
init_ops = self._test_net(train_init_net, init_ops_list)
|
|
W = workspace.FetchBlob(self.model.layers[0].w)
|
|
b = workspace.FetchBlob(self.model.layers[0].b)
|
|
|
|
# Operation specifications
|
|
fc_spec = OpSpec("FC", [input_blob, init_ops[0].output[0],
|
|
init_ops[1].output[0]], None)
|
|
cosine_spec = OpSpec("Cos", None, None)
|
|
scale_spec = OpSpec("Scale", None, rff_output.field_blobs(),
|
|
{'scale': scale})
|
|
ops_list = [
|
|
fc_spec,
|
|
cosine_spec,
|
|
scale_spec
|
|
]
|
|
|
|
# Train net assertions
|
|
self._test_net(train_net, ops_list)
|
|
_rff_hypothesis_test(rff_output(), X, W, b, scale)
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self._test_net(eval_net, ops_list)
|
|
_rff_hypothesis_test(rff_output(), X, W, b, scale)
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self._test_net(predict_net, ops_list)
|
|
_rff_hypothesis_test(rff_output(), X, W, b, scale)
|
|
|
|
@given(
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dims=st.integers(min_value=5, max_value=10),
|
|
output_dims=st.integers(min_value=5, max_value=10),
|
|
s=st.integers(min_value=0, max_value=3),
|
|
scale=st.floats(min_value=0.1, max_value=5),
|
|
set_weight_as_global_constant=st.booleans()
|
|
)
|
|
def testArcCosineFeatureMap(self, batch_size, input_dims, output_dims, s, scale,
|
|
set_weight_as_global_constant):
|
|
|
|
def _arc_cosine_hypothesis_test(ac_output, X, W, b, s):
|
|
'''
|
|
Runs hypothesis test for Arc Cosine layer.
|
|
|
|
Inputs:
|
|
ac_output -- output of net after running arc cosine layer
|
|
X -- input data
|
|
W -- weight parameter from train_init_net
|
|
b -- bias parameter from train_init_net
|
|
s -- degree parameter
|
|
'''
|
|
# Get output from net
|
|
net_output = workspace.FetchBlob(ac_output)
|
|
|
|
# Computing output directly
|
|
x_rand = np.matmul(X, np.transpose(W)) + b
|
|
x_pow = np.power(x_rand, s)
|
|
if s > 0:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, 1])
|
|
else:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, lambda x: x / (1 + x)])
|
|
output_ref = np.multiply(x_pow, h_rand_features)
|
|
|
|
# Comparing net output and computed output
|
|
npt.assert_allclose(net_output, output_ref, rtol=1e-3, atol=1e-3)
|
|
|
|
X = np.random.normal(size=(batch_size, input_dims)).astype(np.float32)
|
|
input_record = self.new_record(schema.Scalar((np.float32, (input_dims,))))
|
|
schema.FeedRecord(input_record, [X])
|
|
input_blob = input_record.field_blobs()[0]
|
|
|
|
ac_output = self.model.ArcCosineFeatureMap(
|
|
input_record,
|
|
output_dims,
|
|
s=s,
|
|
scale=scale,
|
|
set_weight_as_global_constant=set_weight_as_global_constant
|
|
)
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims, ))),
|
|
ac_output
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
# Run create_init_net to initialize the global constants, and W and b
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(self.model.create_init_net(name='init_net'))
|
|
|
|
if set_weight_as_global_constant:
|
|
W = workspace.FetchBlob(
|
|
self.model.global_constants['arc_cosine_feature_map_fixed_rand_W']
|
|
)
|
|
b = workspace.FetchBlob(
|
|
self.model.global_constants['arc_cosine_feature_map_fixed_rand_b']
|
|
)
|
|
else:
|
|
W = workspace.FetchBlob(self.model.layers[0].random_w)
|
|
b = workspace.FetchBlob(self.model.layers[0].random_b)
|
|
|
|
# Operation specifications
|
|
fc_spec = OpSpec("FC", [input_blob, None, None], None)
|
|
softsign_spec = OpSpec("Softsign", None, None)
|
|
relu_spec = OpSpec("Relu", None, None)
|
|
relu_spec_output = OpSpec("Relu", None, ac_output.field_blobs())
|
|
pow_spec = OpSpec("Pow", None, None, {'exponent': float(s - 1)})
|
|
mul_spec = OpSpec("Mul", None, ac_output.field_blobs())
|
|
|
|
if s == 0:
|
|
ops_list = [
|
|
fc_spec,
|
|
softsign_spec,
|
|
relu_spec_output,
|
|
]
|
|
elif s == 1:
|
|
ops_list = [
|
|
fc_spec,
|
|
relu_spec_output,
|
|
]
|
|
else:
|
|
ops_list = [
|
|
fc_spec,
|
|
relu_spec,
|
|
pow_spec,
|
|
mul_spec,
|
|
]
|
|
|
|
# Train net assertions
|
|
self._test_net(train_net, ops_list)
|
|
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self._test_net(eval_net, ops_list)
|
|
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self._test_net(predict_net, ops_list)
|
|
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
|
|
|
|
@given(
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dims=st.integers(min_value=5, max_value=10),
|
|
output_dims=st.integers(min_value=5, max_value=10),
|
|
s=st.integers(min_value=0, max_value=3),
|
|
scale=st.floats(min_value=0.1, max_value=5),
|
|
set_weight_as_global_constant=st.booleans(),
|
|
use_struct_input=st.booleans(),
|
|
)
|
|
def testSemiRandomFeatures(self, batch_size, input_dims, output_dims, s, scale,
|
|
set_weight_as_global_constant, use_struct_input):
|
|
|
|
def _semi_random_hypothesis_test(srf_output, X_full, X_random, rand_w,
|
|
rand_b, s):
|
|
'''
|
|
Runs hypothesis test for Semi Random Features layer.
|
|
|
|
Inputs:
|
|
srf_output -- output of net after running semi random features layer
|
|
X_full -- full input data
|
|
X_random -- random-output input data
|
|
rand_w -- random-initialized weight parameter from train_init_net
|
|
rand_b -- random-initialized bias parameter from train_init_net
|
|
s -- degree parameter
|
|
|
|
'''
|
|
# Get output from net
|
|
net_output = workspace.FetchBlob(srf_output)
|
|
|
|
# Fetch learned parameter blobs
|
|
learned_w = workspace.FetchBlob(self.model.layers[0].learned_w)
|
|
learned_b = workspace.FetchBlob(self.model.layers[0].learned_b)
|
|
|
|
# Computing output directly
|
|
x_rand = np.matmul(X_random, np.transpose(rand_w)) + rand_b
|
|
x_learn = np.matmul(X_full, np.transpose(learned_w)) + learned_b
|
|
x_pow = np.power(x_rand, s)
|
|
if s > 0:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, 1])
|
|
else:
|
|
h_rand_features = np.piecewise(x_rand,
|
|
[x_rand <= 0, x_rand > 0],
|
|
[0, lambda x: x / (1 + x)])
|
|
output_ref = np.multiply(np.multiply(x_pow, h_rand_features), x_learn)
|
|
|
|
# Comparing net output and computed output
|
|
npt.assert_allclose(net_output, output_ref, rtol=1e-3, atol=1e-3)
|
|
|
|
X_full = np.random.normal(size=(batch_size, input_dims)).astype(np.float32)
|
|
if use_struct_input:
|
|
X_random = np.random.normal(size=(batch_size, input_dims)).\
|
|
astype(np.float32)
|
|
input_data = [X_full, X_random]
|
|
input_record = self.new_record(schema.Struct(
|
|
('full', schema.Scalar(
|
|
(np.float32, (input_dims,))
|
|
)),
|
|
('random', schema.Scalar(
|
|
(np.float32, (input_dims,))
|
|
))
|
|
))
|
|
else:
|
|
X_random = X_full
|
|
input_data = [X_full]
|
|
input_record = self.new_record(schema.Scalar(
|
|
(np.float32, (input_dims,))
|
|
))
|
|
|
|
schema.FeedRecord(input_record, input_data)
|
|
srf_output = self.model.SemiRandomFeatures(
|
|
input_record,
|
|
output_dims,
|
|
s=s,
|
|
scale_random=scale,
|
|
scale_learned=scale,
|
|
set_weight_as_global_constant=set_weight_as_global_constant
|
|
)
|
|
|
|
self.model.output_schema = schema.Struct()
|
|
|
|
self.assertEqual(
|
|
schema.Struct(
|
|
('full', schema.Scalar(
|
|
(np.float32, (output_dims,))
|
|
)),
|
|
('random', schema.Scalar(
|
|
(np.float32, (output_dims,))
|
|
))
|
|
),
|
|
srf_output
|
|
)
|
|
|
|
init_ops_list = [
|
|
OpSpec("GaussianFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
OpSpec("GaussianFill", None, None),
|
|
OpSpec("UniformFill", None, None),
|
|
]
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
# Need to run to initialize the global constants for layer
|
|
workspace.RunNetOnce(self.model.create_init_net(name='init_net'))
|
|
|
|
if set_weight_as_global_constant:
|
|
# If weight params are global constants, they won't be in train_init_net
|
|
init_ops = self._test_net(train_init_net, init_ops_list[:2])
|
|
rand_w = workspace.FetchBlob(
|
|
self.model.global_constants['semi_random_features_fixed_rand_W']
|
|
)
|
|
rand_b = workspace.FetchBlob(
|
|
self.model.global_constants['semi_random_features_fixed_rand_b']
|
|
)
|
|
|
|
# Operation specifications
|
|
fc_random_spec = OpSpec("FC", [None, None, None], None)
|
|
fc_learned_spec = OpSpec("FC", [None, init_ops[0].output[0],
|
|
init_ops[1].output[0]], None)
|
|
else:
|
|
init_ops = self._test_net(train_init_net, init_ops_list)
|
|
rand_w = workspace.FetchBlob(self.model.layers[0].random_w)
|
|
rand_b = workspace.FetchBlob(self.model.layers[0].random_b)
|
|
|
|
# Operation specifications
|
|
fc_random_spec = OpSpec("FC", [None, init_ops[0].output[0],
|
|
init_ops[1].output[0]], None)
|
|
fc_learned_spec = OpSpec("FC", [None, init_ops[2].output[0],
|
|
init_ops[3].output[0]], None)
|
|
|
|
softsign_spec = OpSpec("Softsign", None, None)
|
|
relu_spec = OpSpec("Relu", None, None)
|
|
relu_output_spec = OpSpec("Relu", None, srf_output.random.field_blobs())
|
|
pow_spec = OpSpec("Pow", None, None, {'exponent': float(s - 1)})
|
|
mul_interim_spec = OpSpec("Mul", None, srf_output.random.field_blobs())
|
|
mul_spec = OpSpec("Mul", None, srf_output.full.field_blobs())
|
|
|
|
if s == 0:
|
|
ops_list = [
|
|
fc_learned_spec,
|
|
fc_random_spec,
|
|
softsign_spec,
|
|
relu_output_spec,
|
|
mul_spec,
|
|
]
|
|
elif s == 1:
|
|
ops_list = [
|
|
fc_learned_spec,
|
|
fc_random_spec,
|
|
relu_output_spec,
|
|
mul_spec,
|
|
]
|
|
else:
|
|
ops_list = [
|
|
fc_learned_spec,
|
|
fc_random_spec,
|
|
relu_spec,
|
|
pow_spec,
|
|
mul_interim_spec,
|
|
mul_spec,
|
|
]
|
|
|
|
# Train net assertions
|
|
self._test_net(train_net, ops_list)
|
|
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
|
|
rand_w, rand_b, s)
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self._test_net(eval_net, ops_list)
|
|
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
|
|
rand_w, rand_b, s)
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self._test_net(predict_net, ops_list)
|
|
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
|
|
rand_w, rand_b, s)
|
|
|
|
def testConv(self):
|
|
batch_size = 50
|
|
H = 1
|
|
W = 10
|
|
C = 50
|
|
output_dims = 32
|
|
kernel_h = 1
|
|
kernel_w = 3
|
|
stride_h = 1
|
|
stride_w = 1
|
|
pad_t = 0
|
|
pad_b = 0
|
|
pad_r = None
|
|
pad_l = None
|
|
|
|
input_record = self.new_record(schema.Scalar((np.float32, (H, W, C))))
|
|
X = np.random.random((batch_size, H, W, C)).astype(np.float32)
|
|
schema.FeedRecord(input_record, [X])
|
|
conv = self.model.Conv(
|
|
input_record,
|
|
output_dims,
|
|
kernel_h=kernel_h,
|
|
kernel_w=kernel_w,
|
|
stride_h=stride_h,
|
|
stride_w=stride_w,
|
|
pad_t=pad_t,
|
|
pad_b=pad_b,
|
|
pad_r=pad_r,
|
|
pad_l=pad_l,
|
|
order='NHWC'
|
|
)
|
|
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (output_dims,))),
|
|
conv
|
|
)
|
|
|
|
self.run_train_net_forward_only()
|
|
output_record = schema.FetchRecord(conv)
|
|
# check the number of output channels is the same as input in this example
|
|
assert output_record.field_types()[0].shape == (H, W, output_dims)
|
|
assert output_record().shape == (batch_size, H, W, output_dims)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
# Init net assertions
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[
|
|
OpSpec("XavierFill", None, None),
|
|
OpSpec("ConstantFill", None, None),
|
|
]
|
|
)
|
|
conv_spec = OpSpec(
|
|
"Conv",
|
|
[
|
|
input_record.field_blobs()[0],
|
|
init_ops[0].output[0],
|
|
init_ops[1].output[0],
|
|
],
|
|
conv.field_blobs()
|
|
)
|
|
|
|
# Train net assertions
|
|
self.assertNetContainOps(train_net, [conv_spec])
|
|
|
|
# Predict net assertions
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [conv_spec])
|
|
|
|
# Eval net assertions
|
|
eval_net = self.get_eval_net()
|
|
self.assertNetContainOps(eval_net, [conv_spec])
|
|
|
|
@given(
|
|
num=st.integers(min_value=10, max_value=100),
|
|
feed_weight=st.booleans(),
|
|
use_inv_var_parameterization=st.booleans(),
|
|
use_log_barrier=st.booleans(),
|
|
enable_diagnose=st.booleans(),
|
|
**hu.gcs
|
|
)
|
|
@settings(deadline=1000)
|
|
def testAdaptiveWeight(
|
|
self, num, feed_weight, use_inv_var_parameterization, use_log_barrier,
|
|
enable_diagnose, gc, dc
|
|
):
|
|
input_record = self.new_record(schema.RawTuple(num))
|
|
data = np.random.random(num)
|
|
schema.FeedRecord(
|
|
input_record, [np.array(x).astype(np.float32) for x in data]
|
|
)
|
|
weights = np.random.random(num) if feed_weight else None
|
|
result = self.model.AdaptiveWeight(
|
|
input_record,
|
|
weights=weights,
|
|
estimation_method=(
|
|
'inv_var' if use_inv_var_parameterization else 'log_std'
|
|
),
|
|
pos_optim_method=(
|
|
'log_barrier' if use_log_barrier else 'pos_grad_proj'
|
|
),
|
|
enable_diagnose=enable_diagnose
|
|
)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
result = workspace.FetchBlob(result())
|
|
if not feed_weight:
|
|
weights = np.array([1. / num for _ in range(num)])
|
|
expected = np.sum(weights * data + 0.5 * np.log(1. / 2. / weights))
|
|
npt.assert_allclose(expected, result, atol=1e-4, rtol=1e-4)
|
|
if enable_diagnose:
|
|
assert len(self.model.ad_hoc_plot_blobs) == num
|
|
reconst_weights_from_ad_hoc = np.array(
|
|
[workspace.FetchBlob(b) for b in self.model.ad_hoc_plot_blobs]
|
|
).flatten()
|
|
npt.assert_allclose(
|
|
reconst_weights_from_ad_hoc, weights, atol=1e-4, rtol=1e-4
|
|
)
|
|
else:
|
|
assert len(self.model.ad_hoc_plot_blobs) == 0
|
|
|
|
@given(num=st.integers(min_value=10, max_value=100), **hu.gcs)
|
|
def testConstantWeight(self, num, gc, dc):
|
|
input_record = self.new_record(schema.RawTuple(num))
|
|
data = np.random.random(num)
|
|
schema.FeedRecord(
|
|
input_record, [np.array(x).astype(np.float32) for x in data]
|
|
)
|
|
weights = np.random.random(num)
|
|
result = self.model.ConstantWeight(input_record, weights=weights)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
result = workspace.FetchBlob(result())
|
|
expected = np.sum(weights * data)
|
|
npt.assert_allclose(expected, result, atol=1e-4, rtol=1e-4)
|
|
|
|
@given(**hu.gcs)
|
|
@settings(deadline=10000)
|
|
def testHomotopyWeight(self, gc, dc):
|
|
input_record = self.new_record(schema.RawTuple(2))
|
|
data = np.random.random(2)
|
|
schema.FeedRecord(
|
|
input_record, [np.array(x).astype(np.float32) for x in data]
|
|
)
|
|
# ensure: quad_life > 2 * half_life
|
|
half_life = int(np.random.random() * 1e2 + 1)
|
|
quad_life = int(np.random.random() * 1e3 + 2 * half_life + 1)
|
|
min_weight = np.random.random()
|
|
max_weight = np.random.random() + min_weight + 1e-5
|
|
result = self.model.HomotopyWeight(
|
|
input_record,
|
|
min_weight=min_weight,
|
|
max_weight=max_weight,
|
|
half_life=half_life,
|
|
quad_life=quad_life,
|
|
)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.CreateNet(train_net)
|
|
workspace.RunNet(train_net.Name(), num_iter=half_life)
|
|
half_life_result = workspace.FetchBlob(result())
|
|
workspace.RunNet(train_net.Name(), num_iter=quad_life - half_life)
|
|
quad_life_result = workspace.FetchBlob(result())
|
|
|
|
alpha = (min_weight + max_weight) / 2.
|
|
beta = (min_weight + max_weight) / 2.
|
|
expected_half_life_result = alpha * data[0] + beta * data[1]
|
|
alpha = (3 * min_weight + max_weight) / 4.
|
|
beta = (min_weight + 3 * max_weight) / 4.
|
|
expected_quad_life_result = alpha * data[0] + beta * data[1]
|
|
npt.assert_allclose(
|
|
expected_half_life_result, half_life_result, atol=1e-2, rtol=1e-2
|
|
)
|
|
npt.assert_allclose(
|
|
expected_quad_life_result, quad_life_result, atol=1e-2, rtol=1e-2
|
|
)
|
|
|
|
def _testLabelSmooth(self, categories, binary_prob_label, bsz):
|
|
label = self.new_record(schema.Scalar((np.float32, (1, ))))
|
|
label_np = np.random.randint(categories, size=bsz).astype(np.float32)
|
|
schema.FeedRecord(label, [label_np])
|
|
smooth_matrix_shape = (
|
|
2 if binary_prob_label else (categories, categories)
|
|
)
|
|
smooth_matrix = np.random.random(smooth_matrix_shape)
|
|
smoothed_label = self.model.LabelSmooth(label, smooth_matrix)
|
|
train_init_net, train_net = self.get_training_nets(True)
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
smoothed_label_np = workspace.FetchBlob(smoothed_label())
|
|
if binary_prob_label:
|
|
expected = np.array(
|
|
[
|
|
smooth_matrix[0] if x == 0.0 else smooth_matrix[1]
|
|
for x in label_np
|
|
]
|
|
)
|
|
else:
|
|
expected = np.array([smooth_matrix[int(x)] for x in label_np])
|
|
npt.assert_allclose(expected, smoothed_label_np, atol=1e-4, rtol=1e-4)
|
|
|
|
@given(
|
|
categories=st.integers(min_value=2, max_value=10),
|
|
bsz=st.integers(min_value=10, max_value=100),
|
|
**hu.gcs
|
|
)
|
|
def testLabelSmoothForCategoricalLabel(self, categories, bsz, gc, dc):
|
|
self._testLabelSmooth(categories, False, bsz)
|
|
|
|
@given(
|
|
bsz=st.integers(min_value=10, max_value=100),
|
|
**hu.gcs
|
|
)
|
|
def testLabelSmoothForBinaryProbLabel(self, bsz, gc, dc):
|
|
self._testLabelSmooth(2, True, bsz)
|
|
|
|
@given(
|
|
num_inputs=st.integers(min_value=2, max_value=10),
|
|
batch_size=st.integers(min_value=2, max_value=10),
|
|
input_dim=st.integers(min_value=5, max_value=10),
|
|
seed=st.integers(1, 10),
|
|
)
|
|
def testBlobWeightedSum(self, num_inputs, batch_size, input_dim, seed):
|
|
|
|
def get_blob_weighted_sum():
|
|
weights = []
|
|
for i in range(num_inputs):
|
|
w_blob_name = 'blob_weighted_sum/w_{0}'.format(i)
|
|
assert workspace.HasBlob(w_blob_name), (
|
|
"cannot fine blob {}".format(w_blob_name)
|
|
)
|
|
w = workspace.FetchBlob(w_blob_name)
|
|
weights.append(w)
|
|
|
|
result = np.sum([
|
|
input_data[idx] * weights[idx] for idx in range(num_inputs)
|
|
], axis=0)
|
|
return result
|
|
|
|
np.random.seed(seed)
|
|
expected_output_schema = schema.Scalar((np.float32, (input_dim,)))
|
|
input_schema = schema.Tuple(
|
|
*[expected_output_schema for _ in range(num_inputs)]
|
|
)
|
|
input_data = [
|
|
np.random.random((batch_size, input_dim)).astype(np.float32)
|
|
for _ in range(num_inputs)
|
|
]
|
|
input_record = self.new_record(input_schema)
|
|
schema.FeedRecord(input_record, input_data)
|
|
|
|
# test output schema
|
|
ws_output = self.model.BlobWeightedSum(input_record)
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
assert schema.equal_schemas(ws_output, expected_output_schema)
|
|
|
|
# test train net
|
|
train_init_net, train_net = self.get_training_nets()
|
|
workspace.RunNetOnce(train_init_net)
|
|
workspace.RunNetOnce(train_net)
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
self.run_train_net_forward_only()
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
# test eval net
|
|
eval_net = self.get_eval_net()
|
|
workspace.RunNetOnce(eval_net)
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
# test pred net
|
|
pred_net = self.get_predict_net()
|
|
workspace.RunNetOnce(pred_net)
|
|
output = workspace.FetchBlob(ws_output())
|
|
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
|
|
|
|
def testFeatureSparseToDenseGetAccessedFeatures(self):
|
|
float_features_column = "float_features"
|
|
float_features_type = "FLOAT"
|
|
float_features_ids = [1, 2, 3]
|
|
|
|
id_list_features_column = "id_list_features"
|
|
id_list_features_type = "ID_LIST"
|
|
id_list_features_ids = [4, 5, 6]
|
|
|
|
id_score_list_features_column = "id_score_list_features"
|
|
id_score_list_features_type = "ID_SCORE_LIST"
|
|
id_score_list_features_ids = [7, 8 , 9]
|
|
|
|
feature_names = ["a", "b", "c"]
|
|
|
|
input_record = self.new_record(schema.Struct(
|
|
(float_features_column, schema.Map(np.int32, np.float32)),
|
|
(id_list_features_column,
|
|
schema.Map(np.int32, schema.List(np.int64))),
|
|
(id_score_list_features_column,
|
|
schema.Map(np.int32, schema.Map(np.int64, np.float32))),
|
|
))
|
|
|
|
input_specs = [
|
|
(
|
|
float_features_column,
|
|
schema.FeatureSpec(
|
|
feature_type=float_features_type,
|
|
feature_ids=float_features_ids,
|
|
feature_names=feature_names,
|
|
),
|
|
),
|
|
(
|
|
id_list_features_column,
|
|
schema.FeatureSpec(
|
|
feature_type=id_list_features_type,
|
|
feature_ids=id_list_features_ids,
|
|
feature_names=feature_names,
|
|
),
|
|
),
|
|
(
|
|
id_score_list_features_column,
|
|
schema.FeatureSpec(
|
|
feature_type=id_score_list_features_type,
|
|
feature_ids=id_score_list_features_ids,
|
|
feature_names=feature_names,
|
|
),
|
|
),
|
|
]
|
|
|
|
self.model.FeatureSparseToDense(input_record, input_specs)
|
|
|
|
expected_accessed_features = {
|
|
float_features_column: [
|
|
AccessedFeatures(float_features_type, set(float_features_ids))],
|
|
id_list_features_column: [
|
|
AccessedFeatures(id_list_features_type, set(id_list_features_ids))],
|
|
id_score_list_features_column: [
|
|
AccessedFeatures(id_score_list_features_type, set(id_score_list_features_ids))],
|
|
}
|
|
|
|
self.assertEqual(len(self.model.layers), 1)
|
|
self.assertEqual(
|
|
self.model.layers[0].get_accessed_features(),
|
|
expected_accessed_features
|
|
)
|
|
|
|
def test_get_key(self):
|
|
def _is_id_list(input_record):
|
|
return almost_equal_schemas(input_record, IdList)
|
|
|
|
|
|
def _is_id_score_list(input_record):
|
|
return almost_equal_schemas(input_record,
|
|
IdScoreList,
|
|
check_field_types=False)
|
|
|
|
def old_get_sparse_key_logic(input_record):
|
|
if _is_id_list(input_record):
|
|
sparse_key = input_record.items()
|
|
elif _is_id_score_list(input_record):
|
|
sparse_key = input_record.keys()
|
|
else:
|
|
raise NotImplementedError()
|
|
return sparse_key
|
|
|
|
id_score_list_record = schema.NewRecord(
|
|
self.model.net,
|
|
schema.Map(
|
|
schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(
|
|
categorical_limit=1000
|
|
),
|
|
),
|
|
np.float32
|
|
)
|
|
)
|
|
|
|
self.assertEqual(
|
|
get_key(id_score_list_record)(),
|
|
old_get_sparse_key_logic(id_score_list_record)
|
|
)
|
|
|
|
id_list_record = schema.NewRecord(
|
|
self.model.net,
|
|
schema.List(
|
|
schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000)
|
|
)
|
|
)
|
|
)
|
|
|
|
self.assertEqual(
|
|
get_key(id_list_record)(),
|
|
old_get_sparse_key_logic(id_list_record)
|
|
)
|
|
|
|
def testSparseLookupWithAttentionWeightOnIdScoreList(self):
|
|
record = schema.NewRecord(
|
|
self.model.net,
|
|
schema.Map(
|
|
schema.Scalar(
|
|
np.int64,
|
|
metadata=schema.Metadata(categorical_limit=1000),
|
|
),
|
|
np.float32,
|
|
),
|
|
)
|
|
embedding_dim = 64
|
|
embedding_after_pooling = self.model.SparseLookup(
|
|
record, [embedding_dim], "Sum", use_external_weights=True
|
|
)
|
|
self.model.output_schema = schema.Struct()
|
|
self.assertEqual(
|
|
schema.Scalar((np.float32, (embedding_dim,))), embedding_after_pooling
|
|
)
|
|
|
|
train_init_net, train_net = self.get_training_nets()
|
|
|
|
init_ops = self.assertNetContainOps(
|
|
train_init_net,
|
|
[OpSpec("UniformFill", None, None), OpSpec("ConstantFill", None, None)],
|
|
)
|
|
sparse_lookup_op_spec = OpSpec(
|
|
"SparseLengthsWeightedSum",
|
|
[
|
|
init_ops[0].output[0],
|
|
record.values(),
|
|
record.keys(),
|
|
record.lengths(),
|
|
],
|
|
[embedding_after_pooling()],
|
|
)
|
|
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
|
|
|
|
predict_net = self.get_predict_net()
|
|
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
|