How to use evaluate method in chromeless

...44    self.assertEqual(m.name, 'my_mean')45    self.assertTrue(m.stateful)46    self.assertEqual(m.dtype, dtypes.float32)47    self.assertEqual(len(m.variables), 2)48    self.evaluate(variables.variables_initializer(m.variables))49    # check initial state50    self.assertEqual(self.evaluate(m.total), 0)51    self.assertEqual(self.evaluate(m.count), 0)52    # check __call__()53    self.assertEqual(self.evaluate(m(100)), 100)54    self.assertEqual(self.evaluate(m.total), 100)55    self.assertEqual(self.evaluate(m.count), 1)56    # check update_state() and result() + state accumulation + tensor input57    update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))58    self.evaluate(update_op)59    self.assertAlmostEqual(self.evaluate(m.result()), 106 / 3, 2)60    self.assertEqual(self.evaluate(m.total), 106)  # 100 + 1 + 561    self.assertEqual(self.evaluate(m.count), 3)62    # check reset_states()63    m.reset_states()64    self.assertEqual(self.evaluate(m.total), 0)65    self.assertEqual(self.evaluate(m.count), 0)66    # Check save and restore config67    m2 = metrics.Mean.from_config(m.get_config())68    self.assertEqual(m2.name, 'my_mean')69    self.assertTrue(m2.stateful)70    self.assertEqual(m2.dtype, dtypes.float32)71    self.assertEqual(len(m2.variables), 2)72  def test_mean_with_sample_weight(self):73    m = metrics.Mean(dtype=dtypes.float64)74    self.assertEqual(m.dtype, dtypes.float64)75    self.evaluate(variables.variables_initializer(m.variables))76    # check scalar weight77    result_t = m(100, sample_weight=0.5)78    self.assertEqual(self.evaluate(result_t), 50 / 0.5)79    self.assertEqual(self.evaluate(m.total), 50)80    self.assertEqual(self.evaluate(m.count), 0.5)81    # check weights not scalar and weights rank matches values rank82    result_t = m([1, 5], sample_weight=[1, 0.2])83    result = self.evaluate(result_t)84    self.assertAlmostEqual(result, 52 / 1.7, 2)85    self.assertAlmostEqual(self.evaluate(m.total), 52, 2)  # 50 + 1 + 5 * 0.286    self.assertAlmostEqual(self.evaluate(m.count), 1.7, 2)  # 0.5 + 1.287    # check weights broadcast88    result_t = m([1, 2], sample_weight=0.5)89    self.assertAlmostEqual(self.evaluate(result_t), 53.5 / 2.7, 2)90    self.assertAlmostEqual(self.evaluate(m.total), 53.5, 2)  # 52 + 0.5 + 191    self.assertAlmostEqual(self.evaluate(m.count), 2.7, 2)  # 1.7 + 0.5 + 0.592    # check weights squeeze93    result_t = m([1, 5], sample_weight=[[1], [0.2]])94    self.assertAlmostEqual(self.evaluate(result_t), 55.5 / 3.9, 2)95    self.assertAlmostEqual(self.evaluate(m.total), 55.5, 2)  # 53.5 + 1 + 196    self.assertAlmostEqual(self.evaluate(m.count), 3.9, 2)  # 2.7 + 1.297    # check weights expand98    result_t = m([[1], [5]], sample_weight=[1, 0.2])99    self.assertAlmostEqual(self.evaluate(result_t), 57.5 / 5.1, 2)100    self.assertAlmostEqual(self.evaluate(m.total), 57.5, 2)  # 55.5 + 1 + 1101    self.assertAlmostEqual(self.evaluate(m.count), 5.1, 2)  # 3.9 + 1.2102    # check values reduced to the dimensions of weight103    result_t = m([[[1., 2.], [3., 2.], [0.5, 4.]]], sample_weight=[0.5])104    result = np.round(self.evaluate(result_t), decimals=2)  # 58.5 / 5.6105    self.assertEqual(result, 10.45)106    self.assertEqual(np.round(self.evaluate(m.total), decimals=2), 58.54)107    self.assertEqual(np.round(self.evaluate(m.count), decimals=2), 5.6)108  def test_mean_graph_with_placeholder(self):109    with context.graph_mode(), self.cached_session() as sess:110      m = metrics.Mean()111      v = array_ops.placeholder(dtypes.float32)112      w = array_ops.placeholder(dtypes.float32)113      self.evaluate(variables.variables_initializer(m.variables))114      # check __call__()115      result_t = m(v, sample_weight=w)116      result = sess.run(result_t, feed_dict=({v: 100, w: 0.5}))117      self.assertEqual(self.evaluate(m.total), 50)118      self.assertEqual(self.evaluate(m.count), 0.5)119      self.assertEqual(result, 50 / 0.5)120      # check update_state() and result()121      result = sess.run(result_t, feed_dict=({v: [1, 5], w: [1, 0.2]}))122      self.assertAlmostEqual(self.evaluate(m.total), 52, 2)  # 50 + 1 + 5 * 0.2123      self.assertAlmostEqual(self.evaluate(m.count), 1.7, 2)  # 0.5 + 1.2124      self.assertAlmostEqual(result, 52 / 1.7, 2)125  def test_save_restore(self):126    checkpoint_directory = self.get_temp_dir()127    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')128    m = metrics.Mean()129    checkpoint = checkpointable_utils.Checkpoint(mean=m)130    self.evaluate(variables.variables_initializer(m.variables))131    # update state132    self.evaluate(m(100.))133    self.evaluate(m(200.))134    # save checkpoint and then add an update135    save_path = checkpoint.save(checkpoint_prefix)136    self.evaluate(m(1000.))137    # restore to the same checkpoint mean object138    checkpoint.restore(save_path).assert_consumed().run_restore_ops()139    self.evaluate(m(300.))140    self.assertEqual(200., self.evaluate(m.result()))141    # restore to a different checkpoint mean object142    restore_mean = metrics.Mean()143    restore_checkpoint = checkpointable_utils.Checkpoint(mean=restore_mean)144    status = restore_checkpoint.restore(save_path)145    restore_update = restore_mean(300.)146    status.assert_consumed().run_restore_ops()147    self.evaluate(restore_update)148    self.assertEqual(200., self.evaluate(restore_mean.result()))149    self.assertEqual(3, self.evaluate(restore_mean.count))150@test_util.run_all_in_graph_and_eager_modes151class KerasAccuracyTest(test.TestCase):152  def test_accuracy(self):153    acc_obj = metrics.Accuracy(name='my acc')154    # check config155    self.assertEqual(acc_obj.name, 'my acc')156    self.assertTrue(acc_obj.stateful)157    self.assertEqual(len(acc_obj.variables), 2)158    self.assertEqual(acc_obj.dtype, dtypes.float32)159    self.evaluate(variables.variables_initializer(acc_obj.variables))160    # verify that correct value is returned161    update_op = acc_obj.update_state([[1], [2], [3], [4]], [[1], [2], [3], [4]])162    self.evaluate(update_op)163    result = self.evaluate(acc_obj.result())164    self.assertEqual(result, 1)  # 2/2165    # Check save and restore config166    a2 = metrics.Accuracy.from_config(acc_obj.get_config())167    self.assertEqual(a2.name, 'my acc')168    self.assertTrue(a2.stateful)169    self.assertEqual(len(a2.variables), 2)170    self.assertEqual(a2.dtype, dtypes.float32)171    # check with sample_weight172    result_t = acc_obj([[2], [1]], [[2], [0]], sample_weight=[[0.5], [0.2]])173    result = self.evaluate(result_t)174    self.assertAlmostEqual(result, 0.96, 2)  # 4.5/4.7175  def test_binary_accuracy(self):176    acc_obj = metrics.BinaryAccuracy(name='my acc')177    # check config178    self.assertEqual(acc_obj.name, 'my acc')179    self.assertTrue(acc_obj.stateful)180    self.assertEqual(len(acc_obj.variables), 2)181    self.assertEqual(acc_obj.dtype, dtypes.float32)182    self.evaluate(variables.variables_initializer(acc_obj.variables))183    # verify that correct value is returned184    update_op = acc_obj.update_state([[1], [0]], [[1], [0]])185    self.evaluate(update_op)186    result = self.evaluate(acc_obj.result())187    self.assertEqual(result, 1)  # 2/2188    # check y_pred squeeze189    update_op = acc_obj.update_state([[1], [1]], [[[1]], [[0]]])190    self.evaluate(update_op)191    result = self.evaluate(acc_obj.result())192    self.assertAlmostEqual(result, 0.75, 2)  # 3/4193    # check y_true squeeze194    result_t = acc_obj([[[1]], [[1]]], [[1], [0]])195    result = self.evaluate(result_t)196    self.assertAlmostEqual(result, 0.67, 2)  # 4/6197    # check with sample_weight198    result_t = acc_obj([[1], [1]], [[1], [0]], [[0.5], [0.2]])199    result = self.evaluate(result_t)200    self.assertAlmostEqual(result, 0.67, 2)  # 4.5/6.7201  def test_binary_accuracy_threshold(self):202    acc_obj = metrics.BinaryAccuracy(threshold=0.7)203    self.evaluate(variables.variables_initializer(acc_obj.variables))204    result_t = acc_obj([[1], [1], [0], [0]], [[0.9], [0.6], [0.4], [0.8]])205    result = self.evaluate(result_t)206    self.assertAlmostEqual(result, 0.5, 2)207  def test_categorical_accuracy(self):208    acc_obj = metrics.CategoricalAccuracy(name='my acc')209    # check config210    self.assertEqual(acc_obj.name, 'my acc')211    self.assertTrue(acc_obj.stateful)212    self.assertEqual(len(acc_obj.variables), 2)213    self.assertEqual(acc_obj.dtype, dtypes.float32)214    self.evaluate(variables.variables_initializer(acc_obj.variables))215    # verify that correct value is returned216    update_op = acc_obj.update_state([[0, 0, 1], [0, 1, 0]],217                                     [[0.1, 0.1, 0.8], [0.05, 0.95, 0]])218    self.evaluate(update_op)219    result = self.evaluate(acc_obj.result())220    self.assertEqual(result, 1)  # 2/2221    # check with sample_weight222    result_t = acc_obj([[0, 0, 1], [0, 1, 0]],223                       [[0.1, 0.1, 0.8], [0.05, 0, 0.95]], [[0.5], [0.2]])224    result = self.evaluate(result_t)225    self.assertAlmostEqual(result, 0.93, 2)  # 2.5/2.7226  def test_sparse_categorical_accuracy(self):227    acc_obj = metrics.SparseCategoricalAccuracy(name='my acc')228    # check config229    self.assertEqual(acc_obj.name, 'my acc')230    self.assertTrue(acc_obj.stateful)231    self.assertEqual(len(acc_obj.variables), 2)232    self.assertEqual(acc_obj.dtype, dtypes.float32)233    self.evaluate(variables.variables_initializer(acc_obj.variables))234    # verify that correct value is returned235    update_op = acc_obj.update_state([[2], [1]],236                                     [[0.1, 0.1, 0.8], [0.05, 0.95, 0]])237    self.evaluate(update_op)238    result = self.evaluate(acc_obj.result())239    self.assertEqual(result, 1)  # 2/2240    # check with sample_weight241    result_t = acc_obj([[2], [1]], [[0.1, 0.1, 0.8], [0.05, 0, 0.95]],242                       [[0.5], [0.2]])243    result = self.evaluate(result_t)244    self.assertAlmostEqual(result, 0.93, 2)  # 2.5/2.7245  def test_sparse_categorical_accuracy_mismatched_dims(self):246    acc_obj = metrics.SparseCategoricalAccuracy(name='my acc')247    # check config248    self.assertEqual(acc_obj.name, 'my acc')249    self.assertTrue(acc_obj.stateful)250    self.assertEqual(len(acc_obj.variables), 2)251    self.assertEqual(acc_obj.dtype, dtypes.float32)252    self.evaluate(variables.variables_initializer(acc_obj.variables))253    # verify that correct value is returned254    update_op = acc_obj.update_state([2, 1], [[0.1, 0.1, 0.8], [0.05, 0.95, 0]])255    self.evaluate(update_op)256    result = self.evaluate(acc_obj.result())257    self.assertEqual(result, 1)  # 2/2258    # check with sample_weight259    result_t = acc_obj([2, 1], [[0.1, 0.1, 0.8], [0.05, 0, 0.95]],260                       [[0.5], [0.2]])261    result = self.evaluate(result_t)262    self.assertAlmostEqual(result, 0.93, 2)  # 2.5/2.7263  def test_sparse_categorical_accuracy_mismatched_dims_dynamic(self):264    with context.graph_mode(), self.cached_session() as sess:265      acc_obj = metrics.SparseCategoricalAccuracy(name='my acc')266      self.evaluate(variables.variables_initializer(acc_obj.variables))267      t = array_ops.placeholder(dtypes.float32)268      p = array_ops.placeholder(dtypes.float32)269      w = array_ops.placeholder(dtypes.float32)270      result_t = acc_obj(t, p, w)271      result = sess.run(272          result_t,273          feed_dict=({274              t: [2, 1],275              p: [[0.1, 0.1, 0.8], [0.05, 0, 0.95]],276              w: [[0.5], [0.2]]277          }))278      self.assertAlmostEqual(result, 0.71, 2)  # 2.5/2.7279@test_util.run_all_in_graph_and_eager_modes280class CosineProximityTest(test.TestCase):281  def l2_norm(self, x, axis):282    epsilon = 1e-12283    square_sum = np.sum(np.square(x), axis=axis, keepdims=True)284    x_inv_norm = 1 / np.sqrt(np.maximum(square_sum, epsilon))285    return np.multiply(x, x_inv_norm)286  def setup(self, axis=1):287    self.np_y_true = np.asarray([[1, 9, 2], [-5, -2, 6]], dtype=np.float32)288    self.np_y_pred = np.asarray([[4, 8, 12], [8, 1, 3]], dtype=np.float32)289    y_true = self.l2_norm(self.np_y_true, axis)290    y_pred = self.l2_norm(self.np_y_pred, axis)291    self.expected_loss = -np.sum(np.multiply(y_true, y_pred), axis=(axis,))292    self.y_true = constant_op.constant(self.np_y_true)293    self.y_pred = constant_op.constant(self.np_y_pred)294  def test_config(self):295    cosine_obj = metrics.CosineProximity(296        axis=2, name='my_cos', dtype=dtypes.int32)297    self.assertEqual(cosine_obj.name, 'my_cos')298    self.assertEqual(cosine_obj._dtype, dtypes.int32)299    # Check save and restore config300    cosine_obj2 = metrics.CosineProximity.from_config(cosine_obj.get_config())301    self.assertEqual(cosine_obj2.name, 'my_cos')302    self.assertEqual(cosine_obj2._dtype, dtypes.int32)303  def test_unweighted(self):304    self.setup()305    cosine_obj = metrics.CosineProximity()306    self.evaluate(variables.variables_initializer(cosine_obj.variables))307    loss = cosine_obj(self.y_true, self.y_pred)308    expected_loss = np.mean(self.expected_loss)309    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)310  def test_weighted(self):311    self.setup()312    cosine_obj = metrics.CosineProximity()313    self.evaluate(variables.variables_initializer(cosine_obj.variables))314    sample_weight = np.asarray([1.2, 3.4])315    loss = cosine_obj(316        self.y_true,317        self.y_pred,318        sample_weight=constant_op.constant(sample_weight))319    expected_loss = np.sum(320        self.expected_loss * sample_weight) / np.sum(sample_weight)321    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)322  def test_axis(self):323    self.setup(axis=1)324    cosine_obj = metrics.CosineProximity(axis=1)325    self.evaluate(variables.variables_initializer(cosine_obj.variables))326    loss = cosine_obj(self.y_true, self.y_pred)327    expected_loss = np.mean(self.expected_loss)328    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)329@test_util.run_all_in_graph_and_eager_modes330class MeanAbsoluteErrorTest(test.TestCase):331  def test_config(self):332    mae_obj = metrics.MeanAbsoluteError(name='my_mae', dtype=dtypes.int32)333    self.assertEqual(mae_obj.name, 'my_mae')334    self.assertEqual(mae_obj._dtype, dtypes.int32)335    # Check save and restore config336    mae_obj2 = metrics.MeanAbsoluteError.from_config(mae_obj.get_config())337    self.assertEqual(mae_obj2.name, 'my_mae')338    self.assertEqual(mae_obj2._dtype, dtypes.int32)339  def test_unweighted(self):340    mae_obj = metrics.MeanAbsoluteError()341    self.evaluate(variables.variables_initializer(mae_obj.variables))342    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),343                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))344    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),345                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))346    update_op = mae_obj.update_state(y_true, y_pred)347    self.evaluate(update_op)348    result = mae_obj.result()349    self.assertAllClose(0.5, result, atol=1e-5)350  def test_weighted(self):351    mae_obj = metrics.MeanAbsoluteError()352    self.evaluate(variables.variables_initializer(mae_obj.variables))353    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),354                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))355    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),356                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))357    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))358    result = mae_obj(y_true, y_pred, sample_weight=sample_weight)359    self.assertAllClose(0.54285, self.evaluate(result), atol=1e-5)360@test_util.run_all_in_graph_and_eager_modes361class MeanAbsolutePercentageErrorTest(test.TestCase):362  def test_config(self):363    mape_obj = metrics.MeanAbsolutePercentageError(364        name='my_mape', dtype=dtypes.int32)365    self.assertEqual(mape_obj.name, 'my_mape')366    self.assertEqual(mape_obj._dtype, dtypes.int32)367    # Check save and restore config368    mape_obj2 = metrics.MeanAbsolutePercentageError.from_config(369        mape_obj.get_config())370    self.assertEqual(mape_obj2.name, 'my_mape')371    self.assertEqual(mape_obj2._dtype, dtypes.int32)372  def test_unweighted(self):373    mape_obj = metrics.MeanAbsolutePercentageError()374    self.evaluate(variables.variables_initializer(mape_obj.variables))375    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),376                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))377    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),378                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))379    update_op = mape_obj.update_state(y_true, y_pred)380    self.evaluate(update_op)381    result = mape_obj.result()382    self.assertAllClose(35e7, result, atol=1e-5)383  def test_weighted(self):384    mape_obj = metrics.MeanAbsolutePercentageError()385    self.evaluate(variables.variables_initializer(mape_obj.variables))386    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),387                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))388    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),389                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))390    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))391    result = mape_obj(y_true, y_pred, sample_weight=sample_weight)392    self.assertAllClose(40e7, self.evaluate(result), atol=1e-5)393@test_util.run_all_in_graph_and_eager_modes394class MeanSquaredErrorTest(test.TestCase):395  def test_config(self):396    mse_obj = metrics.MeanSquaredError(name='my_mse', dtype=dtypes.int32)397    self.assertEqual(mse_obj.name, 'my_mse')398    self.assertEqual(mse_obj._dtype, dtypes.int32)399    # Check save and restore config400    mse_obj2 = metrics.MeanSquaredError.from_config(mse_obj.get_config())401    self.assertEqual(mse_obj2.name, 'my_mse')402    self.assertEqual(mse_obj2._dtype, dtypes.int32)403  def test_unweighted(self):404    mse_obj = metrics.MeanSquaredError()405    self.evaluate(variables.variables_initializer(mse_obj.variables))406    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),407                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))408    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),409                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))410    update_op = mse_obj.update_state(y_true, y_pred)411    self.evaluate(update_op)412    result = mse_obj.result()413    self.assertAllClose(0.5, result, atol=1e-5)414  def test_weighted(self):415    mse_obj = metrics.MeanSquaredError()416    self.evaluate(variables.variables_initializer(mse_obj.variables))417    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),418                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))419    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),420                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))421    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))422    result = mse_obj(y_true, y_pred, sample_weight=sample_weight)423    self.assertAllClose(0.54285, self.evaluate(result), atol=1e-5)424@test_util.run_all_in_graph_and_eager_modes425class MeanSquaredLogarithmicErrorTest(test.TestCase):426  def test_config(self):427    msle_obj = metrics.MeanSquaredLogarithmicError(428        name='my_msle', dtype=dtypes.int32)429    self.assertEqual(msle_obj.name, 'my_msle')430    self.assertEqual(msle_obj._dtype, dtypes.int32)431    # Check save and restore config432    msle_obj2 = metrics.MeanSquaredLogarithmicError.from_config(433        msle_obj.get_config())434    self.assertEqual(msle_obj2.name, 'my_msle')435    self.assertEqual(msle_obj2._dtype, dtypes.int32)436  def test_unweighted(self):437    msle_obj = metrics.MeanSquaredLogarithmicError()438    self.evaluate(variables.variables_initializer(msle_obj.variables))439    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),440                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))441    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),442                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))443    update_op = msle_obj.update_state(y_true, y_pred)444    self.evaluate(update_op)445    result = msle_obj.result()446    self.assertAllClose(0.24022, result, atol=1e-5)447  def test_weighted(self):448    msle_obj = metrics.MeanSquaredLogarithmicError()449    self.evaluate(variables.variables_initializer(msle_obj.variables))450    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),451                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))452    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),453                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))454    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))455    result = msle_obj(y_true, y_pred, sample_weight=sample_weight)456    self.assertAllClose(0.26082, self.evaluate(result), atol=1e-5)457@test_util.run_all_in_graph_and_eager_modes458class HingeTest(test.TestCase):459  def test_config(self):460    hinge_obj = metrics.Hinge(name='hinge', dtype=dtypes.int32)461    self.assertEqual(hinge_obj.name, 'hinge')462    self.assertEqual(hinge_obj._dtype, dtypes.int32)463    # Check save and restore config464    hinge_obj2 = metrics.Hinge.from_config(hinge_obj.get_config())465    self.assertEqual(hinge_obj2.name, 'hinge')466    self.assertEqual(hinge_obj2._dtype, dtypes.int32)467  def test_unweighted(self):468    hinge_obj = metrics.Hinge()469    self.evaluate(variables.variables_initializer(hinge_obj.variables))470    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),471                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))472    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),473                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))474    update_op = hinge_obj.update_state(y_true, y_pred)475    self.evaluate(update_op)476    result = hinge_obj.result()477    self.assertAllClose(0.65, result, atol=1e-5)478  def test_weighted(self):479    hinge_obj = metrics.Hinge()480    self.evaluate(variables.variables_initializer(hinge_obj.variables))481    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),482                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))483    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),484                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))485    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))486    result = hinge_obj(y_true, y_pred, sample_weight=sample_weight)487    self.assertAllClose(0.65714, self.evaluate(result), atol=1e-5)488@test_util.run_all_in_graph_and_eager_modes489class SquaredHingeTest(test.TestCase):490  def test_config(self):491    sq_hinge_obj = metrics.SquaredHinge(name='sq_hinge', dtype=dtypes.int32)492    self.assertEqual(sq_hinge_obj.name, 'sq_hinge')493    self.assertEqual(sq_hinge_obj._dtype, dtypes.int32)494    # Check save and restore config495    sq_hinge_obj2 = metrics.SquaredHinge.from_config(sq_hinge_obj.get_config())496    self.assertEqual(sq_hinge_obj2.name, 'sq_hinge')497    self.assertEqual(sq_hinge_obj2._dtype, dtypes.int32)498  def test_unweighted(self):499    sq_hinge_obj = metrics.SquaredHinge()500    self.evaluate(variables.variables_initializer(sq_hinge_obj.variables))501    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),502                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))503    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),504                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))505    update_op = sq_hinge_obj.update_state(y_true, y_pred)506    self.evaluate(update_op)507    result = sq_hinge_obj.result()508    self.assertAllClose(0.65, result, atol=1e-5)509  def test_weighted(self):510    sq_hinge_obj = metrics.SquaredHinge()511    self.evaluate(variables.variables_initializer(sq_hinge_obj.variables))512    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),513                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))514    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),515                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))516    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))517    result = sq_hinge_obj(y_true, y_pred, sample_weight=sample_weight)518    self.assertAllClose(0.65714, self.evaluate(result), atol=1e-5)519@test_util.run_all_in_graph_and_eager_modes520class CategoricalHingeTest(test.TestCase):521  def test_config(self):522    cat_hinge_obj = metrics.CategoricalHinge(523        name='cat_hinge', dtype=dtypes.int32)524    self.assertEqual(cat_hinge_obj.name, 'cat_hinge')525    self.assertEqual(cat_hinge_obj._dtype, dtypes.int32)526    # Check save and restore config527    cat_hinge_obj2 = metrics.CategoricalHinge.from_config(528        cat_hinge_obj.get_config())529    self.assertEqual(cat_hinge_obj2.name, 'cat_hinge')530    self.assertEqual(cat_hinge_obj2._dtype, dtypes.int32)531  def test_unweighted(self):532    cat_hinge_obj = metrics.CategoricalHinge()533    self.evaluate(variables.variables_initializer(cat_hinge_obj.variables))534    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),535                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))536    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),537                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))538    update_op = cat_hinge_obj.update_state(y_true, y_pred)539    self.evaluate(update_op)540    result = cat_hinge_obj.result()541    self.assertAllClose(0.5, result, atol=1e-5)542  def test_weighted(self):543    cat_hinge_obj = metrics.CategoricalHinge()544    self.evaluate(variables.variables_initializer(cat_hinge_obj.variables))545    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),546                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))547    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),548                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))549    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))550    result = cat_hinge_obj(y_true, y_pred, sample_weight=sample_weight)551    self.assertAllClose(0.5, self.evaluate(result), atol=1e-5)552@test_util.run_all_in_graph_and_eager_modes553class RootMeanSquaredErrorTest(test.TestCase):554  def test_config(self):555    rmse_obj = metrics.RootMeanSquaredError(name='rmse', dtype=dtypes.int32)556    self.assertEqual(rmse_obj.name, 'rmse')557    self.assertEqual(rmse_obj._dtype, dtypes.int32)558    rmse_obj2 = metrics.RootMeanSquaredError.from_config(rmse_obj.get_config())559    self.assertEqual(rmse_obj2.name, 'rmse')560    self.assertEqual(rmse_obj2._dtype, dtypes.int32)561  def test_unweighted(self):562    rmse_obj = metrics.RootMeanSquaredError()563    self.evaluate(variables.variables_initializer(rmse_obj.variables))564    y_true = constant_op.constant((2, 4, 6))565    y_pred = constant_op.constant((1, 3, 2))566    update_op = rmse_obj.update_state(y_true, y_pred)567    self.evaluate(update_op)568    result = rmse_obj.result()569    # error = [-1, -1, -4], square(error) = [1, 1, 16], mean = 18/3 = 6570    self.assertAllClose(math.sqrt(6), result, atol=1e-3)571  def test_weighted(self):572    rmse_obj = metrics.RootMeanSquaredError()573    self.evaluate(variables.variables_initializer(rmse_obj.variables))574    y_true = constant_op.constant((2, 4, 6, 8))575    y_pred = constant_op.constant((1, 3, 2, 3))576    sample_weight = constant_op.constant((0, 1, 0, 1))577    result = rmse_obj(y_true, y_pred, sample_weight=sample_weight)578    self.assertAllClose(math.sqrt(13), self.evaluate(result), atol=1e-3)579@test_util.run_all_in_graph_and_eager_modes580class TopKCategoricalAccuracyTest(test.TestCase):581  def test_config(self):582    a_obj = metrics.TopKCategoricalAccuracy(name='topkca', dtype=dtypes.int32)583    self.assertEqual(a_obj.name, 'topkca')584    self.assertEqual(a_obj._dtype, dtypes.int32)585    a_obj2 = metrics.TopKCategoricalAccuracy.from_config(a_obj.get_config())586    self.assertEqual(a_obj2.name, 'topkca')587    self.assertEqual(a_obj2._dtype, dtypes.int32)588  def test_correctness(self):589    a_obj = metrics.TopKCategoricalAccuracy()590    self.evaluate(variables.variables_initializer(a_obj.variables))591    y_true = constant_op.constant([[0, 0, 1], [0, 1, 0]])592    y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]])593    result = a_obj(y_true, y_pred)594    self.assertEqual(1, self.evaluate(result))  # both the samples match595    # With `k` < 5.596    a_obj = metrics.TopKCategoricalAccuracy(k=1)597    self.evaluate(variables.variables_initializer(a_obj.variables))598    result = a_obj(y_true, y_pred)599    self.assertEqual(0.5, self.evaluate(result))  # only sample #2 matches600    # With `k` > 5.601    y_true = constant_op.constant([[0, 0, 1, 0, 0, 0, 0],602                                   [0, 1, 0, 0, 0, 0, 0]])603    y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4],604                                   [0.05, 0.95, 0, 0, 0, 0, 0]])605    a_obj = metrics.TopKCategoricalAccuracy(k=6)606    self.evaluate(variables.variables_initializer(a_obj.variables))607    result = a_obj(y_true, y_pred)608    self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.609@test_util.run_all_in_graph_and_eager_modes610class SparseTopKCategoricalAccuracyTest(test.TestCase):611  def test_config(self):612    a_obj = metrics.SparseTopKCategoricalAccuracy(613        name='stopkca', dtype=dtypes.int32)614    self.assertEqual(a_obj.name, 'stopkca')615    self.assertEqual(a_obj._dtype, dtypes.int32)616    a_obj2 = metrics.SparseTopKCategoricalAccuracy.from_config(617        a_obj.get_config())618    self.assertEqual(a_obj2.name, 'stopkca')619    self.assertEqual(a_obj2._dtype, dtypes.int32)620  def test_correctness(self):621    a_obj = metrics.SparseTopKCategoricalAccuracy()622    self.evaluate(variables.variables_initializer(a_obj.variables))623    y_true = constant_op.constant([2, 1])624    y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]])625    result = a_obj(y_true, y_pred)626    self.assertEqual(1, self.evaluate(result))  # both the samples match627    # With `k` < 5.628    a_obj = metrics.SparseTopKCategoricalAccuracy(k=1)629    self.evaluate(variables.variables_initializer(a_obj.variables))630    result = a_obj(y_true, y_pred)631    self.assertEqual(0.5, self.evaluate(result))  # only sample #2 matches632    # With `k` > 5.633    y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4],634                                   [0.05, 0.95, 0, 0, 0, 0, 0]])635    a_obj = metrics.SparseTopKCategoricalAccuracy(k=6)636    self.evaluate(variables.variables_initializer(a_obj.variables))637    result = a_obj(y_true, y_pred)638    self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.639@test_util.run_all_in_graph_and_eager_modes640class LogcoshTest(test.TestCase):641  def setup(self):642    y_pred = np.asarray([1, 9, 2, -5, -2, 6]).reshape((2, 3))643    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))644    self.batch_size = 6645    error = y_pred - y_true646    self.expected_results = np.log((np.exp(error) + np.exp(-error)) / 2)647    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)648    self.y_true = constant_op.constant(y_true)649  def test_config(self):650    logcosh_obj = metrics.Logcosh(name='logcosh', dtype=dtypes.int32)651    self.assertEqual(logcosh_obj.name, 'logcosh')652    self.assertEqual(logcosh_obj._dtype, dtypes.int32)653  def test_unweighted(self):654    self.setup()655    logcosh_obj = metrics.Logcosh()656    self.evaluate(variables.variables_initializer(logcosh_obj.variables))657    update_op = logcosh_obj.update_state(self.y_true, self.y_pred)658    self.evaluate(update_op)659    result = logcosh_obj.result()660    expected_result = np.sum(self.expected_results) / self.batch_size661    self.assertAllClose(result, expected_result, atol=1e-3)662  def test_weighted(self):663    self.setup()664    logcosh_obj = metrics.Logcosh()665    self.evaluate(variables.variables_initializer(logcosh_obj.variables))666    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))667    result = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)668    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))669    expected_result = np.multiply(self.expected_results, sample_weight)670    expected_result = np.sum(expected_result) / np.sum(sample_weight)671    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)672@test_util.run_all_in_graph_and_eager_modes673class PoissonTest(test.TestCase):674  def setup(self):675    y_pred = np.asarray([1, 9, 2, 5, 2, 6]).reshape((2, 3))676    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))677    self.batch_size = 6678    self.expected_results = y_pred - np.multiply(y_true, np.log(y_pred))679    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)680    self.y_true = constant_op.constant(y_true)681  def test_config(self):682    poisson_obj = metrics.Poisson(name='poisson', dtype=dtypes.int32)683    self.assertEqual(poisson_obj.name, 'poisson')684    self.assertEqual(poisson_obj._dtype, dtypes.int32)685    poisson_obj2 = metrics.Poisson.from_config(poisson_obj.get_config())686    self.assertEqual(poisson_obj2.name, 'poisson')687    self.assertEqual(poisson_obj2._dtype, dtypes.int32)688  def test_unweighted(self):689    self.setup()690    poisson_obj = metrics.Poisson()691    self.evaluate(variables.variables_initializer(poisson_obj.variables))692    update_op = poisson_obj.update_state(self.y_true, self.y_pred)693    self.evaluate(update_op)694    result = poisson_obj.result()695    expected_result = np.sum(self.expected_results) / self.batch_size696    self.assertAllClose(result, expected_result, atol=1e-3)697  def test_weighted(self):698    self.setup()699    poisson_obj = metrics.Poisson()700    self.evaluate(variables.variables_initializer(poisson_obj.variables))701    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))702    result = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)703    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))704    expected_result = np.multiply(self.expected_results, sample_weight)705    expected_result = np.sum(expected_result) / np.sum(sample_weight)706    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)707@test_util.run_all_in_graph_and_eager_modes708class KullbackLeiblerDivergenceTest(test.TestCase):709  def setup(self):710    y_pred = np.asarray([.4, .9, .12, .36, .3, .4]).reshape((2, 3))711    y_true = np.asarray([.5, .8, .12, .7, .43, .8]).reshape((2, 3))712    self.batch_size = 2713    self.expected_results = np.multiply(y_true, np.log(y_true / y_pred))714    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)715    self.y_true = constant_op.constant(y_true)716  def test_config(self):717    k_obj = metrics.KullbackLeiblerDivergence(name='kld', dtype=dtypes.int32)718    self.assertEqual(k_obj.name, 'kld')719    self.assertEqual(k_obj._dtype, dtypes.int32)720    k_obj2 = metrics.KullbackLeiblerDivergence.from_config(k_obj.get_config())721    self.assertEqual(k_obj2.name, 'kld')722    self.assertEqual(k_obj2._dtype, dtypes.int32)723  def test_unweighted(self):724    self.setup()725    k_obj = metrics.KullbackLeiblerDivergence()726    self.evaluate(variables.variables_initializer(k_obj.variables))727    update_op = k_obj.update_state(self.y_true, self.y_pred)728    self.evaluate(update_op)729    result = k_obj.result()730    expected_result = np.sum(self.expected_results) / self.batch_size731    self.assertAllClose(result, expected_result, atol=1e-3)732  def test_weighted(self):733    self.setup()734    k_obj = metrics.KullbackLeiblerDivergence()735    self.evaluate(variables.variables_initializer(k_obj.variables))736    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))737    result = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)738    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))739    expected_result = np.multiply(self.expected_results, sample_weight)740    expected_result = np.sum(expected_result) / (1.2 + 3.4)741    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)742@test_util.run_all_in_graph_and_eager_modes743class MeanRelativeErrorTest(test.TestCase):744  def test_config(self):745    normalizer = constant_op.constant([1, 3], dtype=dtypes.float32)746    mre_obj = metrics.MeanRelativeError(normalizer=normalizer, name='mre')747    self.assertEqual(mre_obj.name, 'mre')748    self.assertArrayNear(self.evaluate(mre_obj.normalizer), [1, 3], 1e-1)749    mre_obj2 = metrics.MeanRelativeError.from_config(mre_obj.get_config())750    self.assertEqual(mre_obj2.name, 'mre')751    self.assertArrayNear(self.evaluate(mre_obj2.normalizer), [1, 3], 1e-1)752  def test_unweighted(self):753    np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32)754    np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32)755    expected_error = np.mean(756        np.divide(np.absolute(np_y_pred - np_y_true), np_y_true))757    y_pred = constant_op.constant(np_y_pred, shape=(1, 4), dtype=dtypes.float32)758    y_true = constant_op.constant(np_y_true, shape=(1, 4))759    mre_obj = metrics.MeanRelativeError(normalizer=y_true)760    self.evaluate(variables.variables_initializer(mre_obj.variables))761    result = mre_obj(y_true, y_pred)762    self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)763  def test_weighted(self):764    np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32)765    np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32)766    sample_weight = np.asarray([0.2, 0.3, 0.5, 0], dtype=np.float32)767    rel_errors = np.divide(np.absolute(np_y_pred - np_y_true), np_y_true)768    expected_error = np.sum(rel_errors * sample_weight)769    y_pred = constant_op.constant(np_y_pred, dtype=dtypes.float32)770    y_true = constant_op.constant(np_y_true)771    mre_obj = metrics.MeanRelativeError(normalizer=y_true)772    self.evaluate(variables.variables_initializer(mre_obj.variables))773    result = mre_obj(774        y_true, y_pred, sample_weight=constant_op.constant(sample_weight))775    self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)776  def test_zero_normalizer(self):777    y_pred = constant_op.constant([2, 4], dtype=dtypes.float32)778    y_true = constant_op.constant([1, 3])779    mre_obj = metrics.MeanRelativeError(normalizer=array_ops.zeros_like(y_true))780    self.evaluate(variables.variables_initializer(mre_obj.variables))781    result = mre_obj(y_true, y_pred)782    self.assertEqual(self.evaluate(result), 0)783@test_util.run_all_in_graph_and_eager_modes784class MeanIoUTest(test.TestCase):785  def test_config(self):786    m_obj = metrics.MeanIoU(num_classes=2, name='mean_iou')787    self.assertEqual(m_obj.name, 'mean_iou')788    self.assertEqual(m_obj.num_classes, 2)789    m_obj2 = metrics.MeanIoU.from_config(m_obj.get_config())790    self.assertEqual(m_obj2.name, 'mean_iou')791    self.assertEqual(m_obj2.num_classes, 2)792  def test_unweighted(self):793    y_pred = constant_op.constant([0, 1, 0, 1], dtype=dtypes.float32)794    y_true = constant_op.constant([0, 0, 1, 1])795    m_obj = metrics.MeanIoU(num_classes=2)796    self.evaluate(variables.variables_initializer(m_obj.variables))797    result = m_obj(y_true, y_pred)798    # cm = [[1, 1],799    #       [1, 1]]800    # sum_row = [2, 2], sum_col = [2, 2], true_positives = [1, 1]801    # iou = true_positives / (sum_row + sum_col - true_positives))802    expected_result = (1 / (2 + 2 - 1) + 1 / (2 + 2 - 1)) / 2803    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)804  def test_weighted(self):805    y_pred = constant_op.constant([0, 1, 0, 1], dtype=dtypes.float32)806    y_true = constant_op.constant([0, 0, 1, 1])807    sample_weight = constant_op.constant([0.2, 0.3, 0.4, 0.1])808    m_obj = metrics.MeanIoU(num_classes=2)809    self.evaluate(variables.variables_initializer(m_obj.variables))810    result = m_obj(y_true, y_pred, sample_weight=sample_weight)811    # cm = [[0.2, 0.3],812    #       [0.4, 0.1]]813    # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]814    # iou = true_positives / (sum_row + sum_col - true_positives))815    expected_result = (0.2 / (0.6 + 0.5 - 0.2) + 0.1 / (0.4 + 0.5 - 0.1)) / 2816    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)817  def test_multi_dim_input(self):818    y_pred = constant_op.constant([[0, 1], [0, 1]], dtype=dtypes.float32)819    y_true = constant_op.constant([[0, 0], [1, 1]])820    sample_weight = constant_op.constant([[0.2, 0.3], [0.4, 0.1]])821    m_obj = metrics.MeanIoU(num_classes=2)822    self.evaluate(variables.variables_initializer(m_obj.variables))823    result = m_obj(y_true, y_pred, sample_weight=sample_weight)824    # cm = [[0.2, 0.3],825    #       [0.4, 0.1]]826    # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]827    # iou = true_positives / (sum_row + sum_col - true_positives))828    expected_result = (0.2 / (0.6 + 0.5 - 0.2) + 0.1 / (0.4 + 0.5 - 0.1)) / 2829    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)830  def test_zero_valid_entries(self):831    m_obj = metrics.MeanIoU(num_classes=2)832    self.evaluate(variables.variables_initializer(m_obj.variables))833    self.assertAllClose(self.evaluate(m_obj.result()), 0, atol=1e-3)834  def test_zero_and_non_zero_entries(self):835    y_pred = constant_op.constant([1], dtype=dtypes.float32)836    y_true = constant_op.constant([1])837    m_obj = metrics.MeanIoU(num_classes=2)838    self.evaluate(variables.variables_initializer(m_obj.variables))839    result = m_obj(y_true, y_pred)840    # cm = [[0, 0],841    #       [0, 1]]842    # sum_row = [0, 1], sum_col = [0, 1], true_positives = [0, 1]843    # iou = true_positives / (sum_row + sum_col - true_positives))844    expected_result = (0 + 1 / (1 + 1 - 1)) / 1845    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)846class MeanTensorTest(keras_parameterized.TestCase):847  @test_util.run_in_graph_and_eager_modes848  def test_config(self):849    m = metrics.MeanTensor(name='mean_by_element')850    # check config851    self.assertEqual(m.name, 'mean_by_element')852    self.assertTrue(m.stateful)853    self.assertEqual(m.dtype, dtypes.float32)854    self.assertEqual(len(m.variables), 0)855    with self.assertRaisesRegexp(ValueError, 'does not have any result yet'):856      m.result()857    self.evaluate(m([[3], [5], [3]]))858    self.assertAllEqual(m._shape, [3, 1])859    m2 = metrics.MeanTensor.from_config(m.get_config())860    self.assertEqual(m2.name, 'mean_by_element')861    self.assertTrue(m2.stateful)862    self.assertEqual(m2.dtype, dtypes.float32)863    self.assertEqual(len(m2.variables), 0)864  @test_util.run_in_graph_and_eager_modes865  def test_unweighted(self):866    m = metrics.MeanTensor(dtype=dtypes.float64)867    # check __call__()868    self.assertAllClose(self.evaluate(m([100, 40])), [100, 40])869    self.assertAllClose(self.evaluate(m.total), [100, 40])870    self.assertAllClose(self.evaluate(m.count), [1, 1])871    # check update_state() and result() + state accumulation + tensor input872    update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))873    self.evaluate(update_op)874    self.assertAllClose(self.evaluate(m.result()), [50.5, 22.5])875    self.assertAllClose(self.evaluate(m.total), [101, 45])876    self.assertAllClose(self.evaluate(m.count), [2, 2])877    # check reset_states()878    m.reset_states()879    self.assertAllClose(self.evaluate(m.total), [0, 0])880    self.assertAllClose(self.evaluate(m.count), [0, 0])881  @test_util.run_in_graph_and_eager_modes882  def test_weighted(self):883    m = metrics.MeanTensor(dtype=dtypes.float64)884    self.assertEqual(m.dtype, dtypes.float64)885    # check scalar weight886    result_t = m([100, 30], sample_weight=0.5)887    self.assertAllClose(self.evaluate(result_t), [100, 30])888    self.assertAllClose(self.evaluate(m.total), [50, 15])889    self.assertAllClose(self.evaluate(m.count), [0.5, 0.5])890    # check weights not scalar and weights rank matches values rank891    result_t = m([1, 5], sample_weight=[1, 0.2])892    result = self.evaluate(result_t)893    self.assertAllClose(result, [51 / 1.5, 16 / 0.7], 2)894    self.assertAllClose(self.evaluate(m.total), [51, 16])895    self.assertAllClose(self.evaluate(m.count), [1.5, 0.7])896    # check weights broadcast897    result_t = m([1, 2], sample_weight=0.5)898    self.assertAllClose(self.evaluate(result_t), [51.5 / 2, 17 / 1.2])899    self.assertAllClose(self.evaluate(m.total), [51.5, 17])900    self.assertAllClose(self.evaluate(m.count), [2, 1.2])901    # check weights squeeze902    result_t = m([1, 5], sample_weight=[[1], [0.2]])903    self.assertAllClose(self.evaluate(result_t), [52.5 / 3, 18 / 1.4])904    self.assertAllClose(self.evaluate(m.total), [52.5, 18])905    self.assertAllClose(self.evaluate(m.count), [3, 1.4])906    # check weights expand907    m = metrics.MeanTensor((2, 1), dtype=dtypes.float64)908    self.evaluate(variables.variables_initializer(m.variables))909    result_t = m([[1], [5]], sample_weight=[1, 0.2])910    self.assertAllClose(self.evaluate(result_t), [[1], [5]])911    self.assertAllClose(self.evaluate(m.total), [[1], [1]])912    self.assertAllClose(self.evaluate(m.count), [[1], [0.2]])913  @test_util.run_in_graph_and_eager_modes914  def test_invalid_value_shape(self):915    m = metrics.MeanTensor(dtype=dtypes.float64)916    m([1])917    with self.assertRaisesRegexp(918        ValueError, 'MeanTensor input values must always have the same shape'):919      m([1, 5])920  @test_util.run_in_graph_and_eager_modes921  def test_build_in_tf_function(self):922    """Ensure that variables are created correctly in a tf function."""923    m = metrics.MeanTensor(dtype=dtypes.float64)924    @eager_function.defun925    def call_metric(x):926      return m(x)927    self.assertAllClose(self.evaluate(call_metric([100, 40])), [100, 40])928    self.assertAllClose(self.evaluate(m.total), [100, 40])929    self.assertAllClose(self.evaluate(m.count), [1, 1])930    self.assertAllClose(self.evaluate(call_metric([20, 2])), [60, 21])931  def test_in_keras_model(self):932    with context.eager_mode():933      class ModelWithMetric(Model):934        def __init__(self):935          super(ModelWithMetric, self).__init__()936          self.dense1 = layers.Dense(937              3, activation='relu', kernel_initializer='ones')938          self.dense2 = layers.Dense(939              1, activation='sigmoid', kernel_initializer='ones')940          self.mean_tensor = metrics.MeanTensor()941        def call(self, x):942          x = self.dense1(x)943          x = self.dense2(x)944          self.mean_tensor(self.dense1.kernel)945          return x946      model = ModelWithMetric()947      model.compile(948          loss='mae',949          optimizer='rmsprop',950          run_eagerly=True)951      x = np.ones((100, 4))952      y = np.zeros((100, 1))953      model.evaluate(x, y, batch_size=50)954      self.assertAllClose(self.evaluate(model.mean_tensor.result()),955                          np.ones((4, 3)))956      self.assertAllClose(self.evaluate(model.mean_tensor.total),957                          np.full((4, 3), 2))958      self.assertAllClose(self.evaluate(model.mean_tensor.count),959                          np.full((4, 3), 2))960      model.evaluate(x, y, batch_size=25)961      self.assertAllClose(self.evaluate(model.mean_tensor.result()),962                          np.ones((4, 3)))963      self.assertAllClose(self.evaluate(model.mean_tensor.total),964                          np.full((4, 3), 4))965      self.assertAllClose(self.evaluate(model.mean_tensor.count),966                          np.full((4, 3), 4))967@test_util.run_all_in_graph_and_eager_modes968class BinaryCrossentropyTest(test.TestCase):969  def test_config(self):970    bce_obj = metrics.BinaryCrossentropy(971        name='bce', dtype=dtypes.int32, label_smoothing=0.2)972    self.assertEqual(bce_obj.name, 'bce')973    self.assertEqual(bce_obj._dtype, dtypes.int32)974    old_config = bce_obj.get_config()975    self.assertAllClose(self.evaluate(old_config['label_smoothing']), 0.2, 1e-3)976    # Check save and restore config977    bce_obj2 = metrics.BinaryCrossentropy.from_config(old_config)978    self.assertEqual(bce_obj2.name, 'bce')979    self.assertEqual(bce_obj2._dtype, dtypes.int32)980    new_config = bce_obj2.get_config()981    self.assertAllClose(self.evaluate(new_config['label_smoothing']), 0.2, 1e-3)982  def test_unweighted(self):983    bce_obj = metrics.BinaryCrossentropy()984    self.evaluate(variables.variables_initializer(bce_obj.variables))985    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])986    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])987    result = bce_obj(y_true, y_pred)988    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999989    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)990    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]991    # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))992    #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),993    #           -log(Y_MAX + EPSILON), -log(1)]994    #        = [(0 + 15.33) / 2, (0 + 0) / 2]995    # Reduced metric = 7.665 / 2996    self.assertAllClose(self.evaluate(result), 3.833, atol=1e-3)997  def test_unweighted_with_logits(self):998    bce_obj = metrics.BinaryCrossentropy(from_logits=True)999    self.evaluate(variables.variables_initializer(bce_obj.variables))1000    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])1001    y_pred = constant_op.constant([[100.0, -100.0, 100.0],1002                                   [100.0, 100.0, -100.0]])1003    result = bce_obj(y_true, y_pred)1004    # Metric = max(x, 0) - x * z + log(1 + exp(-abs(x)))1005    #              (where x = logits and z = y_true)1006    #        = [((100 - 100 * 1 + log(1 + exp(-100))) +1007    #            (0 + 100 * 0 + log(1 + exp(-100))) +1008    #            (100 - 100 * 1 + log(1 + exp(-100))),1009    #           ((100 - 100 * 0 + log(1 + exp(-100))) +1010    #            (100 - 100 * 1 + log(1 + exp(-100))) +1011    #            (0 + 100 * 1 + log(1 + exp(-100))))]1012    #        = [(0 + 0 + 0) / 3, 200 / 3]1013    # Reduced metric = (0 + 66.666) / 21014    self.assertAllClose(self.evaluate(result), 33.333, atol=1e-3)1015  def test_weighted(self):1016    bce_obj = metrics.BinaryCrossentropy()1017    self.evaluate(variables.variables_initializer(bce_obj.variables))1018    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])1019    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])1020    sample_weight = constant_op.constant([1.5, 2.])1021    result = bce_obj(y_true, y_pred, sample_weight=sample_weight)1022    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.99999991023    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)1024    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]1025    # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))1026    #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),1027    #           -log(Y_MAX + EPSILON), -log(1)]1028    #        = [(0 + 15.33) / 2, (0 + 0) / 2]1029    # Weighted metric = [7.665 * 1.5, 0]1030    # Reduced metric = 7.665 * 1.5 / (1.5 + 2)1031    self.assertAllClose(self.evaluate(result), 3.285, atol=1e-3)1032  def test_weighted_from_logits(self):1033    bce_obj = metrics.BinaryCrossentropy(from_logits=True)1034    self.evaluate(variables.variables_initializer(bce_obj.variables))1035    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])1036    y_pred = constant_op.constant([[100.0, -100.0, 100.0],1037                                   [100.0, 100.0, -100.0]])1038    sample_weight = constant_op.constant([2., 2.5])1039    result = bce_obj(y_true, y_pred, sample_weight=sample_weight)1040    # Metric = max(x, 0) - x * z + log(1 + exp(-abs(x)))1041    #              (where x = logits and z = y_true)1042    #        = [(0 + 0 + 0) / 3, 200 / 3]1043    # Weighted metric = [0, 66.666 * 2.5]1044    # Reduced metric = 66.666 * 2.5 / (2 + 2.5)1045    self.assertAllClose(self.evaluate(result), 37.037, atol=1e-3)1046  def test_label_smoothing(self):1047    logits = constant_op.constant(((100., -100., -100.)))1048    y_true = constant_op.constant(((1, 0, 1)))1049    label_smoothing = 0.11050    # Metric: max(x, 0) - x * z + log(1 + exp(-abs(x)))1051    #             (where x = logits and z = y_true)1052    # Label smoothing: z' = z * (1 - L) + 0.5L1053    # After label smoothing, label 1 becomes 1 - 0.5L1054    #                        label 0 becomes 0.5L1055    # Applying the above two fns to the given input:1056    # (100 - 100 * (1 - 0.5 L)  + 0 +1057    #  0   + 100 * (0.5 L)      + 0 +1058    #  0   + 100 * (1 - 0.5 L)  + 0) * (1/3)1059    #  = (100 + 50L) * 1/31060    bce_obj = metrics.BinaryCrossentropy(1061        from_logits=True, label_smoothing=label_smoothing)1062    self.evaluate(variables.variables_initializer(bce_obj.variables))1063    result = bce_obj(y_true, logits)1064    expected_value = (100.0 + 50.0 * label_smoothing) / 3.01065    self.assertAllClose(expected_value, self.evaluate(result), atol=1e-3)1066def _get_model(compile_metrics):1067  model_layers = [1068      layers.Dense(3, activation='relu', kernel_initializer='ones'),1069      layers.Dense(1, activation='sigmoid', kernel_initializer='ones')]1070  model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))1071  model.compile(1072      loss='mae',1073      metrics=compile_metrics,1074      optimizer='rmsprop',1075      run_eagerly=testing_utils.should_run_eagerly())1076  return model1077@keras_parameterized.run_with_all_model_types1078@keras_parameterized.run_all_keras_modes1079class ResetStatesTest(keras_parameterized.TestCase):1080  def test_reset_states_false_positives(self):1081    fp_obj = metrics.FalsePositives()1082    model = _get_model([fp_obj])1083    x = np.ones((100, 4))1084    y = np.zeros((100, 1))1085    model.evaluate(x, y)1086    self.assertEqual(self.evaluate(fp_obj.accumulator), 100.)1087    model.evaluate(x, y)1088    self.assertEqual(self.evaluate(fp_obj.accumulator), 100.)1089  def test_reset_states_false_negatives(self):1090    fn_obj = metrics.FalseNegatives()1091    model = _get_model([fn_obj])1092    x = np.zeros((100, 4))1093    y = np.ones((100, 1))1094    model.evaluate(x, y)1095    self.assertEqual(self.evaluate(fn_obj.accumulator), 100.)1096    model.evaluate(x, y)1097    self.assertEqual(self.evaluate(fn_obj.accumulator), 100.)1098  def test_reset_states_true_negatives(self):1099    tn_obj = metrics.TrueNegatives()1100    model = _get_model([tn_obj])1101    x = np.zeros((100, 4))1102    y = np.zeros((100, 1))1103    model.evaluate(x, y)1104    self.assertEqual(self.evaluate(tn_obj.accumulator), 100.)1105    model.evaluate(x, y)1106    self.assertEqual(self.evaluate(tn_obj.accumulator), 100.)1107  def test_reset_states_true_positives(self):1108    tp_obj = metrics.TruePositives()1109    model = _get_model([tp_obj])1110    x = np.ones((100, 4))1111    y = np.ones((100, 1))1112    model.evaluate(x, y)1113    self.assertEqual(self.evaluate(tp_obj.accumulator), 100.)1114    model.evaluate(x, y)1115    self.assertEqual(self.evaluate(tp_obj.accumulator), 100.)1116  def test_reset_states_precision(self):1117    p_obj = metrics.Precision()1118    model = _get_model([p_obj])1119    x = np.concatenate((np.ones((50, 4)), np.ones((50, 4))))1120    y = np.concatenate((np.ones((50, 1)), np.zeros((50, 1))))1121    model.evaluate(x, y)1122    self.assertEqual(self.evaluate(p_obj.true_positives), 50.)1123    self.assertEqual(self.evaluate(p_obj.false_positives), 50.)1124    model.evaluate(x, y)1125    self.assertEqual(self.evaluate(p_obj.true_positives), 50.)1126    self.assertEqual(self.evaluate(p_obj.false_positives), 50.)1127  def test_reset_states_recall(self):1128    r_obj = metrics.Recall()1129    model = _get_model([r_obj])1130    x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4))))1131    y = np.concatenate((np.ones((50, 1)), np.ones((50, 1))))1132    model.evaluate(x, y)1133    self.assertEqual(self.evaluate(r_obj.true_positives), 50.)1134    self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)1135    model.evaluate(x, y)1136    self.assertEqual(self.evaluate(r_obj.true_positives), 50.)1137    self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)1138  def test_reset_states_sensitivity_at_specificity(self):1139    s_obj = metrics.SensitivityAtSpecificity(0.5, num_thresholds=1)1140    model = _get_model([s_obj])1141    x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)),1142                        np.ones((25, 4))))1143    y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),1144                        np.zeros((25, 1))))1145    for _ in range(2):1146      model.evaluate(x, y)1147      self.assertEqual(self.evaluate(s_obj.true_positives), 25.)1148      self.assertEqual(self.evaluate(s_obj.false_positives), 25.)1149      self.assertEqual(self.evaluate(s_obj.false_negatives), 25.)1150      self.assertEqual(self.evaluate(s_obj.true_negatives), 25.)1151  def test_reset_states_specificity_at_sensitivity(self):1152    s_obj = metrics.SpecificityAtSensitivity(0.5, num_thresholds=1)1153    model = _get_model([s_obj])1154    x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)),1155                        np.ones((25, 4))))1156    y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),1157                        np.zeros((25, 1))))1158    for _ in range(2):1159      model.evaluate(x, y)1160      self.assertEqual(self.evaluate(s_obj.true_positives), 25.)1161      self.assertEqual(self.evaluate(s_obj.false_positives), 25.)1162      self.assertEqual(self.evaluate(s_obj.false_negatives), 25.)1163      self.assertEqual(self.evaluate(s_obj.true_negatives), 25.)1164  def test_reset_states_auc(self):1165    auc_obj = metrics.AUC(num_thresholds=3)1166    model = _get_model([auc_obj])1167    x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)),1168                        np.ones((25, 4))))1169    y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),1170                        np.zeros((25, 1))))1171    for _ in range(2):1172      model.evaluate(x, y)1173      self.assertEqual(self.evaluate(auc_obj.true_positives[1]), 25.)1174      self.assertEqual(self.evaluate(auc_obj.false_positives[1]), 25.)1175      self.assertEqual(self.evaluate(auc_obj.false_negatives[1]), 25.)1176      self.assertEqual(self.evaluate(auc_obj.true_negatives[1]), 25.)1177  def test_reset_states_mean_iou(self):1178    m_obj = metrics.MeanIoU(num_classes=2)1179    model = _get_model([m_obj])1180    x = np.asarray([[0, 0, 0, 0], [1, 1, 1, 1], [1, 0, 1, 0], [0, 1, 0, 1]],1181                   dtype=np.float32)1182    y = np.asarray([[0], [1], [1], [1]], dtype=np.float32)1183    model.evaluate(x, y)1184    self.assertArrayNear(self.evaluate(m_obj.total_cm)[0], [1, 0], 1e-1)1185    self.assertArrayNear(self.evaluate(m_obj.total_cm)[1], [3, 0], 1e-1)1186    model.evaluate(x, y)1187    self.assertArrayNear(self.evaluate(m_obj.total_cm)[0], [1, 0], 1e-1)1188    self.assertArrayNear(self.evaluate(m_obj.total_cm)[1], [3, 0], 1e-1)1189if __name__ == '__main__':...

...40    self.assertEqual(len(fp_obj2.variables), 1)41    self.assertEqual(fp_obj2.thresholds, [0.4, 0.9])42  def test_unweighted(self):43    fp_obj = metrics.FalsePositives()44    self.evaluate(variables.variables_initializer(fp_obj.variables))45    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),46                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))47    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),48                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))49    update_op = fp_obj.update_state(y_true, y_pred)50    self.evaluate(update_op)51    result = fp_obj.result()52    self.assertAllClose(7., result)53  def test_weighted(self):54    fp_obj = metrics.FalsePositives()55    self.evaluate(variables.variables_initializer(fp_obj.variables))56    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),57                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))58    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),59                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))60    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))61    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)62    self.assertAllClose(14., self.evaluate(result))63  def test_unweighted_with_thresholds(self):64    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])65    self.evaluate(variables.variables_initializer(fp_obj.variables))66    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),67                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))68    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),69                                   (1, 1, 1, 1)))70    update_op = fp_obj.update_state(y_true, y_pred)71    self.evaluate(update_op)72    result = fp_obj.result()73    self.assertAllClose([7., 4., 2.], result)74  def test_weighted_with_thresholds(self):75    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])76    self.evaluate(variables.variables_initializer(fp_obj.variables))77    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),78                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))79    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),80                                   (1, 1, 1, 1)))81    sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0),82                     (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0))83    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)84    self.assertAllClose([125., 42., 12.], self.evaluate(result))85  def test_threshold_limit(self):86    with self.assertRaisesRegexp(87        ValueError,88        r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'):89      metrics.FalsePositives(thresholds=[-1, 0.5, 2])90    with self.assertRaisesRegexp(91        ValueError,92        r'Threshold values must be in \[0, 1\]. Invalid values: \[None\]'):93      metrics.FalsePositives(thresholds=[None])94@test_util.run_all_in_graph_and_eager_modes95class FalseNegativesTest(test.TestCase):96  def test_config(self):97    fn_obj = metrics.FalseNegatives(name='my_fn', thresholds=[0.4, 0.9])98    self.assertEqual(fn_obj.name, 'my_fn')99    self.assertEqual(len(fn_obj.variables), 1)100    self.assertEqual(fn_obj.thresholds, [0.4, 0.9])101    # Check save and restore config102    fn_obj2 = metrics.FalseNegatives.from_config(fn_obj.get_config())103    self.assertEqual(fn_obj2.name, 'my_fn')104    self.assertEqual(len(fn_obj2.variables), 1)105    self.assertEqual(fn_obj2.thresholds, [0.4, 0.9])106  def test_unweighted(self):107    fn_obj = metrics.FalseNegatives()108    self.evaluate(variables.variables_initializer(fn_obj.variables))109    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),110                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))111    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),112                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))113    update_op = fn_obj.update_state(y_true, y_pred)114    self.evaluate(update_op)115    result = fn_obj.result()116    self.assertAllClose(3., result)117  def test_weighted(self):118    fn_obj = metrics.FalseNegatives()119    self.evaluate(variables.variables_initializer(fn_obj.variables))120    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),121                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))122    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),123                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))124    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))125    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)126    self.assertAllClose(5., self.evaluate(result))127  def test_unweighted_with_thresholds(self):128    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])129    self.evaluate(variables.variables_initializer(fn_obj.variables))130    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),131                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))132    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),133                                   (1, 1, 1, 1)))134    update_op = fn_obj.update_state(y_true, y_pred)135    self.evaluate(update_op)136    result = fn_obj.result()137    self.assertAllClose([1., 4., 6.], result)138  def test_weighted_with_thresholds(self):139    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])140    self.evaluate(variables.variables_initializer(fn_obj.variables))141    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),142                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))143    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),144                                   (1, 1, 1, 1)))145    sample_weight = ((3.0,), (5.0,), (7.0,), (4.0,))146    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)147    self.assertAllClose([4., 16., 23.], self.evaluate(result))148@test_util.run_all_in_graph_and_eager_modes149class TrueNegativesTest(test.TestCase):150  def test_config(self):151    tn_obj = metrics.TrueNegatives(name='my_tn', thresholds=[0.4, 0.9])152    self.assertEqual(tn_obj.name, 'my_tn')153    self.assertEqual(len(tn_obj.variables), 1)154    self.assertEqual(tn_obj.thresholds, [0.4, 0.9])155    # Check save and restore config156    tn_obj2 = metrics.TrueNegatives.from_config(tn_obj.get_config())157    self.assertEqual(tn_obj2.name, 'my_tn')158    self.assertEqual(len(tn_obj2.variables), 1)159    self.assertEqual(tn_obj2.thresholds, [0.4, 0.9])160  def test_unweighted(self):161    tn_obj = metrics.TrueNegatives()162    self.evaluate(variables.variables_initializer(tn_obj.variables))163    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),164                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))165    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),166                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))167    update_op = tn_obj.update_state(y_true, y_pred)168    self.evaluate(update_op)169    result = tn_obj.result()170    self.assertAllClose(3., result)171  def test_weighted(self):172    tn_obj = metrics.TrueNegatives()173    self.evaluate(variables.variables_initializer(tn_obj.variables))174    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),175                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))176    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),177                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))178    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))179    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)180    self.assertAllClose(4., self.evaluate(result))181  def test_unweighted_with_thresholds(self):182    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])183    self.evaluate(variables.variables_initializer(tn_obj.variables))184    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),185                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))186    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),187                                   (1, 1, 1, 1)))188    update_op = tn_obj.update_state(y_true, y_pred)189    self.evaluate(update_op)190    result = tn_obj.result()191    self.assertAllClose([2., 5., 7.], result)192  def test_weighted_with_thresholds(self):193    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])194    self.evaluate(variables.variables_initializer(tn_obj.variables))195    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),196                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))197    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),198                                   (1, 1, 1, 1)))199    sample_weight = ((0.0, 2.0, 3.0, 5.0),)200    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)201    self.assertAllClose([5., 15., 23.], self.evaluate(result))202@test_util.run_all_in_graph_and_eager_modes203class TruePositivesTest(test.TestCase):204  def test_config(self):205    tp_obj = metrics.TruePositives(name='my_tp', thresholds=[0.4, 0.9])206    self.assertEqual(tp_obj.name, 'my_tp')207    self.assertEqual(len(tp_obj.variables), 1)208    self.assertEqual(tp_obj.thresholds, [0.4, 0.9])209    # Check save and restore config210    tp_obj2 = metrics.TruePositives.from_config(tp_obj.get_config())211    self.assertEqual(tp_obj2.name, 'my_tp')212    self.assertEqual(len(tp_obj2.variables), 1)213    self.assertEqual(tp_obj2.thresholds, [0.4, 0.9])214  def test_unweighted(self):215    tp_obj = metrics.TruePositives()216    self.evaluate(variables.variables_initializer(tp_obj.variables))217    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),218                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))219    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),220                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))221    update_op = tp_obj.update_state(y_true, y_pred)222    self.evaluate(update_op)223    result = tp_obj.result()224    self.assertAllClose(7., result)225  def test_weighted(self):226    tp_obj = metrics.TruePositives()227    self.evaluate(variables.variables_initializer(tp_obj.variables))228    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),229                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))230    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),231                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))232    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))233    result = tp_obj(y_true, y_pred, sample_weight=sample_weight)234    self.assertAllClose(12., self.evaluate(result))235  def test_unweighted_with_thresholds(self):236    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])237    self.evaluate(variables.variables_initializer(tp_obj.variables))238    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),239                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))240    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),241                                   (1, 1, 1, 1)))242    update_op = tp_obj.update_state(y_true, y_pred)243    self.evaluate(update_op)244    result = tp_obj.result()245    self.assertAllClose([6., 3., 1.], result)246  def test_weighted_with_thresholds(self):247    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])248    self.evaluate(variables.variables_initializer(tp_obj.variables))249    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),250                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))251    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),252                                   (1, 1, 1, 1)))253    result = tp_obj(y_true, y_pred, sample_weight=37.)254    self.assertAllClose([222., 111., 37.], self.evaluate(result))255@test_util.run_all_in_graph_and_eager_modes256class PrecisionTest(test.TestCase):257  def test_config(self):258    p_obj = metrics.Precision(259        name='my_precision', thresholds=[0.4, 0.9], top_k=15, class_id=12)260    self.assertEqual(p_obj.name, 'my_precision')261    self.assertEqual(len(p_obj.variables), 2)262    self.assertEqual([v.name for v in p_obj.variables],263                     ['true_positives:0', 'false_positives:0'])264    self.assertEqual(p_obj.thresholds, [0.4, 0.9])265    self.assertEqual(p_obj.top_k, 15)266    self.assertEqual(p_obj.class_id, 12)267    # Check save and restore config268    p_obj2 = metrics.Precision.from_config(p_obj.get_config())269    self.assertEqual(p_obj2.name, 'my_precision')270    self.assertEqual(len(p_obj2.variables), 2)271    self.assertEqual(p_obj2.thresholds, [0.4, 0.9])272    self.assertEqual(p_obj2.top_k, 15)273    self.assertEqual(p_obj2.class_id, 12)274  def test_value_is_idempotent(self):275    p_obj = metrics.Precision(thresholds=[0.3, 0.72])276    y_pred = random_ops.random_uniform(shape=(10, 3))277    y_true = random_ops.random_uniform(shape=(10, 3))278    update_op = p_obj.update_state(y_true, y_pred)279    self.evaluate(variables.variables_initializer(p_obj.variables))280    # Run several updates.281    for _ in range(10):282      self.evaluate(update_op)283    # Then verify idempotency.284    initial_precision = self.evaluate(p_obj.result())285    for _ in range(10):286      self.assertArrayNear(initial_precision, self.evaluate(p_obj.result()),287                           1e-3)288  def test_unweighted(self):289    p_obj = metrics.Precision()290    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))291    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))292    self.evaluate(variables.variables_initializer(p_obj.variables))293    result = p_obj(y_true, y_pred)294    self.assertAlmostEqual(0.5, self.evaluate(result))295  def test_unweighted_all_incorrect(self):296    p_obj = metrics.Precision(thresholds=[0.5])297    inputs = np.random.randint(0, 2, size=(100, 1))298    y_pred = constant_op.constant(inputs)299    y_true = constant_op.constant(1 - inputs)300    self.evaluate(variables.variables_initializer(p_obj.variables))301    result = p_obj(y_true, y_pred)302    self.assertAlmostEqual(0, self.evaluate(result))303  def test_weighted(self):304    p_obj = metrics.Precision()305    y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])306    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])307    self.evaluate(variables.variables_initializer(p_obj.variables))308    result = p_obj(309        y_true,310        y_pred,311        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))312    weighted_tp = 3.0 + 4.0313    weighted_positives = (1.0 + 3.0) + (4.0 + 2.0)314    expected_precision = weighted_tp / weighted_positives315    self.assertAlmostEqual(expected_precision, self.evaluate(result))316  def test_div_by_zero(self):317    p_obj = metrics.Precision()318    y_pred = constant_op.constant([0, 0, 0, 0])319    y_true = constant_op.constant([0, 0, 0, 0])320    self.evaluate(variables.variables_initializer(p_obj.variables))321    result = p_obj(y_true, y_pred)322    self.assertEqual(0, self.evaluate(result))323  def test_unweighted_with_threshold(self):324    p_obj = metrics.Precision(thresholds=[0.5, 0.7])325    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))326    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))327    self.evaluate(variables.variables_initializer(p_obj.variables))328    result = p_obj(y_true, y_pred)329    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)330  def test_weighted_with_threshold(self):331    p_obj = metrics.Precision(thresholds=[0.5, 1.])332    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))333    y_pred = constant_op.constant([[1, 0], [0.6, 0]],334                                  shape=(2, 2),335                                  dtype=dtypes.float32)336    weights = constant_op.constant([[4, 0], [3, 1]],337                                   shape=(2, 2),338                                   dtype=dtypes.float32)339    self.evaluate(variables.variables_initializer(p_obj.variables))340    result = p_obj(y_true, y_pred, sample_weight=weights)341    weighted_tp = 0 + 3.342    weighted_positives = (0 + 3.) + (4. + 0.)343    expected_precision = weighted_tp / weighted_positives344    self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3)345  def test_multiple_updates(self):346    p_obj = metrics.Precision(thresholds=[0.5, 1.])347    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))348    y_pred = constant_op.constant([[1, 0], [0.6, 0]],349                                  shape=(2, 2),350                                  dtype=dtypes.float32)351    weights = constant_op.constant([[4, 0], [3, 1]],352                                   shape=(2, 2),353                                   dtype=dtypes.float32)354    self.evaluate(variables.variables_initializer(p_obj.variables))355    update_op = p_obj.update_state(y_true, y_pred, sample_weight=weights)356    for _ in range(2):357      self.evaluate(update_op)358    weighted_tp = (0 + 3.) + (0 + 3.)359    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))360    expected_precision = weighted_tp / weighted_positives361    self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),362                         1e-3)363  def test_unweighted_top_k(self):364    p_obj = metrics.Precision(top_k=3)365    y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5))366    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))367    self.evaluate(variables.variables_initializer(p_obj.variables))368    result = p_obj(y_true, y_pred)369    self.assertAlmostEqual(1. / 3, self.evaluate(result))370  def test_weighted_top_k(self):371    p_obj = metrics.Precision(top_k=3)372    y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5))373    y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))374    self.evaluate(variables.variables_initializer(p_obj.variables))375    self.evaluate(376        p_obj(377            y_true1,378            y_pred1,379            sample_weight=constant_op.constant([[1, 4, 2, 3, 5]])))380    y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5))381    y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5))382    result = p_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3))383    tp = (2 + 5) + (3 + 3)384    predicted_positives = (1 + 2 + 5) + (3 + 3 + 3)385    expected_precision = tp / predicted_positives386    self.assertAlmostEqual(expected_precision, self.evaluate(result))387  def test_unweighted_class_id(self):388    p_obj = metrics.Precision(class_id=2)389    self.evaluate(variables.variables_initializer(p_obj.variables))390    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))391    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))392    result = p_obj(y_true, y_pred)393    self.assertAlmostEqual(1, self.evaluate(result))394    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))395    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))396    y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))397    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))398    result = p_obj(y_true, y_pred)399    self.assertAlmostEqual(1, self.evaluate(result))400    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))401    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))402    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))403    y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))404    result = p_obj(y_true, y_pred)405    self.assertAlmostEqual(0.5, self.evaluate(result))406    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))407    self.assertAlmostEqual(1, self.evaluate(p_obj.false_positives))408  def test_unweighted_top_k_and_class_id(self):409    p_obj = metrics.Precision(class_id=2, top_k=2)410    self.evaluate(variables.variables_initializer(p_obj.variables))411    y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5))412    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))413    result = p_obj(y_true, y_pred)414    self.assertAlmostEqual(1, self.evaluate(result))415    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))416    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))417    y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))418    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))419    result = p_obj(y_true, y_pred)420    self.assertAlmostEqual(1, self.evaluate(result))421    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))422    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))423  def test_unweighted_top_k_and_threshold(self):424    p_obj = metrics.Precision(thresholds=.7, top_k=2)425    self.evaluate(variables.variables_initializer(p_obj.variables))426    y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))427    y_true = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))428    result = p_obj(y_true, y_pred)429    self.assertAlmostEqual(1, self.evaluate(result))430    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))431    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))432@test_util.run_all_in_graph_and_eager_modes433class RecallTest(test.TestCase):434  def test_config(self):435    r_obj = metrics.Recall(436        name='my_recall', thresholds=[0.4, 0.9], top_k=15, class_id=12)437    self.assertEqual(r_obj.name, 'my_recall')438    self.assertEqual(len(r_obj.variables), 2)439    self.assertEqual([v.name for v in r_obj.variables],440                     ['true_positives:0', 'false_negatives:0'])441    self.assertEqual(r_obj.thresholds, [0.4, 0.9])442    self.assertEqual(r_obj.top_k, 15)443    self.assertEqual(r_obj.class_id, 12)444    # Check save and restore config445    r_obj2 = metrics.Recall.from_config(r_obj.get_config())446    self.assertEqual(r_obj2.name, 'my_recall')447    self.assertEqual(len(r_obj2.variables), 2)448    self.assertEqual(r_obj2.thresholds, [0.4, 0.9])449    self.assertEqual(r_obj2.top_k, 15)450    self.assertEqual(r_obj2.class_id, 12)451  def test_value_is_idempotent(self):452    r_obj = metrics.Recall(thresholds=[0.3, 0.72])453    y_pred = random_ops.random_uniform(shape=(10, 3))454    y_true = random_ops.random_uniform(shape=(10, 3))455    update_op = r_obj.update_state(y_true, y_pred)456    self.evaluate(variables.variables_initializer(r_obj.variables))457    # Run several updates.458    for _ in range(10):459      self.evaluate(update_op)460    # Then verify idempotency.461    initial_recall = self.evaluate(r_obj.result())462    for _ in range(10):463      self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)464  def test_unweighted(self):465    r_obj = metrics.Recall()466    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))467    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))468    self.evaluate(variables.variables_initializer(r_obj.variables))469    result = r_obj(y_true, y_pred)470    self.assertAlmostEqual(0.5, self.evaluate(result))471  def test_unweighted_all_incorrect(self):472    r_obj = metrics.Recall(thresholds=[0.5])473    inputs = np.random.randint(0, 2, size=(100, 1))474    y_pred = constant_op.constant(inputs)475    y_true = constant_op.constant(1 - inputs)476    self.evaluate(variables.variables_initializer(r_obj.variables))477    result = r_obj(y_true, y_pred)478    self.assertAlmostEqual(0, self.evaluate(result))479  def test_weighted(self):480    r_obj = metrics.Recall()481    y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])482    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])483    self.evaluate(variables.variables_initializer(r_obj.variables))484    result = r_obj(485        y_true,486        y_pred,487        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))488    weighted_tp = 3.0 + 1.0489    weighted_t = (2.0 + 3.0) + (4.0 + 1.0)490    expected_recall = weighted_tp / weighted_t491    self.assertAlmostEqual(expected_recall, self.evaluate(result))492  def test_div_by_zero(self):493    r_obj = metrics.Recall()494    y_pred = constant_op.constant([0, 0, 0, 0])495    y_true = constant_op.constant([0, 0, 0, 0])496    self.evaluate(variables.variables_initializer(r_obj.variables))497    result = r_obj(y_true, y_pred)498    self.assertEqual(0, self.evaluate(result))499  def test_unweighted_with_threshold(self):500    r_obj = metrics.Recall(thresholds=[0.5, 0.7])501    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))502    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))503    self.evaluate(variables.variables_initializer(r_obj.variables))504    result = r_obj(y_true, y_pred)505    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)506  def test_weighted_with_threshold(self):507    r_obj = metrics.Recall(thresholds=[0.5, 1.])508    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))509    y_pred = constant_op.constant([[1, 0], [0.6, 0]],510                                  shape=(2, 2),511                                  dtype=dtypes.float32)512    weights = constant_op.constant([[1, 4], [3, 2]],513                                   shape=(2, 2),514                                   dtype=dtypes.float32)515    self.evaluate(variables.variables_initializer(r_obj.variables))516    result = r_obj(y_true, y_pred, sample_weight=weights)517    weighted_tp = 0 + 3.518    weighted_positives = (0 + 3.) + (4. + 0.)519    expected_recall = weighted_tp / weighted_positives520    self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)521  def test_multiple_updates(self):522    r_obj = metrics.Recall(thresholds=[0.5, 1.])523    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))524    y_pred = constant_op.constant([[1, 0], [0.6, 0]],525                                  shape=(2, 2),526                                  dtype=dtypes.float32)527    weights = constant_op.constant([[1, 4], [3, 2]],528                                   shape=(2, 2),529                                   dtype=dtypes.float32)530    self.evaluate(variables.variables_initializer(r_obj.variables))531    update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)532    for _ in range(2):533      self.evaluate(update_op)534    weighted_tp = (0 + 3.) + (0 + 3.)535    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))536    expected_recall = weighted_tp / weighted_positives537    self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),538                         1e-3)539  def test_unweighted_top_k(self):540    r_obj = metrics.Recall(top_k=3)541    y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5))542    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))543    self.evaluate(variables.variables_initializer(r_obj.variables))544    result = r_obj(y_true, y_pred)545    self.assertAlmostEqual(0.5, self.evaluate(result))546  def test_weighted_top_k(self):547    r_obj = metrics.Recall(top_k=3)548    y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5))549    y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))550    self.evaluate(variables.variables_initializer(r_obj.variables))551    self.evaluate(552        r_obj(553            y_true1,554            y_pred1,555            sample_weight=constant_op.constant([[1, 4, 2, 3, 5]])))556    y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5))557    y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5))558    result = r_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3))559    tp = (2 + 5) + (3 + 3)560    positives = (4 + 2 + 5) + (3 + 3 + 3 + 3)561    expected_recall = tp / positives562    self.assertAlmostEqual(expected_recall, self.evaluate(result))563  def test_unweighted_class_id(self):564    r_obj = metrics.Recall(class_id=2)565    self.evaluate(variables.variables_initializer(r_obj.variables))566    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))567    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))568    result = r_obj(y_true, y_pred)569    self.assertAlmostEqual(1, self.evaluate(result))570    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))571    self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives))572    y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))573    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))574    result = r_obj(y_true, y_pred)575    self.assertAlmostEqual(0.5, self.evaluate(result))576    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))577    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))578    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))579    y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))580    result = r_obj(y_true, y_pred)581    self.assertAlmostEqual(0.5, self.evaluate(result))582    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))583    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))584  def test_unweighted_top_k_and_class_id(self):585    r_obj = metrics.Recall(class_id=2, top_k=2)586    self.evaluate(variables.variables_initializer(r_obj.variables))587    y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5))588    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))589    result = r_obj(y_true, y_pred)590    self.assertAlmostEqual(1, self.evaluate(result))591    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))592    self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives))593    y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))594    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))595    result = r_obj(y_true, y_pred)596    self.assertAlmostEqual(0.5, self.evaluate(result))597    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))598    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))599  def test_unweighted_top_k_and_threshold(self):600    r_obj = metrics.Recall(thresholds=.7, top_k=2)601    self.evaluate(variables.variables_initializer(r_obj.variables))602    y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))603    y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5))604    result = r_obj(y_true, y_pred)605    self.assertAlmostEqual(0.25, self.evaluate(result))606    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))607    self.assertAlmostEqual(3, self.evaluate(r_obj.false_negatives))608@test_util.run_all_in_graph_and_eager_modes609class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):610  def test_config(self):611    s_obj = metrics.SensitivityAtSpecificity(612        0.4, num_thresholds=100, name='sensitivity_at_specificity_1')613    self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1')614    self.assertLen(s_obj.variables, 4)615    self.assertEqual(s_obj.specificity, 0.4)616    self.assertEqual(s_obj.num_thresholds, 100)617    # Check save and restore config618    s_obj2 = metrics.SensitivityAtSpecificity.from_config(s_obj.get_config())619    self.assertEqual(s_obj2.name, 'sensitivity_at_specificity_1')620    self.assertLen(s_obj2.variables, 4)621    self.assertEqual(s_obj2.specificity, 0.4)622    self.assertEqual(s_obj2.num_thresholds, 100)623  def test_value_is_idempotent(self):624    s_obj = metrics.SensitivityAtSpecificity(0.7)625    y_pred = random_ops.random_uniform((10, 3),626                                       maxval=1,627                                       dtype=dtypes.float32,628                                       seed=1)629    y_true = random_ops.random_uniform((10, 3),630                                       maxval=2,631                                       dtype=dtypes.int64,632                                       seed=1)633    update_op = s_obj.update_state(y_true, y_pred)634    self.evaluate(variables.variables_initializer(s_obj.variables))635    # Run several updates.636    for _ in range(10):637      self.evaluate(update_op)638    # Then verify idempotency.639    initial_sensitivity = self.evaluate(s_obj.result())640    for _ in range(10):641      self.assertAlmostEqual(initial_sensitivity, self.evaluate(s_obj.result()),642                             1e-3)643  def test_unweighted_all_correct(self):644    s_obj = metrics.SensitivityAtSpecificity(0.7)645    inputs = np.random.randint(0, 2, size=(100, 1))646    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)647    y_true = constant_op.constant(inputs)648    self.evaluate(variables.variables_initializer(s_obj.variables))649    result = s_obj(y_true, y_pred)650    self.assertAlmostEqual(1, self.evaluate(result))651  def test_unweighted_high_specificity(self):652    s_obj = metrics.SensitivityAtSpecificity(0.8)653    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]654    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]655    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)656    y_true = constant_op.constant(label_values)657    self.evaluate(variables.variables_initializer(s_obj.variables))658    result = s_obj(y_true, y_pred)659    self.assertAlmostEqual(0.8, self.evaluate(result))660  def test_unweighted_low_specificity(self):661    s_obj = metrics.SensitivityAtSpecificity(0.4)662    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]663    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]664    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)665    y_true = constant_op.constant(label_values)666    self.evaluate(variables.variables_initializer(s_obj.variables))667    result = s_obj(y_true, y_pred)668    self.assertAlmostEqual(0.6, self.evaluate(result))669  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])670  def test_weighted(self, label_dtype):671    s_obj = metrics.SensitivityAtSpecificity(0.4)672    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]673    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]674    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]675    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)676    y_true = math_ops.cast(label_values, dtype=label_dtype)677    weights = constant_op.constant(weight_values)678    self.evaluate(variables.variables_initializer(s_obj.variables))679    result = s_obj(y_true, y_pred, sample_weight=weights)680    self.assertAlmostEqual(0.675, self.evaluate(result))681  def test_invalid_specificity(self):682    with self.assertRaisesRegexp(683        ValueError, r'`specificity` must be in the range \[0, 1\].'):684      metrics.SensitivityAtSpecificity(-1)685  def test_invalid_num_thresholds(self):686    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):687      metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1)688@test_util.run_all_in_graph_and_eager_modes689class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):690  def test_config(self):691    s_obj = metrics.SpecificityAtSensitivity(692        0.4, num_thresholds=100, name='specificity_at_sensitivity_1')693    self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1')694    self.assertLen(s_obj.variables, 4)695    self.assertEqual(s_obj.sensitivity, 0.4)696    self.assertEqual(s_obj.num_thresholds, 100)697    # Check save and restore config698    s_obj2 = metrics.SpecificityAtSensitivity.from_config(s_obj.get_config())699    self.assertEqual(s_obj2.name, 'specificity_at_sensitivity_1')700    self.assertLen(s_obj2.variables, 4)701    self.assertEqual(s_obj2.sensitivity, 0.4)702    self.assertEqual(s_obj2.num_thresholds, 100)703  def test_value_is_idempotent(self):704    s_obj = metrics.SpecificityAtSensitivity(0.7)705    y_pred = random_ops.random_uniform((10, 3),706                                       maxval=1,707                                       dtype=dtypes.float32,708                                       seed=1)709    y_true = random_ops.random_uniform((10, 3),710                                       maxval=2,711                                       dtype=dtypes.int64,712                                       seed=1)713    update_op = s_obj.update_state(y_true, y_pred)714    self.evaluate(variables.variables_initializer(s_obj.variables))715    # Run several updates.716    for _ in range(10):717      self.evaluate(update_op)718    # Then verify idempotency.719    initial_specificity = self.evaluate(s_obj.result())720    for _ in range(10):721      self.assertAlmostEqual(initial_specificity, self.evaluate(s_obj.result()),722                             1e-3)723  def test_unweighted_all_correct(self):724    s_obj = metrics.SpecificityAtSensitivity(0.7)725    inputs = np.random.randint(0, 2, size=(100, 1))726    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)727    y_true = constant_op.constant(inputs)728    self.evaluate(variables.variables_initializer(s_obj.variables))729    result = s_obj(y_true, y_pred)730    self.assertAlmostEqual(1, self.evaluate(result))731  def test_unweighted_high_sensitivity(self):732    s_obj = metrics.SpecificityAtSensitivity(0.8)733    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]734    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]735    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)736    y_true = constant_op.constant(label_values)737    self.evaluate(variables.variables_initializer(s_obj.variables))738    result = s_obj(y_true, y_pred)739    self.assertAlmostEqual(0.4, self.evaluate(result))740  def test_unweighted_low_sensitivity(self):741    s_obj = metrics.SpecificityAtSensitivity(0.4)742    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]743    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]744    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)745    y_true = constant_op.constant(label_values)746    self.evaluate(variables.variables_initializer(s_obj.variables))747    result = s_obj(y_true, y_pred)748    self.assertAlmostEqual(0.6, self.evaluate(result))749  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])750  def test_weighted(self, label_dtype):751    s_obj = metrics.SpecificityAtSensitivity(0.4)752    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]753    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]754    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]755    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)756    y_true = math_ops.cast(label_values, dtype=label_dtype)757    weights = constant_op.constant(weight_values)758    self.evaluate(variables.variables_initializer(s_obj.variables))759    result = s_obj(y_true, y_pred, sample_weight=weights)760    self.assertAlmostEqual(0.4, self.evaluate(result))761  def test_invalid_sensitivity(self):762    with self.assertRaisesRegexp(763        ValueError, r'`sensitivity` must be in the range \[0, 1\].'):764      metrics.SpecificityAtSensitivity(-1)765  def test_invalid_num_thresholds(self):766    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):767      metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)768@test_util.run_all_in_graph_and_eager_modes769class AUCTest(test.TestCase):770  def setup(self):771    self.num_thresholds = 3772    self.y_pred = constant_op.constant([0, 0.5, 0.3, 0.9], dtype=dtypes.float32)773    self.y_true = constant_op.constant([0, 0, 1, 1])774    self.sample_weight = [1, 2, 3, 4]775    # threshold values are [0 - 1e-7, 0.5, 1 + 1e-7]776    # y_pred when threshold = 0 - 1e-7  : [1, 1, 1, 1]777    # y_pred when threshold = 0.5       : [0, 0, 0, 1]778    # y_pred when threshold = 1 + 1e-7  : [0, 0, 0, 0]779    # without sample_weight:780    # tp = np.sum([[0, 0, 1, 1], [0, 0, 0, 1], [0, 0, 0, 0]], axis=1)781    # fp = np.sum([[1, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], axis=1)782    # fn = np.sum([[0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 1, 1]], axis=1)783    # tn = np.sum([[0, 0, 0, 0], [1, 1, 0, 0], [1, 1, 0, 0]], axis=1)784    # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]785    # with sample_weight:786    # tp = np.sum([[0, 0, 3, 4], [0, 0, 0, 4], [0, 0, 0, 0]], axis=1)787    # fp = np.sum([[1, 2, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], axis=1)788    # fn = np.sum([[0, 0, 0, 0], [0, 0, 3, 0], [0, 0, 3, 4]], axis=1)789    # tn = np.sum([[0, 0, 0, 0], [1, 2, 0, 0], [1, 2, 0, 0]], axis=1)790    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]791  def test_config(self):792    auc_obj = metrics.AUC(793        num_thresholds=100,794        curve=metrics_utils.AUCCurve.PR,795        summation_method=metrics_utils.AUCSummationMethod.MAJORING,796        name='auc_1')797    self.assertEqual(auc_obj.name, 'auc_1')798    self.assertEqual(len(auc_obj.variables), 4)799    self.assertEqual(auc_obj.num_thresholds, 100)800    self.assertEqual(auc_obj.curve, metrics_utils.AUCCurve.PR)801    self.assertEqual(auc_obj.summation_method,802                     metrics_utils.AUCSummationMethod.MAJORING)803    # Check save and restore config804    auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())805    self.assertEqual(auc_obj2.name, 'auc_1')806    self.assertEqual(len(auc_obj2.variables), 4)807    self.assertEqual(auc_obj2.num_thresholds, 100)808    self.assertEqual(auc_obj2.curve, metrics_utils.AUCCurve.PR)809    self.assertEqual(auc_obj2.summation_method,810                     metrics_utils.AUCSummationMethod.MAJORING)811  def test_value_is_idempotent(self):812    self.setup()813    auc_obj = metrics.AUC(num_thresholds=3)814    self.evaluate(variables.variables_initializer(auc_obj.variables))815    # Run several updates.816    update_op = auc_obj.update_state(self.y_true, self.y_pred)817    for _ in range(10):818      self.evaluate(update_op)819    # Then verify idempotency.820    initial_auc = self.evaluate(auc_obj.result())821    for _ in range(10):822      self.assertAllClose(initial_auc, self.evaluate(auc_obj.result()), 1e-3)823  def test_unweighted_all_correct(self):824    self.setup()825    auc_obj = metrics.AUC()826    self.evaluate(variables.variables_initializer(auc_obj.variables))827    result = auc_obj(self.y_true, self.y_true)828    self.assertEqual(self.evaluate(result), 1)829  def test_unweighted(self):830    self.setup()831    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)832    self.evaluate(variables.variables_initializer(auc_obj.variables))833    result = auc_obj(self.y_true, self.y_pred)834    # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]835    # recall = [2/2, 1/(1+1), 0] = [1, 0.5, 0]836    # fp_rate = [2/2, 0, 0] = [1, 0, 0]837    # heights = [(1 + 0.5)/2, (0.5 + 0)/2] = [0.75, 0.25]838    # widths = [(1 - 0), (0 - 0)] = [1, 0]839    expected_result = (0.75 * 1 + 0.25 * 0)840    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)841  def test_weighted_roc_interpolation(self):842    self.setup()843    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)844    self.evaluate(variables.variables_initializer(auc_obj.variables))845    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)846    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]847    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]848    # fp_rate = [3/3, 0, 0] = [1, 0, 0]849    # heights = [(1 + 0.571)/2, (0.571 + 0)/2] = [0.7855, 0.2855]850    # widths = [(1 - 0), (0 - 0)] = [1, 0]851    expected_result = (0.7855 * 1 + 0.2855 * 0)852    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)853  def test_weighted_roc_majoring(self):854    self.setup()855    auc_obj = metrics.AUC(856        num_thresholds=self.num_thresholds,857        summation_method=metrics_utils.AUCSummationMethod.MAJORING)858    self.evaluate(variables.variables_initializer(auc_obj.variables))859    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)860    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]861    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]862    # fp_rate = [3/3, 0, 0] = [1, 0, 0]863    # heights = [max(1, 0.571), max(0.571, 0)] = [1, 0.571]864    # widths = [(1 - 0), (0 - 0)] = [1, 0]865    expected_result = (1 * 1 + 0.571 * 0)866    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)867  def test_weighted_roc_minoring(self):868    self.setup()869    auc_obj = metrics.AUC(870        num_thresholds=self.num_thresholds,871        summation_method=metrics_utils.AUCSummationMethod.MINORING)872    self.evaluate(variables.variables_initializer(auc_obj.variables))873    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)874    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]875    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]876    # fp_rate = [3/3, 0, 0] = [1, 0, 0]877    # heights = [min(1, 0.571), min(0.571, 0)] = [0.571, 0]878    # widths = [(1 - 0), (0 - 0)] = [1, 0]879    expected_result = (0.571 * 1 + 0 * 0)880    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)881  def test_weighted_pr_majoring(self):882    self.setup()883    auc_obj = metrics.AUC(884        num_thresholds=self.num_thresholds,885        curve=metrics_utils.AUCCurve.PR,886        summation_method=metrics_utils.AUCSummationMethod.MAJORING)887    self.evaluate(variables.variables_initializer(auc_obj.variables))888    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)889    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]890    # precision = [7/(7+3), 4/4, 0] = [0.7, 1, 0]891    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]892    # heights = [max(0.7, 1), max(1, 0)] = [1, 1]893    # widths = [(1 - 0.571), (0.571 - 0)] = [0.429, 0.571]894    expected_result = (1 * 0.429 + 1 * 0.571)895    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)896  def test_weighted_pr_minoring(self):897    self.setup()898    auc_obj = metrics.AUC(899        num_thresholds=self.num_thresholds,900        curve=metrics_utils.AUCCurve.PR,901        summation_method=metrics_utils.AUCSummationMethod.MINORING)902    self.evaluate(variables.variables_initializer(auc_obj.variables))903    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)904    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]905    # precision = [7/(7+3), 4/4, 0] = [0.7, 1, 0]906    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]907    # heights = [min(0.7, 1), min(1, 0)] = [0.7, 0]908    # widths = [(1 - 0.571), (0.571 - 0)] = [0.429, 0.571]909    expected_result = (0.7 * 0.429 + 0 * 0.571)910    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)911  def test_weighted_pr_interpolation(self):912    self.setup()913    auc_obj = metrics.AUC(914        num_thresholds=self.num_thresholds,915        curve=metrics_utils.AUCCurve.PR,916        summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION)917    self.evaluate(variables.variables_initializer(auc_obj.variables))918    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)919    # auc = (slope / Total Pos) * [dTP - intercept * log(Pb/Pa)]920    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]921    # P = tp + fp = [10, 4, 0]922    # dTP = [7-4, 4-0] = [3, 4]923    # dP = [10-4, 4-0] = [6, 4]924    # slope = dTP/dP = [0.5, 1]925    # intercept = (TPa+(slope*Pa) = [(4 - 0.5*4), (0 - 1*0)] = [2, 0]926    # (Pb/Pa) = (Pb/Pa) if Pb > 0 AND Pa > 0 else 1 = [10/4, 4/0] = [2.5, 1]927    # auc * TotalPos = [(0.5 * (3 + 2 * log(2.5))), (1 * (4 + 0))]928    #                = [2.416, 4]929    # auc = [2.416, 4]/(tp[1:]+fn[1:])930    expected_result = (2.416/7 + 4/7)931    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)932  def test_invalid_num_thresholds(self):933    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):934      metrics.AUC(num_thresholds=-1)935    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):936      metrics.AUC(num_thresholds=1)937if __name__ == '__main__':...