tensorflow log_softmax tf.nn.log(tf.nn.softmax(predict)) tf.nn.softmax_cross_entropy_with_logits

Posted 2023-02-16

【中文标题】tensorflow log_softmax tf.nn.log(tf.nn.softmax(predict)) tf.nn.softmax_cross_entropy_with_logits

【英文标题】：

【发布时间】：2017-04-02 04:37:16

【问题描述】：

我尝试按照 tensorflow 教程实现 MNIST CNN 神经网络，并找到这些实现 softmax 交叉熵的方法给出不同的结果：

(1) 不好的结果

softmax = tf.nn.softmax(pred)
cross_entropy_cnn = - y * tf.log(softmax + 1e-10)
cost = tf.reduce_sum(cross_entropy_cnn)

(2) 好结果

cross_entropy_cnn = -y * tf.nn.log_softmax(pred)
cost = tf.reduce_sum(cross_entropy_cnn) 

(3) 好结果

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))

在数学上，这些方法是相同的。我的测试程序如下：

import tutorials.mnist.input_data as input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_iters = 20000
batch_size = 100
display_step = 10

# Network Parameters
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
dropout = 0.75 # Dropout, probability to keep units

W_conv1 = tf.Variable(tf.random_normal(shape=[5,5,1,32]))
b_conv1 = tf.Variable(tf.random_normal(shape=[1,32]))
W_conv2 = tf.Variable(tf.random_normal(shape=[5,5,32,64]))
b_conv2 = tf.Variable(tf.random_normal( shape=[1,64]))
W_full = tf.Variable(tf.random_normal(shape=[7 * 7 * 64, 1024]))
b_full = tf.Variable(tf.random_normal(shape=[1, 1024]))
W_softmax = tf.Variable(tf.truncated_normal(shape=[1024, 10]))
b_softmax = tf.Variable(tf.truncated_normal(shape=[1,10]))


# tf Graph input
x = tf.placeholder(tf.float32, [None, n_input])
y = tf.placeholder(tf.float32, [None, n_classes])
keep_prob = tf.placeholder(tf.float32, shape=()) #dropout (keep probability)
# Create some wrappers for simplicity
def conv2d(x, W, b, strides=1):
    # Conv2D wrapper, with bias and relu activation
    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')
    x = tf.nn.bias_add(x, b)
    return tf.nn.relu(x)


def maxpool2d(x, k=2):
    # MaxPool2D wrapper
    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],
                          padding='SAME')


# Create model
def conv_net(x,dropout):
    # Reshape input picture
    x = tf.reshape(x, shape=[-1, 28, 28, 1])

    # Convolution Layer
#     conv1 = conv2d(x, weights['wc1'], biases['bc1'])

    # Max Pooling (down-sampling)
    convOne = tf.nn.conv2d(x, W_conv1, strides=[1,1,1,1], padding="SAME")
    reluOne = tf.nn.relu(convOne + b_conv1)
    conv1 = tf.nn.max_pool(reluOne, ksize=[1,2,2,1],strides=[1,2,2,1],padding="SAME")

    # Convolution Layer

    convTwo = tf.nn.conv2d(conv1, W_conv2, strides=[1,1,1,1], padding="SAME")
    reluTwo = tf.nn.relu(convTwo + b_conv2)
    conv2 = tf.nn.max_pool(reluTwo, ksize=[1,2,2,1], strides=[1,2,2,1],padding="SAME")

    # Fully connected layer
    input_flat=tf.reshape(conv2, shape=[-1, 7 * 7 * 64])
    fc1 = tf.nn.relu(tf.matmul(input_flat, W_full) + b_full)

    # Apply Dropout
    drop_out = tf.nn.dropout(fc1,keep_prob)

    # Output, class prediction
    y_predict = tf.matmul(drop_out, W_softmax) + b_softmax

    return y_predict

# Construct model
pred = conv_net(x, keep_prob)

# Define loss and optimizer
# cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))  #(method (3)

# softmax = tf.nn.softmax(pred)              #method (1)
# cross_entropy_cnn = - y * tf.log(softmax + 1e-10) #method (1)
cross_entropy_cnn = -y * tf.nn.log_softmax(pred)  #method (2)
cost = tf.reduce_sum(cross_entropy_cnn) 

optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initializing the variables
init = tf.initialize_all_variables()

sess = tf.Session()
sess.run(tf.initialize_all_variables())
for i in range(20000):
    batch = mnist.train.next_batch(128)

    if i% 100 == 0:
        train_accuracy = accuracy.eval(feed_dict=x:batch[0], y:batch[1], keep_prob:1.0,session=sess)
        print ("step "+ str(i) +", training accuracy :"+ str(train_accuracy))
        cross_entropy_val = cross_entropy_cnn.eval(feed_dict=x:batch[0], y:batch[1], keep_prob:1.0,session=sess)

    sess.run(optimizer, feed_dict=x:batch[0], y:batch[1], keep_prob:0.75)
print("test accuracy :" + str(accuracy.eval(feed_dict=x:mnist.test.images, y:mnist.test.labels, keep_prob:1.0,session=sess)))
sess.close()

当我使用方法（1）时，结果是这样的：

step 0, training accuracy :0.109375
step 100, training accuracy :0.0703125
step 200, training accuracy :0.0546875
step 300, training accuracy :0.109375
step 400, training accuracy :0.132812
step 500, training accuracy :0.0390625
step 600, training accuracy :0.0859375
step 700, training accuracy :0.0703125
step 800, training accuracy :0.109375
step 900, training accuracy :0.101562
step 1000, training accuracy :0.140625
step 1100, training accuracy :0.0703125
step 1200, training accuracy :0.117188
step 1300, training accuracy :0.109375
step 1400, training accuracy :0.132812
step 1500, training accuracy :0.101562
step 1600, training accuracy :0.109375
step 1700, training accuracy :0.125
step 1800, training accuracy :0.117188
step 1900, training accuracy :0.0859375
step 2000, training accuracy :0.078125
step 2100, training accuracy :0.09375
step 2200, training accuracy :0.117188
step 2300, training accuracy :0.0546875
step 2400, training accuracy :0.117188
step 2500, training accuracy :0.0859375
step 2600, training accuracy :0.0703125
step 2700, training accuracy :0.078125
step 2800, training accuracy :0.117188
step 2900, training accuracy :0.09375
step 3000, training accuracy :0.0546875
step 3100, training accuracy :0.09375
step 3200, training accuracy :0.117188
step 3300, training accuracy :0.0703125
step 3400, training accuracy :0.125
step 3500, training accuracy :0.132812
step 3600, training accuracy :0.0859375
step 3700, training accuracy :0.078125
step 3800, training accuracy :0.0859375
step 3900, training accuracy :0.109375
step 4000, training accuracy :0.101562
step 4100, training accuracy :0.140625
step 4200, training accuracy :0.0859375
step 4300, training accuracy :0.125
step 4400, training accuracy :0.109375
step 4500, training accuracy :0.0859375
step 4600, training accuracy :0.09375
step 4700, training accuracy :0.117188
step 4800, training accuracy :0.132812
step 4900, training accuracy :0.0625
step 5000, training accuracy :0.09375
step 5100, training accuracy :0.078125
step 5200, training accuracy :0.09375
step 5300, training accuracy :0.0859375
step 5400, training accuracy :0.0703125
step 5500, training accuracy :0.109375
step 5600, training accuracy :0.132812
step 5700, training accuracy :0.09375
step 5800, training accuracy :0.117188
step 5900, training accuracy :0.0703125
step 6000, training accuracy :0.078125
step 6100, training accuracy :0.078125
step 6200, training accuracy :0.0703125
step 6300, training accuracy :0.09375
step 6400, training accuracy :0.09375
step 6500, training accuracy :0.117188
step 6600, training accuracy :0.0859375
step 6700, training accuracy :0.117188
step 6800, training accuracy :0.0859375
step 6900, training accuracy :0.078125
step 7000, training accuracy :0.109375
step 7100, training accuracy :0.09375
step 7200, training accuracy :0.117188
step 7300, training accuracy :0.140625
step 7400, training accuracy :0.101562
step 7500, training accuracy :0.0703125
step 7600, training accuracy :0.101562
step 7700, training accuracy :0.0703125
step 7800, training accuracy :0.078125
step 7900, training accuracy :0.0859375
step 8000, training accuracy :0.117188
step 8100, training accuracy :0.101562
step 8200, training accuracy :0.101562
step 8300, training accuracy :0.125
step 8400, training accuracy :0.125
step 8500, training accuracy :0.101562
step 8600, training accuracy :0.078125
step 8700, training accuracy :0.046875
step 8800, training accuracy :0.0859375
step 8900, training accuracy :0.109375
step 9000, training accuracy :0.101562
step 9100, training accuracy :0.132812
step 9200, training accuracy :0.109375
step 9300, training accuracy :0.109375
step 9400, training accuracy :0.0859375
step 9500, training accuracy :0.101562
step 9600, training accuracy :0.117188
step 9700, training accuracy :0.0703125
step 9800, training accuracy :0.0625
step 9900, training accuracy :0.0859375
step 10000, training accuracy :0.0625
step 10100, training accuracy :0.09375
step 10200, training accuracy :0.0859375
step 10300, training accuracy :0.09375
step 10400, training accuracy :0.078125
step 10500, training accuracy :0.148438
step 10600, training accuracy :0.101562
step 10700, training accuracy :0.125
step 10800, training accuracy :0.109375
step 10900, training accuracy :0.109375
step 11000, training accuracy :0.0625
step 11100, training accuracy :0.0859375
step 11200, training accuracy :0.078125
step 11300, training accuracy :0.148438
step 11400, training accuracy :0.078125
step 11500, training accuracy :0.109375
step 11600, training accuracy :0.117188
step 11700, training accuracy :0.09375
step 11800, training accuracy :0.078125
step 11900, training accuracy :0.0859375
step 12000, training accuracy :0.148438
step 12100, training accuracy :0.0859375
step 12200, training accuracy :0.09375
step 12300, training accuracy :0.101562
step 12400, training accuracy :0.078125
step 12500, training accuracy :0.109375
step 12600, training accuracy :0.078125
step 12700, training accuracy :0.101562
step 12800, training accuracy :0.0625
step 12900, training accuracy :0.101562
step 13000, training accuracy :0.109375
step 13100, training accuracy :0.125
step 13200, training accuracy :0.0703125
step 13300, training accuracy :0.117188
step 13400, training accuracy :0.101562
step 13500, training accuracy :0.140625
step 13600, training accuracy :0.132812
step 13700, training accuracy :0.109375
step 13800, training accuracy :0.148438
step 13900, training accuracy :0.09375
step 14000, training accuracy :0.109375
step 14100, training accuracy :0.0625
step 14200, training accuracy :0.125
step 14300, training accuracy :0.09375
step 14400, training accuracy :0.101562
step 14500, training accuracy :0.132812
step 14600, training accuracy :0.09375
step 14700, training accuracy :0.132812
step 14800, training accuracy :0.148438
step 14900, training accuracy :0.109375
step 15000, training accuracy :0.117188
step 15100, training accuracy :0.125
step 15200, training accuracy :0.117188
step 15300, training accuracy :0.109375
step 15400, training accuracy :0.0859375
step 15500, training accuracy :0.148438
step 15600, training accuracy :0.078125
step 15700, training accuracy :0.117188
step 15800, training accuracy :0.0859375
step 15900, training accuracy :0.09375
step 16000, training accuracy :0.078125
step 16100, training accuracy :0.109375
step 16200, training accuracy :0.101562
step 16300, training accuracy :0.125
step 16400, training accuracy :0.109375
step 16500, training accuracy :0.109375
step 16600, training accuracy :0.078125
step 16700, training accuracy :0.117188
step 16800, training accuracy :0.125
step 16900, training accuracy :0.109375
step 17000, training accuracy :0.132812
step 17100, training accuracy :0.109375
step 17200, training accuracy :0.117188
step 17300, training accuracy :0.148438
step 17400, training accuracy :0.0859375
step 17500, training accuracy :0.109375
step 17600, training accuracy :0.09375
step 17700, training accuracy :0.09375
step 17800, training accuracy :0.101562
step 17900, training accuracy :0.078125
step 18000, training accuracy :0.148438
step 18100, training accuracy :0.09375
step 18200, training accuracy :0.171875
step 18300, training accuracy :0.101562
step 18400, training accuracy :0.078125
step 18500, training accuracy :0.109375
step 18600, training accuracy :0.0859375
step 18700, training accuracy :0.078125
step 18800, training accuracy :0.101562
step 18900, training accuracy :0.140625
step 19000, training accuracy :0.0546875
step 19100, training accuracy :0.0859375
step 19200, training accuracy :0.0859375
step 19300, training accuracy :0.0859375
step 19400, training accuracy :0.078125
step 19500, training accuracy :0.117188
step 19600, training accuracy :0.078125
step 19700, training accuracy :0.117188
step 19800, training accuracy :0.0859375
step 19900, training accuracy :0.148438
test accuracy :0.1032

和方法（2）和（3）是这样的：

step 0, training accuracy :0.101562
step 100, training accuracy :0.789062
step 200, training accuracy :0.875
step 300, training accuracy :0.921875
step 400, training accuracy :0.929688
step 500, training accuracy :0.953125
step 600, training accuracy :0.960938
step 700, training accuracy :0.96875
step 800, training accuracy :0.960938
step 900, training accuracy :0.984375
step 1000, training accuracy :0.984375
step 1100, training accuracy :0.96875
step 1200, training accuracy :0.984375
step 1300, training accuracy :0.960938
step 1400, training accuracy :0.984375
step 1500, training accuracy :1.0
step 1600, training accuracy :1.0
step 1700, training accuracy :0.992188
step 1800, training accuracy :0.96875
step 1900, training accuracy :0.96875
step 2000, training accuracy :1.0
step 2100, training accuracy :0.984375
step 2200, training accuracy :0.96875
step 2300, training accuracy :0.984375
step 2400, training accuracy :0.984375
step 2500, training accuracy :0.96875
step 2600, training accuracy :0.992188
step 2700, training accuracy :0.984375
step 2800, training accuracy :0.96875
step 2900, training accuracy :0.984375
step 3000, training accuracy :0.992188
step 3100, training accuracy :0.976562
step 3200, training accuracy :1.0
step 3300, training accuracy :0.984375
step 3400, training accuracy :0.984375
step 3500, training accuracy :0.984375
step 3600, training accuracy :0.992188
step 3700, training accuracy :0.984375
step 3800, training accuracy :0.984375
step 3900, training accuracy :0.984375
step 4000, training accuracy :0.96875
step 4100, training accuracy :1.0
step 4200, training accuracy :1.0
step 4300, training accuracy :1.0
step 4400, training accuracy :0.984375
step 4500, training accuracy :1.0
step 4600, training accuracy :0.984375
step 4700, training accuracy :0.984375
step 4800, training accuracy :1.0
step 4900, training accuracy :1.0
step 5000, training accuracy :1.0
step 5100, training accuracy :0.984375
step 5200, training accuracy :0.992188
step 5300, training accuracy :0.992188
step 5400, training accuracy :1.0
step 5500, training accuracy :1.0
step 5600, training accuracy :1.0
step 5700, training accuracy :1.0
step 5800, training accuracy :1.0
step 5900, training accuracy :0.992188
step 6000, training accuracy :1.0
step 6100, training accuracy :1.0
step 6200, training accuracy :0.992188
step 6300, training accuracy :0.992188
step 6400, training accuracy :0.992188
step 6500, training accuracy :0.992188
step 6600, training accuracy :0.992188
step 6700, training accuracy :1.0
step 6800, training accuracy :1.0
step 6900, training accuracy :1.0
step 7000, training accuracy :1.0
step 7100, training accuracy :1.0
step 7200, training accuracy :0.992188
step 7300, training accuracy :0.992188
step 7400, training accuracy :1.0
step 7500, training accuracy :1.0
step 7600, training accuracy :0.992188
step 7700, training accuracy :1.0
step 7800, training accuracy :0.984375
step 7900, training accuracy :1.0
step 8000, training accuracy :1.0
step 8100, training accuracy :0.992188
step 8200, training accuracy :1.0
step 8300, training accuracy :1.0
step 8400, training accuracy :1.0
step 8500, training accuracy :1.0
step 8600, training accuracy :1.0
step 8700, training accuracy :1.0
step 8800, training accuracy :1.0
step 8900, training accuracy :1.0
step 9000, training accuracy :1.0
step 9100, training accuracy :1.0
step 9200, training accuracy :1.0
step 9300, training accuracy :1.0
step 9400, training accuracy :1.0
step 9500, training accuracy :1.0
step 9600, training accuracy :0.992188
step 9700, training accuracy :0.992188
step 9800, training accuracy :1.0
step 9900, training accuracy :1.0
step 10000, training accuracy :1.0
step 10100, training accuracy :1.0
step 10200, training accuracy :0.992188
step 10300, training accuracy :1.0
step 10400, training accuracy :1.0
step 10500, training accuracy :1.0
step 10600, training accuracy :0.992188
step 10700, training accuracy :1.0
step 10800, training accuracy :1.0
step 10900, training accuracy :1.0
step 11000, training accuracy :1.0
step 11100, training accuracy :1.0
step 11200, training accuracy :1.0
step 11300, training accuracy :1.0
step 11400, training accuracy :0.992188
step 11500, training accuracy :1.0
step 11600, training accuracy :1.0
step 11700, training accuracy :1.0
step 11800, training accuracy :1.0
step 11900, training accuracy :1.0
step 12000, training accuracy :1.0
step 12100, training accuracy :1.0
step 12200, training accuracy :0.992188
step 12300, training accuracy :1.0
step 12400, training accuracy :1.0
step 12500, training accuracy :1.0
step 12600, training accuracy :1.0
step 12700, training accuracy :1.0
step 12800, training accuracy :1.0
step 12900, training accuracy :1.0
step 13000, training accuracy :1.0
step 13100, training accuracy :1.0
step 13200, training accuracy :0.992188
step 13300, training accuracy :1.0
step 13400, training accuracy :1.0
step 13500, training accuracy :1.0
step 13600, training accuracy :1.0
step 13700, training accuracy :1.0
step 13800, training accuracy :1.0
step 13900, training accuracy :1.0
step 14000, training accuracy :1.0
step 14100, training accuracy :1.0
step 14200, training accuracy :1.0
step 14300, training accuracy :1.0
step 14400, training accuracy :1.0
step 14500, training accuracy :1.0
step 14600, training accuracy :1.0
step 14700, training accuracy :1.0
step 14800, training accuracy :1.0
step 14900, training accuracy :1.0
step 15000, training accuracy :1.0
step 15100, training accuracy :1.0
step 15200, training accuracy :1.0
step 15300, training accuracy :1.0
step 15400, training accuracy :0.992188
step 15500, training accuracy :1.0
step 15600, training accuracy :1.0
step 15700, training accuracy :1.0
step 15800, training accuracy :1.0
step 15900, training accuracy :1.0
step 16000, training accuracy :1.0
step 16100, training accuracy :1.0
step 16200, training accuracy :1.0
step 16300, training accuracy :1.0
step 16400, training accuracy :1.0
step 16500, training accuracy :1.0
step 16600, training accuracy :1.0
step 16700, training accuracy :0.992188
step 16800, training accuracy :1.0
step 16900, training accuracy :1.0
step 17000, training accuracy :1.0
step 17100, training accuracy :1.0
step 17200, training accuracy :1.0
step 17300, training accuracy :1.0
step 17400, training accuracy :1.0
step 17500, training accuracy :1.0
step 17600, training accuracy :1.0
step 17700, training accuracy :1.0
step 17800, training accuracy :1.0
step 17900, training accuracy :1.0
step 18000, training accuracy :1.0
step 18100, training accuracy :1.0
step 18200, training accuracy :1.0
step 18300, training accuracy :1.0
step 18400, training accuracy :1.0
step 18500, training accuracy :1.0
step 18600, training accuracy :1.0
step 18700, training accuracy :1.0
step 18800, training accuracy :0.992188
step 18900, training accuracy :1.0
step 19000, training accuracy :1.0
step 19100, training accuracy :1.0
step 19200, training accuracy :1.0
step 19300, training accuracy :1.0
step 19400, training accuracy :1.0
step 19500, training accuracy :1.0
step 19600, training accuracy :1.0
step 19700, training accuracy :1.0
step 19800, training accuracy :1.0
step 19900, training accuracy :1.0
test accuracy :0.987

这两个结果大相径庭。我想知道（1）方法有什么问题，因为（1）方法在数学上是正确的。我想方法（2）和（3）有一些特殊的过程。

【问题讨论】：

你没有把结果放在“正确”的方法中。

这会有所帮助：***.com/questions/34240703/…

@sygi 抱歉忘记附上结果。我修改了问题并添加了方法（2）和（3）的结果。

我认为这是两个完全不同的问题。 @假

使用mean和sum有区别...

【参考方案1】：

tf.nn.softmax_cross_entropy_with_logits

这是一个非常糟糕的名字。 这部分正好"softmax_cross_entropy_with_logits" 它应该只是"cross_entropy"，因为如果你认为，“交叉熵”是"nll_loss" 的"log_softmax"；所以 "softmax" 作为前缀是错误的。

然后"logits" 为“softmax”中的任何内容提供完善的名称，所以又是非常可疑的名称。

【参考方案2】：

下面解释方法一和方法二的区别。

1.softmax变量包含0时，cost的值不同。因为0的对数处理不同。

代码

#label
y = tf.constant([0,1])
#predicted result
pred = tf.Variable([1.0,1000.0])
#method (1)
softmax = tf.nn.softmax(pred)
cross_entropy_cnn = tf.log(softmax + 1e-10)
cost = tf.reduce_sum(cross_entropy_cnn)
#method (2)
cross_entropy_cnn2 = tf.nn.log_softmax(pred)
cost2 = tf.reduce_sum(cross_entropy_cnn2)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print('softmax value:',sess.run(softmax))
print('method_1 cost:',sess.run(cost))
print('method_2 cost:',sess.run(cost2))

输出

softmax value: [0. 1.]
method_1 cost: -23.02585
method_2 cost: -999.0

2.在方法一和方法二中运行你的代码会导致成本值的巨大差距。

代码

softmax = tf.nn.softmax(pred)              #method (1)
cross_entropy_cnn = - y * tf.log(softmax+1e-10) #method (1)
cross_entropy_cnn2 = -y * tf.nn.log_softmax(pred)  #method (2)
cost = tf.reduce_sum(cross_entropy_cnn)
cost2 = tf.reduce_sum(cross_entropy_cnn2)
sess = tf.Session()
sess.run(tf.initialize_all_variables())
batch = mnist.train.next_batch(1)
print('softmax:',sess.run(softmax, feed_dict=x:batch[0], y:batch[1], keep_prob:0.75))
print('method_1:',sess.run(cost, feed_dict=x:batch[0], y:batch[1], keep_prob:0.75))
print('method_2:',sess.run(cost2, feed_dict=x:batch[0], y:batch[1], keep_prob:0.75))

输出

softmax: [[0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]]
method_1: 23.02585
method_2: 17056.938