基于MNIST数据的循环神经网络RNN

blake

本文输入数据是MNIST，全称是Modified National Institute of Standards and Technology，是一组由这个机构搜集的手写数字扫描文件和每个文件对应标签的数据集，经过一定的修改使其适合机器学习算法读取。这个数据集可以从牛的不行的Yann LeCun教授的网站获取。
本系列的其他文章已经根据TensorFlow的官方教程基于MNIST数据集采用了softmax regression和CNN进行建模。为了完整性，本文对MNIST数据应用RNN模型求解，具体使用的RNN为LSTM。
关于RNN/LSTM的理论知识，可以参考这篇文章

1. # coding: utf-8
   
2. # @author: 陈水平
   
3. # @date：2017-02-14
   
4. #
   
5. 
   
6. # In[1]:
   
7. 
   
8. import tensorflow as tf
   
9. import numpy as np
   
10. 
    
11. 
    
12. # In[2]:
    
13. 
    
14. sess = tf.InteractiveSession()
    
15. 
    
16. 
    
17. # In[3]:
    
18. 
    
19. from tensorflow.examples.tutorials.mnist import input_data
    
20. mnist = input_data.read_data_sets('mnist/', one_hot=True)
    
21. 
    
22. 
    
23. # In[4]:
    
24. 
    
25. learning_rate = 0.001
    
26. batch_size = 128
    
27. 
    
28. n_input = 28
    
29. n_steps = 28
    
30. n_hidden = 128
    
31. n_classes = 10
    
32. 
    
33. x = tf.placeholder(tf.float32, [None, n_steps, n_input])
    
34. y = tf.placeholder(tf.float32, [None, n_classes])
    
35. 
    
36. 
    
37. # In[5]:
    
38. 
    
39. def RNN(x, weight, biases):
    
40.     # x shape: (batch_size, n_steps, n_input)
    
41.     # desired shape: list of n_steps with element shape (batch_size, n_input)
    
42.     x = tf.transpose(x, [1, 0, 2])
    
43.     x = tf.reshape(x, [-1, n_input])
    
44.     x = tf.split(0, n_steps, x)
    
45.     outputs = list()
    
46.     lstm = tf.nn.rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
    
47.     state = (tf.zeros([n_steps, n_hidden]),)*2
    
48.     sess.run(state)
    
49.     with tf.variable_scope("myrnn2") as scope:
    
50.         for i in range(n_steps-1):
    
51.             if i > 0:
    
52.                 scope.reuse_variables()
    
53.             output, state = lstm(x[i], state)
    
54.             outputs.append(output)
    
55.     final = tf.matmul(outputs[-1], weight) + biases
    
56.     return final
    
57. 
    
58. 
    
59. # In[6]:
    
60. 
    
61. def RNN(x, n_steps, n_input, n_hidden, n_classes):
    
62.     # Parameters:
    
63.     # Input gate: input, previous output, and bias
    
64.     ix = tf.Variable(tf.truncated_normal([n_input, n_hidden], -0.1, 0.1))
    
65.     im = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], -0.1, 0.1))
    
66.     ib = tf.Variable(tf.zeros([1, n_hidden]))
    
67.     # Forget gate: input, previous output, and bias
    
68.     fx = tf.Variable(tf.truncated_normal([n_input, n_hidden], -0.1, 0.1))
    
69.     fm = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], -0.1, 0.1))
    
70.     fb = tf.Variable(tf.zeros([1, n_hidden]))
    
71.     # Memory cell: input, state, and bias
    
72.     cx = tf.Variable(tf.truncated_normal([n_input, n_hidden], -0.1, 0.1))
    
73.     cm = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], -0.1, 0.1))
    
74.     cb = tf.Variable(tf.zeros([1, n_hidden]))
    
75.     # Output gate: input, previous output, and bias
    
76.     ox = tf.Variable(tf.truncated_normal([n_input, n_hidden], -0.1, 0.1))
    
77.     om = tf.Variable(tf.truncated_normal([n_hidden, n_hidden], -0.1, 0.1))
    
78.     ob = tf.Variable(tf.zeros([1, n_hidden]))
    
79.     # Classifier weights and biases
    
80.     w = tf.Variable(tf.truncated_normal([n_hidden, n_classes]))
    
81.     b = tf.Variable(tf.zeros([n_classes]))
    
82. 
    
83.     # Definition of the cell computation
    
84.     def lstm_cell(i, o, state):
    
85.         input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)
    
86.         forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)
    
87.         update = tf.tanh(tf.matmul(i, cx) + tf.matmul(o, cm) + cb)
    
88.         state = forget_gate * state + input_gate * update
    
89.         output_gate = tf.sigmoid(tf.matmul(i, ox) +  tf.matmul(o, om) + ob)
    
90.         return output_gate * tf.tanh(state), state
    
91.    
    
92.     # Unrolled LSTM loop
    
93.     outputs = list()
    
94.     state = tf.Variable(tf.zeros([batch_size, n_hidden]))
    
95.     output = tf.Variable(tf.zeros([batch_size, n_hidden]))
    
96.    
    
97.     # x shape: (batch_size, n_steps, n_input)
    
98.     # desired shape: list of n_steps with element shape (batch_size, n_input)
    
99.     x = tf.transpose(x, [1, 0, 2])
    
100.     x = tf.reshape(x, [-1, n_input])
     
101.     x = tf.split(0, n_steps, x)
     
102.     for i in x:
     
103.         output, state = lstm_cell(i, output, state)
     
104.         outputs.append(output)
     
105.     logits =tf.matmul(outputs[-1], w) + b
     
106.     return logits
     
107. 
     
108. 
     
109. # In[7]:
     
110. 
     
111. pred = RNN(x, n_steps, n_input, n_hidden, n_classes)
     
112. 
     
113. cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
     
114. optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
     
115. 
     
116. correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
     
117. accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
     
118. 
     
119. # Initializing the variables
     
120. init = tf.global_variables_initializer()
     
121. 
     
122. 
     
123. # In[8]:
     
124. 
     
125. # Launch the graph
     
126. sess.run(init)
     
127. for step in range(20000):
     
128.     batch_x, batch_y = mnist.train.next_batch(batch_size)
     
129.     batch_x = batch_x.reshape((batch_size, n_steps, n_input))
     
130.     sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
     
131. 
     
132.     if step % 50 == 0:
     
133.         acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
     
134.         loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
     
135.         print "Iter " + str(step) + ", Minibatch Loss= " +               "{:.6f}".format(loss) + ", Training Accuracy= " +               "{:.5f}".format(acc)
     
136. print "Optimization Finished!"
     
137. 
     
138. 
     
139. # In[9]:
     
140. 
     
141. # Calculate accuracy for 128 mnist test images
     
142. test_len = batch_size
     
143. test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
     
144. test_label = mnist.test.labels[:test_len]
     
145. print "Testing Accuracy:", sess.run(accuracy, feed_dict={x: test_data, y: test_label})
     

复制代码

输出如下：

1. Iter 0, Minibatch Loss= 2.540429, Training Accuracy= 0.07812
   
2. Iter 50, Minibatch Loss= 2.423611, Training Accuracy= 0.06250
   
3. Iter 100, Minibatch Loss= 2.318830, Training Accuracy= 0.13281
   
4. Iter 150, Minibatch Loss= 2.276640, Training Accuracy= 0.13281
   
5. Iter 200, Minibatch Loss= 2.276727, Training Accuracy= 0.12500
   
6. Iter 250, Minibatch Loss= 2.267064, Training Accuracy= 0.16406
   
7. Iter 300, Minibatch Loss= 2.234139, Training Accuracy= 0.19531
   
8. Iter 350, Minibatch Loss= 2.295060, Training Accuracy= 0.12500
   
9. Iter 400, Minibatch Loss= 2.261856, Training Accuracy= 0.16406
   
10. Iter 450, Minibatch Loss= 2.220284, Training Accuracy= 0.17969
    
11. Iter 500, Minibatch Loss= 2.276015, Training Accuracy= 0.13281
    
12. Iter 550, Minibatch Loss= 2.220499, Training Accuracy= 0.14062
    
13. Iter 600, Minibatch Loss= 2.219574, Training Accuracy= 0.11719
    
14. Iter 650, Minibatch Loss= 2.189177, Training Accuracy= 0.25781
    
15. Iter 700, Minibatch Loss= 2.195167, Training Accuracy= 0.19531
    
16. Iter 750, Minibatch Loss= 2.226459, Training Accuracy= 0.18750
    
17. Iter 800, Minibatch Loss= 2.148620, Training Accuracy= 0.23438
    
18. Iter 850, Minibatch Loss= 2.122925, Training Accuracy= 0.21875
    
19. Iter 900, Minibatch Loss= 2.065122, Training Accuracy= 0.24219
    
20. ...
    
21. Iter 19350, Minibatch Loss= 0.001304, Training Accuracy= 1.00000
    
22. Iter 19400, Minibatch Loss= 0.000144, Training Accuracy= 1.00000
    
23. Iter 19450, Minibatch Loss= 0.000907, Training Accuracy= 1.00000
    
24. Iter 19500, Minibatch Loss= 0.002555, Training Accuracy= 1.00000
    
25. Iter 19550, Minibatch Loss= 0.002018, Training Accuracy= 1.00000
    
26. Iter 19600, Minibatch Loss= 0.000853, Training Accuracy= 1.00000
    
27. Iter 19650, Minibatch Loss= 0.001035, Training Accuracy= 1.00000
    
28. Iter 19700, Minibatch Loss= 0.007034, Training Accuracy= 0.99219
    
29. Iter 19750, Minibatch Loss= 0.000608, Training Accuracy= 1.00000
    
30. Iter 19800, Minibatch Loss= 0.002913, Training Accuracy= 1.00000
    
31. Iter 19850, Minibatch Loss= 0.003484, Training Accuracy= 1.00000
    
32. Iter 19900, Minibatch Loss= 0.005693, Training Accuracy= 1.00000
    
33. Iter 19950, Minibatch Loss= 0.001904, Training Accuracy= 1.00000
    
34. Optimization Finished!
    
35. 
    
36. Testing Accuracy: 0.992188

复制代码