逻辑回归

blake

1. #!/usr/bin/env python
   
2. # -*- coding=utf-8 -*-
   
3. # @author: 陈水平
   
4. # @date: 2017-01-04
   
5. # @description: compare the logistics regression of tensorflow with sklearn based on the exercise of deep learning course of Andrew Ng.
   
6. # @ref: http://openclassroom.stanford.edu/MainFolder/DocumentPage.php?course=DeepLearning&doc=exercises/ex4/ex4.html
   
7. 
   
8. import tensorflow as tf
   
9. import numpy as np
   
10. from sklearn.linear_model import LogisticRegression
    
11. from sklearn import preprocessing
    
12. 
    
13. # Read x and y
    
14. x_data = np.loadtxt("ex4x.dat").astype(np.float32)
    
15. y_data = np.loadtxt("ex4y.dat").astype(np.float32)
    
16. 
    
17. scaler = preprocessing.StandardScaler().fit(x_data)
    
18. x_data_standard = scaler.transform(x_data)
    
19. 
    
20. # We evaluate the x and y by sklearn to get a sense of the coefficients.
    
21. reg = LogisticRegression(C=999999999, solver="newton-cg")  # Set C as a large positive number to minimize the regularization effect
    
22. reg.fit(x_data, y_data)
    
23. print "Coefficients of sklearn: K=%s, b=%f" % (reg.coef_, reg.intercept_)
    
24. 
    
25. # Now we use tensorflow to get similar results.
    
26. W = tf.Variable(tf.zeros([2, 1]))
    
27. b = tf.Variable(tf.zeros([1, 1]))
    
28. y = 1 / (1 + tf.exp(-tf.matmul(x_data_standard, W) + b))
    
29. loss = tf.reduce_mean(- y_data.reshape(-1, 1) *  tf.log(y) - (1 - y_data.reshape(-1, 1)) * tf.log(1 - y))
    
30. 
    
31. optimizer = tf.train.GradientDescentOptimizer(1.3)
    
32. train = optimizer.minimize(loss)
    
33. 
    
34. init = tf.initialize_all_variables()
    
35. 
    
36. sess = tf.Session()
    
37. sess.run(init)
    
38. for step in range(100):
    
39.     sess.run(train)
    
40.     if step % 10 == 0:
    
41.         print step, sess.run(W).flatten(), sess.run(b).flatten()
    
42. 
    
43. print "Coefficients of tensorflow (input should be standardized): K=%s, b=%s" % (sess.run(W).flatten(), sess.run(b).flatten())
    
44. print "Coefficients of tensorflow (raw input): K=%s, b=%s" % (sess.run(W).flatten() / scaler.scale_, sess.run(b).flatten() - np.dot(scaler.mean_ / scaler.scale_, sess.run(W)))
    
45. 
    
46. 
    
47. # Problem solved and we are happy. But...
    
48. # I'd like to implement the logistic regression from a multi-class viewpoint instead of binary.
    
49. # In machine learning domain, it is called softmax regression
    
50. # In economic and statistics domain, it is called multinomial logit (MNL) model, proposed by Daniel McFadden, who shared the 2000  Nobel Memorial Prize in Economic Sciences.
    
51. 
    
52. print "------------------------------------------------"
    
53. print "We solve this binary classification problem again from the viewpoint of multinomial classification"
    
54. print "------------------------------------------------"
    
55. 
    
56. # As a tradition, sklearn first
    
57. reg = LogisticRegression(C=9999999999, solver="newton-cg", multi_class="multinomial")
    
58. reg.fit(x_data, y_data)
    
59. print "Coefficients of sklearn: K=%s, b=%f" % (reg.coef_, reg.intercept_)
    
60. print "A little bit difference at first glance. What about multiply them with 2?"
    
61. 
    
62. # Then try tensorflow
    
63. W = tf.Variable(tf.zeros([2, 2]))  # first 2 is feature number, second 2 is class number
    
64. b = tf.Variable(tf.zeros([1, 2]))
    
65. V = tf.matmul(x_data_standard, W) + b
    
66. y = tf.nn.softmax(V)  # tensorflow provide a utility function to calculate the probability of observer n choose alternative i, you can replace it with `y = tf.exp(V) / tf.reduce_sum(tf.exp(V), keep_dims=True, reduction_indices=[1])`
    
67. 
    
68. # Encode the y label in one-hot manner
    
69. lb = preprocessing.LabelBinarizer()
    
70. lb.fit(y_data)
    
71. y_data_trans = lb.transform(y_data)
    
72. y_data_trans = np.concatenate((1 - y_data_trans, y_data_trans), axis=1)  # Only necessary for binary class
    
73. 
    
74. loss = tf.reduce_mean(-tf.reduce_sum(y_data_trans * tf.log(y), reduction_indices=[1]))
    
75. optimizer = tf.train.GradientDescentOptimizer(1.3)
    
76. train = optimizer.minimize(loss)
    
77. 
    
78. init = tf.initialize_all_variables()
    
79. 
    
80. sess = tf.Session()
    
81. sess.run(init)
    
82. for step in range(100):
    
83.     sess.run(train)
    
84.     if step % 10 == 0:
    
85.         print step, sess.run(W).flatten(), sess.run(b).flatten()
    
86. 
    
87. print "Coefficients of tensorflow (input should be standardized): K=%s, b=%s" % (sess.run(W).flatten(), sess.run(b).flatten())
    
88. print "Coefficients of tensorflow (raw input): K=%s, b=%s" % ((sess.run(W) / scaler.scale_).flatten(),  sess.run(b).flatten() - np.dot(scaler.mean_ / scaler.scale_, sess.run(W)))

复制代码

输出如下：

1. Coefficients of sklearn: K=[[ 0.14834077  0.15890845]], b=-16.378743
   
2. 0 [ 0.33699557  0.34786162] [ -4.84287721e-09]
   
3. 10 [ 1.15830743  1.22841871] [ 0.02142336]
   
4. 20 [ 1.3378191   1.42655993] [ 0.03946959]
   
5. 30 [ 1.40735555  1.50197577] [ 0.04853692]
   
6. 40 [ 1.43754184  1.53418231] [ 0.05283691]
   
7. 50 [ 1.45117068  1.54856908] [ 0.05484771]
   
8. 60 [ 1.45742035  1.55512536] [ 0.05578374]
   
9. 70 [ 1.46030474  1.55814099] [ 0.05621871]
   
10. 80 [ 1.46163988  1.55953443] [ 0.05642065]
    
11. 90 [ 1.46225858  1.56017959] [ 0.0565144]
    
12. Coefficients of tensorflow (input should be standardized): K=[ 1.46252561  1.56045783], b=[ 0.05655487]
    
13. Coefficients of tensorflow (raw input): K=[ 0.14831361  0.15888004], b=[-16.26265144]
    
14. ------------------------------------------------
    
15. We solve this binary classification problem again from the viewpoint of multinomial classification
    
16. ------------------------------------------------
    
17. Coefficients of sklearn: K=[[ 0.07417039  0.07945423]], b=-8.189372
    
18. A little bit difference at first glance. What about multiply them with 2?
    
19. 0 [-0.33699557  0.33699557 -0.34786162  0.34786162] [  6.05359674e-09  -6.05359674e-09]
    
20. 10 [-0.68416572  0.68416572 -0.72988117  0.72988123] [ 0.02157043 -0.02157041]
    
21. 20 [-0.72234094  0.72234106 -0.77087188  0.77087194] [ 0.02693938 -0.02693932]
    
22. 30 [-0.72958517  0.72958535 -0.7784785   0.77847856] [ 0.02802362 -0.02802352]
    
23. 40 [-0.73103166  0.73103184 -0.77998811  0.77998811] [ 0.02824244 -0.02824241]
    
24. 50 [-0.73132294  0.73132324 -0.78029168  0.78029174] [ 0.02828659 -0.02828649]
    
25. 60 [-0.73138171  0.73138207 -0.78035289  0.78035301] [ 0.02829553 -0.02829544]
    
26. 70 [-0.73139352  0.73139393 -0.78036523  0.78036535] [ 0.02829732 -0.0282972 ]
    
27. 80 [-0.73139596  0.73139632 -0.78036767  0.78036791] [ 0.02829764 -0.02829755]
    
28. 90 [-0.73139644  0.73139679 -0.78036815  0.78036839] [ 0.02829781 -0.02829765]
    
29. Coefficients of tensorflow (input should be standardized): K=[-0.7313965   0.73139679 -0.78036827  0.78036839], b=[ 0.02829777 -0.02829769]
    
30. Coefficients of tensorflow (raw input): K=[-0.07417037  0.07446811 -0.07913655  0.07945422], b=[ 8.1893692  -8.18937111]

复制代码