Weights and Biases not updating in tensorflow

Question

I've made this neural net to figure out whether a house is a good buy or a bad buy. For some reasons the code is not updating weights and biases. My loss stays same. This is my code:

I've made this neural net to figure out whether a house is a good buy or a bad buy. For some reasons the code is not updating weights and biases. My loss stays same. This is my code:

import pandas as pd
import tensorflow as tf

data = pd.read_csv("E:/workspace_py/datasets/good_bad_buy.csv")

features = data.drop(['index', 'good buy'], axis = 1)
lbls = data.drop(['index', 'area', 'bathrooms', 'price', 'sq_price'], axis = 1)

features = features[0:20]
lbls = lbls[0:20]

print(features)
print(lbls)
n_examples = len(lbls)

# Model

# Hyper parameters

epochs = 100
learning_rate = 0.1
batch_size = 1

input_data = tf.placeholder('float', [None, 4])
labels = tf.placeholder('float', [None, 1])

weights = {
            'hl1': tf.Variable(tf.random_normal([4, 10])),
            'hl2': tf.Variable(tf.random_normal([10, 10])),
            'hl3': tf.Variable(tf.random_normal([10, 4])),
            'ol': tf.Variable(tf.random_normal([4, 1]))
            }

biases = {
            'hl1': tf.Variable(tf.random_normal([10])),
            'hl2': tf.Variable(tf.random_normal([10])),
            'hl3': tf.Variable(tf.random_normal([4])),
            'ol': tf.Variable(tf.random_normal([1]))
            }

hl1 = tf.nn.relu(tf.add(tf.matmul(input_data, weights['hl1']), biases['hl1']))
hl2 = tf.nn.relu(tf.add(tf.matmul(hl1, weights['hl2']), biases['hl2']))
hl3 = tf.nn.relu(tf.add(tf.matmul(hl2, weights['hl3']), biases['hl3']))
ol = tf.nn.sigmoid(tf.add(tf.matmul(hl3, weights['ol']), biases['ol']))

loss = tf.reduce_mean((labels - ol)**2)
train = tf.train.AdamOptimizer(learning_rate).minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

iterations = int(n_examples/batch_size)


for epoch_no in range(epochs):
    ptr = 0
    for iteration_no in range(iterations):
        epoch_input = features[ptr:ptr+batch_size]
        epoch_label = lbls[ptr: ptr+batch_size]
        ptr = ptr + batch_size
        _, err = sess.run([train, loss], feed_dict={input_data: features, labels: lbls})
    print("Error at epoch ", epoch_no, ": ", err)

print(sess.run(ol, feed_dict={input_data: [[2104, 3, 399900, 190.0665]]}))

This is the dataset:

Features:

    area  bathrooms   price    sq_price
0   2104          3  399900  190.066540
1   1600          3  329900  206.187500
2   2400          3  369000  153.750000
3   1416          2  232000  163.841808
4   3000          4  539900  179.966667
5   1985          4  299900  151.083123
6   1534          3  314900  205.280313
7   1427          3  198999  139.452698
8   1380          3  212000  153.623188
9   1494          3  242500  162.315930
10  1940          4  239999  123.710825
11  2000          3  347000  173.500000
12  1890          3  329999  174.602645
13  4478          5  699900  156.297454
14  1268          3  259900  204.968454
15  2300          4  449900  195.608696
16  1320          2  299900  227.196970
17  1236          3  199900  161.731392
18  2609          4  499998  191.643542
19  3031          4  599000  197.624546

labels:

    good buy
0        1.0
1        0.0
2        1.0
3        0.0
4        1.0
5        0.0
6        0.0
7        1.0
8        0.0
9        0.0
10       1.0
11       1.0
12       1.0
13       1.0
14       0.0
15       1.0
16       0.0
17       1.0
18       1.0
19       1.0

Any suggestions on how to fix this? I've tried tf.reduce_sum other than tf.reduce_mean. I've also tried a larger batch_size.

Answer 1

A few things to consider

• Minibatch not being evaluated correctly since you feed in features and lbls instead of epoch_input and epoch_label.
• You do not precondition your data in any way, so it is completely out of range. I.e. my code below normalizes the features into stddev and mean. You might consider using batch_normalization.
• You are not evaluating error at any point. You need a held out training and testing set. My code below doesn't hold out data, but it does test in terms of error % rather than just loss (which is a weak proxy for error, so you shouldn't call it error).
• You initialize biases to random normals. You probably want to just start those at zero.
• You probably should use tf.layers or another high level api.

The below code achieves a training error of 95%. You'd want to test with a held out data set not used for training to evaluate the testing error.

#!/usr/bin/env python
import sys
import pandas as pd
import numpy as np
import tensorflow as tf


data = pd.read_csv("data.csv")

features = data.drop(['good buy'], axis = 1)
lbls = data.drop([ 'area', 'bathrooms', 'price', 'sq_price'], axis = 1)

features = features[0:20]
lbls = lbls[0:20]

mu = np.mean(features, axis=0)
sigma = (np.std(features, axis=0))
features = (features - mu) / sigma

n_examples = len(lbls)

# Model

# Hyper parameters

epochs = 100
learning_rate = 0.01
batch_size = 5

input_data = tf.placeholder('float', [None, 4])
labels = tf.placeholder('float', [None, 1])

weights = {
      'hl1': tf.Variable(tf.random_normal([4, 10])),
      'hl2': tf.Variable(tf.random_normal([10, 10])),
      'hl3': tf.Variable(tf.random_normal([10, 4])),
      'ol': tf.Variable(tf.random_normal([4, 1]))
      }

biases = {
      'hl1': tf.Variable(tf.zeros([10])),
      'hl2': tf.Variable(tf.zeros([10])),
      'hl3': tf.Variable(tf.zeros([4])),
      'ol': tf.Variable(tf.zeros([1]))
      }



hl1 = tf.nn.relu(tf.add(tf.matmul(input_data, weights['hl1']), biases['hl1']))
hl2 = tf.nn.relu(tf.add(tf.matmul(hl1, weights['hl2']), biases['hl2']))
hl3 = tf.nn.relu(tf.add(tf.matmul(hl2, weights['hl3']), biases['hl3']))
ol = tf.nn.sigmoid(tf.add(tf.matmul(hl3, weights['ol']), biases['ol']))

loss = tf.reduce_mean((labels - ol)**2)
train = tf.train.AdamOptimizer(learning_rate).minimize(loss)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

iterations = int(n_examples/batch_size)


def training_accuracy():
  foo,  = sess.run([ol], feed_dict={input_data: features, labels: lbls})
  return (float(np.count_nonzero(np.equal(np.round(foo), lbls))) / float(lbls.shape[0]))


print("Initial training accuracy %f" % training_accuracy())


for epoch_no in range(epochs):
  ptr = 0
  for iteration_no in range(iterations):
    epoch_input = features[ptr:ptr+batch_size]
    epoch_label = lbls[ptr: ptr+batch_size]
    ptr = (ptr + batch_size)%len(features)
    _, err = sess.run([train, loss], feed_dict={input_data: epoch_input, labels: epoch_label})
  print("Error at epoch ", epoch_no, ": ", err)
  print("  Training accuracy %f" % training_accuracy())

Also, please do not post usage questions like this on github, they belong here on StackOverflow.

Answer 2

I'm not sure if this is the problem for you. But sigmoid functions gradient can get very small if its input is to big, this can make updates very slow.

To check if this is the case for you try initializing all your weights at very small values. You can adjust this by setting a standard deviation for your random norms.

tf.Variable(tf.random_normal([4, 10],  stddev=0.1))

Answer 3

There are several things not ok with your code. First, you mean

    epoch_input = features[ptr:ptr+batch_size]
    epoch_label = lbls[ptr: ptr+batch_size]
    ptr = ptr + batch_size
    // _, err = sess.run([train, loss], feed_dict={input_data: features, labels: lbls}
    _, err = sess.run([train, loss], feed_dict={input_data: epoch_input, labels: epoch_label}

Now it uses minibatch.

Debugging the gradient:

You can always check some stuff by adding

loss = tf.Print(loss, [tf.reduce_sum(weights['hl1'])])

This will print the elements of that list [tf.reduce_sum(weights['hl1'])]. To investigate further your problem, you can check the gradients instead of using minimize

grads = tf.reduce_sum(tf.gradients(loss, ol)[0])
sess.run(grads, {input_data: features, labels: lbls})

And finally, the loss function is inappropriate/numerical instable for classification. With your version, I get:

variables
   Variable:0
   Variable_1:0
   Variable_2:0
   Variable_3:0
   Variable_4:0
   Variable_5:0
   Variable_6:0
   Variable_7:0
I tensorflow/core/kernels/logging_ops.cc:79] [-6.2784553]
-----------------------------------------
name MatMul_grad
gradient [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]
value [[-0.59977376 -0.30060738  0.55068201  0.15304407  1.39992142  0.07495346
  -0.87189424 -0.22595075 -0.30094525 -1.2688272 ]
 [-0.44018757  1.08651936 -0.26267499 -0.54463315  0.47019768  0.69873857
   0.56195319  0.20222363  0.38143152 -0.92212462]
 [-0.39977714 -1.07244122  0.41926911  1.4951371  -2.28751612  0.45676312
   0.88010246 -0.88077509 -1.25860023  0.56874037]
 [-0.98260719 -1.30747247 -1.4460088   1.0717535   0.08794415 -0.53184992
  -1.17537284 -0.51598179 -0.15323587  0.91142744]]
-----------------------------------------
name MatMul_1_grad
gradient [[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]
value [[-0.1170694   0.12174897  0.91696155  0.59427398  0.90844423  0.29010534
  -0.34039831 -0.62824941  0.37833953  0.27777222]
 [-0.34947088  1.09264851  0.27353975  1.31722498 -0.42032316 -2.74952078
  -0.66349608 -0.61844724 -0.82141227  1.21691799]
 [ 0.10453336 -1.68631995  0.45700032 -1.58120835 -1.23378754 -0.05648948
  -1.64761281 -0.57684237 -0.06499017 -0.49623618]
 [ 1.47821534 -0.5329541   0.09209292  1.78089786  1.71149898  0.30547267
   0.39544162  1.00369155  1.0097307  -0.92320329]
 [ 1.27038908 -2.17246103 -0.31276336  0.8945803   0.30964327  1.15329361
   0.9711507  -0.36301252 -0.05652813  0.63399518]
 [-0.30909851 -0.41660413 -0.50603527  0.11735299 -0.26837045  0.16547598
  -0.33875859 -0.46821991  0.25723135 -0.80380815]
 [-0.86255074 -1.11751068  0.01365725  0.66119182  0.48947951  1.6353699
  -0.794447    0.43182942 -0.97692633 -1.62605619]
 [ 1.38552308  0.83679706 -0.87287223  2.59401655 -0.61855     0.38301265
   1.09983373  0.49209142  1.03003716 -1.33537853]
 [ 0.74452382  1.57940936 -0.90974236 -1.2211293  -1.1076287   0.92846316
  -0.46856263 -0.3179535   0.75120807 -0.86442506]
 [ 0.31622764 -0.35965034 -0.02351121 -0.0650174   0.4714573   0.35687482
   1.43354905  0.39608309  0.42744714 -0.37226421]]
-----------------------------------------
name MatMul_2_grad
gradient [[ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]]
value [[-1.50904143  0.00228321  1.45787132  0.68312413]
 [-0.16627057  1.31303644  1.16326404  0.72901946]
 [ 0.8004092   0.37329885  0.89361066 -0.19850619]
 [ 1.58354807 -1.05612624  0.69891322 -0.32565734]
 [-1.57602286 -0.41256282  0.69086516 -0.54095054]
 [ 1.72376788 -0.53928965 -0.71574098 -0.94974124]
 [-0.62061429  1.51380932 -0.72585452 -0.07695383]
 [ 0.35537818  1.49691582  0.03931179  0.93435526]
 [ 0.20697887  1.39266443  0.73217523 -0.64737892]
 [ 1.00519872  0.90984046  1.68565321 -0.28157935]]
-----------------------------------------
name MatMul_3_grad
gradient [[ 0.]
 [ 0.]
 [ 0.]
 [ 0.]]
value [[ 0.94082022]
 [ 0.14753926]
 [-0.08765228]
 [ 1.32516992]]
-----------------------------------------
name Add_grad
gradient [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
value [ 1.71239722  1.12632215  0.75409448  0.01951236  0.32135537 -1.46281374
  0.40413955  0.54653352 -0.57894999  0.2746354 ]
-----------------------------------------
name Add_1_grad
gradient [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
value [ 0.74800217 -0.43517059 -0.77706921  1.46858656  1.09103405 -0.46681881
  0.6126743  -2.27877688  1.48809242 -1.19616997]
-----------------------------------------
name Add_2_grad
gradient [ 0.  0.  0.  0.]
value [-0.12137324 -0.23238407  0.17909229 -0.75496733]
-----------------------------------------
name Add_3_grad
gradient [ 0.]
value [-0.91176724]

As you see, almost all gradients are zero. Why?

• by definition (labels - ol) is in [0, 1]
• the squared value is much smaller than one
• the derivative of sigmoid s(x) is s'(x) = s(x)*(1-s(x)) the gradients are multiplied by this value which is again much smaller than one.

But after using sparse_softmax_cross_entropy_with_logits which is numerically stable and operates in the log-domain I get

variables
   Variable:0
   Variable_1:0
   Variable_2:0
   Variable_3:0
   Variable_4:0
   Variable_5:0
   Variable_6:0
   Variable_7:0
-----------------------------------------
name MatMul_grad
gradient [[ -1.42780918e-05  -1.96137808e-05  -2.44040220e-05  -2.25691911e-05
    0.00000000e+00   2.95208647e-05   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]
 [ -2.54181440e-08  -3.49168410e-08  -4.34445262e-08  -4.01781257e-08
    0.00000000e+00   5.25536308e-08   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]
 [ -2.45539122e-03  -3.37296468e-03  -4.19673882e-03  -3.88120394e-03
    0.00000000e+00   5.07667707e-03   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]
 [ -1.42123906e-06  -1.95235293e-06  -2.42917258e-06  -2.24653377e-06
    0.00000000e+00   2.93850212e-06   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]]
value [[ 0.43133125 -0.40009859 -0.08456381  0.59587955  0.57171088 -0.9824872
   1.18876612  0.9704771   0.74798232  0.15660612]
 [-1.18380785  0.22617982 -1.15734088 -0.50478351  1.43819618  1.55950046
  -1.1510663  -0.88835335  0.58378232  0.56860197]
 [ 0.29826403  0.02192715  0.62225986  2.47716165 -0.9223454   1.70159853
  -1.03968358 -0.26019615 -0.33808291 -0.30873826]
 [ 0.59774327 -1.28855145 -0.43420359 -0.4413566  -0.19220066  0.96984953
  -0.04922202  0.32994318 -1.05539823 -0.80112725]]
-----------------------------------------
name MatMul_1_grad
gradient [[  0.00000000e+00   1.15650124e-03   0.00000000e+00   0.00000000e+00
    6.59449317e-04  -1.09400018e-03   0.00000000e+00  -4.02117817e-04
    5.44495881e-04  -8.90314346e-04]
 [  0.00000000e+00   7.24206184e-05   0.00000000e+00   0.00000000e+00
    4.12950030e-05  -6.85067716e-05   0.00000000e+00  -2.51807924e-05
    3.40965707e-05  -5.57518724e-05]
 [  0.00000000e+00   2.38713808e-03   0.00000000e+00   0.00000000e+00
    1.36117137e-03  -2.25812919e-03   0.00000000e+00  -8.30012548e-04
    1.12389564e-03  -1.83770037e-03]
 [  0.00000000e+00   9.52679198e-03   0.00000000e+00   0.00000000e+00
    5.43227792e-03  -9.01193265e-03   0.00000000e+00  -3.31248436e-03
    4.48533799e-03  -7.33405072e-03]
 [  0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]
 [  0.00000000e+00   6.51591457e-03   0.00000000e+00   0.00000000e+00
    3.71544389e-03  -6.16377220e-03   0.00000000e+00  -2.26559630e-03
    3.06777749e-03  -5.01617463e-03]
 [  0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]
 [  0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]
 [  0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]
 [  0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
    0.00000000e+00   0.00000000e+00]]
value [[ 0.38902158 -2.14370036 -1.02228141 -0.6492967   1.87193418 -0.06453216
   1.0013988  -1.26857054  0.59826601  0.45045251]
 [ 0.51465249 -1.09108925 -0.21368918 -0.49310678 -0.87893176 -0.07944249
  -0.15810326  1.65703297  1.01812947 -0.95572269]
 [-1.76351583 -1.46950841  1.43533802  2.15617752  1.30682683  0.77409673
  -1.50309181  0.81978178  0.6672287  -0.434971  ]
 [-0.7291944   2.16516733 -1.39850736 -1.06059277  0.40035763  1.23335707
  -0.03707252  1.88107574  0.09459961  2.11439633]
 [-1.39152992 -1.39924514 -0.35704514 -0.71152836 -2.68857026  0.78129828
  -1.0077033  -1.26149333  0.4403404  -0.10159389]
 [ 0.37354535  0.12654085  0.7632165  -0.76493222  0.68177891 -0.34254205
  -1.11582613  2.60665917  1.53196526 -0.867055  ]
 [ 0.62746197 -0.01072595  3.26629376  1.28371656 -0.88725293  3.55530715
   0.67065352 -0.61927503  1.20604384 -0.87207574]
 [-0.68954837  1.89912283  0.90083456  0.02054735 -0.23425011  0.39949065
  -0.08969283 -0.75943565  1.0924015   0.28920195]
 [-0.64865923 -1.29299021 -0.39945969  0.02289505  1.46024895  0.94282049
  -0.99704605 -1.36124468  0.76788425  0.86770487]
 [ 0.63794595  1.68530416 -0.15548207 -0.22658408 -0.45446202 -0.77308726
  -0.12694608  1.17369819  2.25879693  0.20346723]]
-----------------------------------------
name MatMul_2_grad
gradient [[ 0.          0.          0.          0.        ]
 [-0.02205572  0.          0.00960038  0.        ]
 [ 0.          0.          0.          0.        ]
 [ 0.          0.          0.          0.        ]
 [-0.01932034  0.          0.00840973  0.        ]
 [-0.01617817  0.          0.00704201  0.        ]
 [ 0.          0.          0.          0.        ]
 [-0.05091252  0.          0.02216113  0.        ]
 [-0.0189826   0.          0.00826272  0.        ]
 [-0.01993647  0.          0.00867792  0.        ]]
value [[-0.18724969 -0.0544498  -0.69153035  0.47535184]
 [-0.75444973 -1.33321464 -0.13066645  1.56889391]
 [-0.6458627   1.17859495 -0.75926393  0.30138403]
 [ 1.0069555  -0.69344127  0.49295315  0.54917085]
 [-0.55954564 -1.13277721 -0.37167427 -0.64837182]
 [ 0.93753678  1.12197697  0.63789612  0.52438796]
 [ 0.77543265 -1.241382    1.78230286 -0.6928125 ]
 [ 0.95383584 -2.00331807  1.63409865 -0.36474878]
 [-0.73891008  2.066082   -0.94303596 -0.42322466]
 [ 0.38519588  0.03278512 -0.3487882  -1.50447905]]
-----------------------------------------
name MatMul_3_grad
gradient [[ 0.08460998]
 [ 0.        ]
 [ 0.16564058]
 [ 0.        ]]
value [[-0.35376808]
 [-0.07330427]
 [ 0.15398768]
 [-0.06484076]]
-----------------------------------------
name Add_grad
gradient [ -8.22783885e-09  -1.13025616e-08  -1.40629695e-08  -1.30056375e-08
   0.00000000e+00   1.70115797e-08   0.00000000e+00   0.00000000e+00
   0.00000000e+00   0.00000000e+00]
value [-1.00038147 -0.56519473  0.59372097 -1.1646167  -0.16213787 -0.69313556
  0.62788707  1.03768504  0.57876503 -0.5201084 ]
-----------------------------------------
name Add_1_grad
gradient [  0.00000000e+00   1.28705375e-08   0.00000000e+00   0.00000000e+00
   7.33891703e-09  -1.21749730e-08   0.00000000e+00  -4.47511184e-09
   6.05961770e-09  -9.90818183e-09]
value [ 0.02854451 -1.46039021 -0.03916361  0.40116394  0.16030532  0.88267213
 -0.46328214  0.18927227 -1.7536788  -0.46590349]
-----------------------------------------
name Add_2_grad
gradient [ -1.84504412e-08   0.00000000e+00   8.03108247e-09   0.00000000e+00]
value [ 0.94534302 -0.9080081  -1.86719894 -1.31547296]
-----------------------------------------
name Add_3_grad
gradient [ 0.29727879 -0.29727876]
value [ 0.07999782 -0.75647992]

The gradients are (while very small) this time non zero. The code for reproducing that is

import numpy as np
import tensorflow as tf

features = [
[2104, 3, 399900, 190.066540],
[1600, 3, 329900, 206.187500],
[2400, 3, 369000, 153.750000],
[1416, 2, 232000, 163.841808],
[3000, 4, 539900, 179.966667],
[1985, 4, 299900, 151.083123],
[1534, 3, 314900, 205.280313],
[1427, 3, 198999, 139.452698],
[1380, 3, 212000, 153.623188],
[1494, 3, 242500, 162.315930],
[1940, 4, 239999, 123.710825],
[2000, 3, 347000, 173.500000],
[1890, 3, 329999, 174.602645],
[4478, 5, 699900, 156.297454],
[1268, 3, 259900, 204.968454],
[2300, 4, 449900, 195.608696],
[1320, 2, 299900, 227.196970],
[1236, 3, 199900, 161.731392],
[2609, 4, 499998, 191.643542],
[3031, 4, 599000, 197.624546]]

lbls = [1,0,1,0,1,0,0,1,0,0,1,1,1,1,0,1,0,1,1,1]
features = np.array(features, dtype=np.float32)
lbls = np.array(lbls, dtype=np.int32)

n_examples = len(lbls)
epochs = 100
learning_rate = 0.1
batch_size = 1

input_data = tf.placeholder('float', [None, 4])
labels = tf.placeholder('int32', [None])

weights = {
            'hl1': tf.Variable(tf.random_normal([4, 10])),
            'hl2': tf.Variable(tf.random_normal([10, 10])),
            'hl3': tf.Variable(tf.random_normal([10, 4])),
            'ol': tf.Variable(tf.random_normal([4, 1]))
            }

biases = {
            'hl1': tf.Variable(tf.random_normal([10])),
            'hl2': tf.Variable(tf.random_normal([10])),
            'hl3': tf.Variable(tf.random_normal([4])),
            # 'ol': tf.Variable(tf.random_normal([1])),
            'ol': tf.Variable(tf.random_normal([2]))
            }

hl1 = tf.nn.relu(tf.add(tf.matmul(input_data, weights['hl1']), biases['hl1']))
hl2 = tf.nn.relu(tf.add(tf.matmul(hl1, weights['hl2']), biases['hl2']))
hl3 = tf.nn.relu(tf.add(tf.matmul(hl2, weights['hl3']), biases['hl3']))
# ol = tf.nn.sigmoid(tf.add(tf.matmul(hl3, weights['ol']), biases['ol']))
logits = tf.add(tf.matmul(hl3, weights['ol']), biases['ol'])

# ol = tf.Print(ol, [tf.reduce_sum(weights['hl1'])])
# loss = tf.reduce_mean((labels - ol)**2)
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)
# loss = tf.reduce_mean((labels - ol)**2)
loss = tf.reduce_mean(cost)
optimizer = tf.train.AdamOptimizer(learning_rate)

iterations = int(n_examples/batch_size)

def debug_minimize(optimizer, loss, sess):
    from tensorflow.python.ops import variables
    from tensorflow.python.framework import ops
    # get all varibles
    var_list = (variables.trainable_variables() + ops.get_collection(ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))
    print 'variables'
    for v in var_list:
        print '  ', v.name
    # get all gradients
    grads_and_vars = optimizer.compute_gradients(loss)
    train_op = optimizer.apply_gradients(grads_and_vars)

    zipped_val = sess.run(grads_and_vars, {input_data: features, labels: lbls})

    for rsl, tensor in zip(zipped_val, grads_and_vars):
        print '-----------------------------------------'
        print 'name', tensor[0].name.replace('/tuple/control_dependency_1:0', '').replace('gradients/', '')
        print 'gradient', rsl[0]
        print 'value', rsl[1]
    return train_op

sess = tf.Session()
sess.run(tf.global_variables_initializer())
debug_minimize(optimizer, loss, sess)