Logistic regression: objects are not aligned

Question

I am trying to do logistic regression on this dataset from A Ng's machihne learning class in coursera.

The idea is that we have a cost function, which we need to minimize to find the parameters theta.

import numpy as np
from scipy.optimize import fmin_bfgs

data = np.loadtxt('ex2data1.txt',delimiter=",")
m,n = data.shape
X = np.array(np.column_stack((np.ones(m),data[:,:-1])))
y = np.array(data[:,2].reshape(m,1))
theta = np.array(np.zeros(n).reshape(n,1))

def sigmoid(z):
    return 1/(1+np.exp(-z))

def hypothesis(X,theta):
    return sigmoid( X.dot(theta) )

def cost(theta):
    print theta.shape
    h = hypothesis(X,theta)
    cost = (-y.T.dot(np.log(h))-(1-y).T.dot(np.log(1-h)))/m
    return cost

def gradient(theta):
    h = hypothesis(X,theta)
    grad = ((h-y).T.dot(X)).T/m
    return grad.flatten()

def fmin():
    initial_theta=np.zeros(n).reshape(n,1)
    theta=fmin_bfgs(cost,initial_theta,fprime=gradient)
    return theta

print fmin()

I am getting ValueError: Objects are not aligned but I have checked the shapes of all entities and still can't figure it out. Here is the traceback:

---> 32     theta=fmin_bfgs(cost,initial_theta,fprime=gradient)
     33 

/usr/lib/python2.7/dist-packages/scipy/optimize/optimize.pyc in fmin_bfgs(f, x0, fprime, args, gtol, norm, epsilon, maxiter, full_output, disp, retall, callback)
    775             'return_all': retall}
    776 
--> 777     res = _minimize_bfgs(f, x0, args, fprime, callback=callback, **opts)
    778 
    779     if full_output:

/usr/lib/python2.7/dist-packages/scipy/optimize/optimize.pyc in _minimize_bfgs(fun, x0, args, jac, callback, gtol, norm, eps, maxiter, disp, return_all, **unknown_options)
    844     gnorm = vecnorm(gfk, ord=norm)
    845     while (gnorm > gtol) and (k < maxiter):
--> 846         pk = -numpy.dot(Hk, gfk)
    847         try:
    848             alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \

ValueError: objects are not aligned

Answer 1

I modified your code, it can get the same result as LogisticRegression in sklearn with c=inf:

import numpy as np
from scipy.optimize import fmin_bfgs
import io
data = np.loadtxt('ex2data1.txt',delimiter=",")
m,n = data.shape
X = np.array(np.column_stack((np.ones(m),data[:,:-1])))
y = np.array(data[:,2].reshape(m,1))
theta = np.array(np.zeros(n).reshape(n,1))

def sigmoid(z):
    return 1/(1+np.exp(-z))

def hypothesis(X,theta):
    return sigmoid( X.dot(theta) )

def cost(theta):
    h = hypothesis(X,theta)
    cost = (-y.T.dot(np.log(h))-(1-y).T.dot(np.log(1-h)))/m
    r = cost[0]
    if np.isnan(r):
        return np.inf
    return r

def gradient(theta):
    theta = theta.reshape(-1, 1)
    h = hypothesis(X,theta)
    grad = ((h-y).T.dot(X)).T/m
    return grad.flatten()

def fmin():
    initial_theta=np.zeros(n)
    theta=fmin_bfgs(cost,initial_theta,fprime=gradient)
    return theta

theta = fmin()