Google cloud ML with Scikit-Learn raises: 'dic

Question

I used the following tutorial to use my Scikit-learn sentiment-analysis model in Google Cloud: https://cloud.google.com/ml-engine/docs/scikit/quickstart

My model defined as follows:

import csv
import os
from collections import defaultdict
import sys
import re
import numpy as np
import random
import math

import sklearn.datasets
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.linear_model import SGDClassifier
from sklearn import metrics
from sklearn.model_selection import GridSearchCV
from sklearn.externals import joblib
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score

def build_data_cv(datafile, trait_number):
    """
    Loads data
    """

    with open(datafile, "rb") as csvf:
        csvreader = csv.reader(csvf,delimiter=',',quotechar='"')
        data = []
        target = np.array([])

        for index, line in enumerate(csvreader):

            # escape header
            if index < 1:
                continue

            document = unicode(line[1], errors='replace')

            data.append(document)

            index_of_trait = trait_number + 2

            if line[index_of_trait].lower()=='y':
                target = np.append(target, 1.0)
            else:
                target = np.append(target, 0.0)

        dataset = sklearn.datasets.base.Bunch(data=data, target=target)
        dataset.target_names = ["positive", "negative"]

        return dataset

# main program
if __name__=="__main__":
    current_directory = os.getcwd() + "/"
    data_file = current_directory + "essays.csv"

    class_labels = ['EXT','NEU','AGR','CON','OPN']

    for index, selected_trait in enumerate(class_labels):
        print selected_trait

        dataset = build_data_cv(data_file, index)

        X_train, X_test, y_train, y_test = train_test_split(dataset.data, dataset.target, test_size=0.2, random_state=0)

        clf = Pipeline([('vect', CountVectorizer()),
                      ('tfidf', TfidfTransformer()),
                      ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                            alpha=1e-3, random_state=42,
                                            max_iter=5, tol=None)),
        ])

        # clf.fit(X_train, y_train)
        parameters = {'vect__ngram_range': [(1, 1), (1, 2)],
               'tfidf__use_idf': (True, False),
               'clf__alpha': (1e-2, 1e-3),
        }
        gs_clf = GridSearchCV(clf, parameters, n_jobs=-1)

        # fit the model
        gs_clf.fit(X_train, y_train)

        # simple test score
        # print clf.score(X_test, y_test)

        # 10-fold cross-validation score
        scores = cross_val_score(gs_clf, dataset.data, dataset.target, cv=10)
        print("Accuracy: %0.4f (+/- %0.4f)" % (scores.mean(), scores.std() * 2))

        # Export the classifier to a file
        joblib.dump(gs_clf, 'svm_gs_'+selected_trait+'.joblib')

        print "______________________"

And the input file is available here: https://github.com/novinfard/profiler-sentiment-analysis/blob/master/model/input_dataset/essays.csv

When I want to test my model with gcloud ml-engine local predict with this shell command:

    MODEL_DIR="gs://MY_BUCKET/"
    
    INPUT_FILE="input.json"
    
    FRAMEWORK="SCIKIT_LEARN"

   gcloud ml-engine local predict --model-dir=$MODEL_DIR \
   --json-instances $INPUT_FILE \
    
   --framework $FRAMEWORK

It raises the follwing error:

  File "/Users/XXX/google-cloud-sdk/lib/third_party/ml_sdk/cloud/ml/prediction/frameworks/sk_xg_prediction_lib.py", line 57, in predict
    "Exception during sklearn prediction: " + str(e))
cloud.ml.prediction.prediction_utils.PredictionError: Failed to run the provided model: Exception during sklearn prediction: 'dict' object has no attribute 'lower' (Error code: 2)

The input.json defined as below:

{"instances": [["the quick brown fox"],["another test"]]}

What is the issue and how it can be solved?

Answer 1

As discussed in the comments, the file which contains the JSON_INSTANCES needs to have a single JSON on each line.

In this case it would be

"the quick brown fox"
"another test"

which may look weird but it is a valid JSON.