Error message when launching PySpark from Jupyter

Question

This same approach to run Apache spark on Jupyter used to work, but now it is throwing Exception: Java gateway process exited before sending the driver its port number

Here is the configuration in Jupyter notebook which was working previously.

import os
import sys

spark_home = os.environ.get('SPARK_HOME', None)
print(spark_home)
spark_home= spark_home+"/python"
sys.path.insert(0, spark_home)
sys.path.insert(0, os.path.join(spark_home, 'python/lib/py4j-0.8.2.1-  src.zip'))

filename = os.path.join(spark_home, 'pyspark/shell.py')
print(filename)
exec(compile(open(filename, "rb").read(), filename, 'exec'))

spark_release_file = spark_home + "/RELEASE"

if os.path.exists(spark_release_file) and "Spark 1.5" in   open(spark_release_file).read():
pyspark_submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "")
if not "pyspark-shell" in pyspark_submit_args: 
    pyspark_submit_args += " pyspark-shell"
    os.environ["PYSPARK_SUBMIT_ARGS"] = pyspark_submit_args

exec statement is throwing the exception.

Please let me know what I am doing wrong.

Answer 1

You need to invoke execute statement in your if statement

    import os
    import sys

    spark_home = os.environ.get('SPARK_HOME', None)
    print(spark_home)
    spark_home= spark_home+"/python"
    sys.path.insert(0, spark_home)
    sys.path.insert(0, os.path.join(spark_home, 'python/lib/py4j-0.8.2.1-  src.zip'))

    filename = os.path.join(spark_home, 'pyspark/shell.py')
    print(filename)


    spark_release_file = spark_home + "/RELEASE"

    if os.path.exists(spark_release_file) and "Spark 1.5" in   open(spark_release_file).read():
        argsstr= "--master yarn  pyspark-shell ";
        pyspark_submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", argsstr)
        if not "pyspark-shell" in pyspark_submit_args: 
            pyspark_submit_args += " pyspark-shell"
            os.environ["PYSPARK_SUBMIT_ARGS"] = pyspark_submit_args
        exec(compile(open(filename, "rb").read(), filename, 'exec'))