pacci.py 文件源码

def getSparkConf(mode="mesos", node=0):
    """
    get the spark configuration according to the setting
    :param mode:
    :param node:
    :return:
    """
    global options

    '''
    get spark configuration
    '''
    sconf=SparkConf()

    '''
    set spark configuration
    '''
    sconf.setAppName("%s" % (str(options)))

    # set run mode, now only support spark standalone and mesos coarse mode
    if (mode == 'spark'):
        sconf.setMaster(Setting.SPARK_STANDALONE_URL)
    elif (mode == 'mesos'):
        sconf.setMaster(Setting.MESOS_COARSE_URL)
        # sconf.set("spark.mesos.coarse", "false")
        sconf.set("spark.mesos.coarse", "true")
        sconf.set("spark.mesos.executor.home", Setting.SPARK_HOME)
    else:
        print("****unknown mode")
        exit(0)

    # set core limit if need
    if (0 >= node):
        print "****Spark:no cores max"
    else:
        sconf.set("spark.cores.max", "%d" % (options.cpu * node))

    return sconf