modularDistFW.py 文件源码

def safeWrite(rdd,outputfile,dvrdump=False):
    """Save the rdd in the given directory.

    Keyword arguments:
    --rdd: given rdd to be saved
    --outputfile: desired directory to save rdd
    """
    if os.path.isfile(outputfile):
       os.remove(outputfile)    
    elif os.path.isdir(outputfile):
       shutil.rmtree(outputfile)    

    if dvrdump:
    rdd_list = rdd.collect()
    with open(outputfile,'wb') as f:
        count = 0
        for item in rdd_list:
            f.write(str(item))   
            count = count+1
            if count < len(rdd_list):
            f.write("\n")  
    else:
       rdd.saveAsTextFile(outputfile)