fastq_splitter.py 文件源码

def upload_split(file_name):
    global parser_result

    base_file_name = file_name.split("/")[-1]

    if parser_result.output_dir.startswith("s3://"):
        bucket_name, folder_name = parser_result.output_dir[5:].split("/", 1)
        key_name = "{}/{}".format(folder_name.rstrip("/"), base_file_name)

        s3 = boto3.resource("s3", region_name=parser_result.s3_region)
        s3.Bucket(bucket_name).upload_file(file_name, key_name)

        os.remove(file_name)

    else:
        subprocess.call(["hdfs", "dfs", "-mkdir", "-p", parser_result.output_dir])
        subprocess.call(["hdfs", "dfs", "-put", file_name, parser_result.output_dir])

    print(base_file_name)