split_tsv_per_user.py 文件源码

def main(filename):
    """
    Will split the big input file of 1000 user into 1000 files of 1 user
    """
    with open(filename) as origfile:
        dir = os.path.dirname(filename)
        csv_reader = csv.reader(origfile, delimiter='\t')

        # Fixes a bug:
        # http://stackoverflow.com/questions/15063936/csv-error-field-larger-than-field-limit-131072
        csv.field_size_limit(sys.maxsize)
        lastuser = None
        for row in csv_reader:
            if lastuser != row[0]:
                print(row[0])
                lastuser = row[0]
            with open(os.path.join(dir, "split", lastuser + ".tsv"), "a") as f:
                f.write("{}\n".format("\t".join(row)))