pysos.py 文件源码

def csv2sos(path, keys=None, encoding=None, dialect=None):

    if not encoding:
        encoding = detectEncoding(path)
        print('Detected encoding: %s' % encoding)

    csvfile = open(path, 'rt', encoding=encoding)
    sosfile = open(path + '.sos', 'wt', encoding='utf8')

    if not dialect:
        dialect = csv.Sniffer().sniff(csvfile.read(1024*1024), delimiters=[';','\t',','])
        print('Detected csv dialect: %s' % dialect)

    csvfile.seek(0)
    reader = csv.DictReader(csvfile, dialect=dialect)
    i = 0
    for row in reader:
        sosfile.write(str(i) + '\t' + json.dumps(row, ensure_ascii=False) + '\n')
        i += 1
        if i % 100000 == 0:
            print("%10d items converted" % i)

    csvfile.close()    
    sosfile.close()