k_means_kdd.py 文件源码

def split(p):
    output = os.path.join(get_data_home(), "kddcup.parq")
    if not os.path.exists(output):

        dtype = {
            1: 'category',
            2: 'category',
            3: 'category',
            41: 'category',
        }

        df = pd.read_csv(p, header=None, dtype=dtype)
        cat_cols = df.select_dtypes(include=['category']).columns
        df[cat_cols] = df[cat_cols].apply(lambda col: col.cat.codes)
        df.columns = list(string.ascii_letters[:len(df.columns)])

        ddf = dd.from_pandas(df, npartitions=16)
        ddf.to_parquet(output)

    return output