preprocessing.py 文件源码

def preprocess(rawdatapath):
    rawfacts = get_raw(os.path.join(rawdatapath, "facts.json"))
    rawmetadata = get_raw(os.path.join(rawdatapath, "metadata.json"))
    parsed_facts = rawfacts.join(pd.DataFrame(rawfacts["_source"].to_dict()).T).drop("_source", axis=1)
    parsed_metadata = rawmetadata.join(pd.DataFrame(rawmetadata["_source"].to_dict()).T).drop("_source", axis=1)
    parsed_metadata.rename(columns={"title":"articleTitle"}, inplace=True)
    clean(parsed_facts)
    clean(parsed_metadata)
    parsed_metadata = parsed_metadata.join(pd.DataFrame(parsed_metadata["journalInfo"].to_dict()).T).drop("journalInfo", axis=1)
    clean(parsed_metadata)
    parsed_metadata = parsed_metadata.join(pd.DataFrame(parsed_metadata["journal"].to_dict()).T).drop("journal", axis=1)
    clean(parsed_metadata)
    df = pd.merge(parsed_facts, parsed_metadata, how="inner", on="cprojectID", suffixes=('_fact', '_meta'))
    df.rename(columns={"title":"journalTitle"}, inplace=True)
    df["sourcedict"] = get_dictionary(df)
    df["term"] = df["term"].map(str.lower)
    df["wikidataID"] = get_wikidataIDs(df)
    df.drop_duplicates("_id_fact", inplace=True)
    return df