preprocessing.py 文件源码

def generate_icamodel(train_vocabulary='./vocabulary/vocabulary_nv_4w.txt',model_path='./model/ICA/ica_ourword2vec.model'):
    train_vocab =[v.strip() for v in open(train_vocabulary,'r').readlines()]
    train_sample = np.zeros([len(train_vocab),300])
    for i,v in enumerate(train_vocab):
        word = v.split(' ')[0]
        try:
            train_sample[i]= word2vec_model[word]
        except:
            print word
    ica = FastICA(300,max_iter=800)
    ica.fit(train_sample)
    joblib.dump(ica,model_path)

    pass