AbstractNetPreprocessor.py 文件源码

def extra_processing(self):

        data_dir = BASE_DIR + "/Data/Generated_Data/Sentences_And_SummaryBool/Abstract_Neg/AbstractNet/data.pkl"
        write_dir = BASE_DIR + \
            "/Data/Generated_Data/Sentences_And_SummaryBool/Abstract_Neg/AbstractNet/abstractnet_data.pkl"

        print("----> Reading data...")
        t = time.time()
        data = useful_functions.load_pickled_object(data_dir)
        print("----> Done, took ", time.time() - t, " seconds")

        print("----> Beginning processing...")
        t = time.time()
        self.start_time = t
        new_data = self.pool2.map(self.process_item, data)
        # new_data = np.concatenate(new_data, axis=0)
        print("----> Done, took ", (time.time() - t) / 60, " minutes")

        useful_functions.pickle_list(new_data, write_dir)