mini_batch_handler.py 文件源码

def check_out_data_set(self):

        for set in ['train', 'valid', 'test']:
            if self.prm.data[set + "_data_name"] != None:
                file_name = self.prm.data["data_location"] + self.prm.data[set + "_data_name"]
                try:
                    d = klepto.archives.file_archive(file_name, cached=True,serialized=True)
                    d.load()
                    data_set_x = d['x']
                    data_set_y = d['y']
                    d.clear()
                    self.prm.data[set + "_set_len"] = data_set_x.__len__()
                    if data_set_x.__len__() != data_set_y.__len__():
                        raise Warning("x and y " + set + "_data_name have not the same length")
                    self.prm.data["x_size"] = data_set_x[0].shape[1]
                    if self.prm.data["x_size"] != int(self.prm.struct["net_size"][0]):
                        raise Warning(set + " data x size and net input size are unequal")
                    if self.prm.optimize['CTC'] == False:
                        self.prm.data["y_size"] = data_set_y[0].shape[1]
                        if self.prm.data["y_size"] != int(self.prm.struct["net_size"][-1]):
                            raise Warning(set + " data y size and net input size are unequal")
                    else:
                        self.prm.data["y_size"] = self.prm.struct["net_size"][-1]
                    del data_set_x
                    del data_set_y
                    self.prm.data[set + "_batch_quantity"] = int(np.trunc(self.prm.data[set + "_set_len" ]/self.prm.data["batch_size"]))
                    self.prm.data["checked_data"][set] = True
                except KeyError:
                    raise Warning("data_location or " + set + "_data_name wrong")





    ###### Create mini batches and storage them in klepto files
    ########################################