models.py 文件源码

def reload(self):
        self.prepare_log()
        self.log("Loading dataset " + self.text_id + "...")

        Term.objects.filter(dataset=self).delete()
        Document.objects.filter(dataset=self).delete()
        Modality.objects.filter(dataset=self).delete()
        from models.models import ArtmModel
        ArtmModel.objects.filter(dataset=self).delete()

        try:
            meta_file = os.path.join(self.get_folder(), "meta", "meta.json")
            with open(meta_file) as f:
                self.docs_info = json.load(f)
        except BaseException as ex:
            self.log("WARNING! Wasn't able to load meta.json")
            self.log(str(ex))
            self.time_provided = False
            self.docs_info = {}

        try:
            preprocessing_params = json.loads(self.preprocessing_params)
            self.log("Preprocessing params:" + str(preprocessing_params))
        except BaseException:
            preprocessing_params = {}
            self.log("Warning! Failed to load preprocessing parameters.")

        # Preprocessing
        custom_vocab = False
        if "parse" in preprocessing_params:
            self.preprocess_parse(preprocessing_params["parse"])
        if "filter" in preprocessing_params:
            self.preprocess_filter(preprocessing_params["filter"])
            custom_vocab = True
        if "custom_vocab" in preprocessing_params and preprocessing_params[
                "custom_vocab"]:
            self.log("Will use custom vocab.txt")
            custom_vocab = True

        self.create_batches()
        self.gather_dictionary(custom_vocab=custom_vocab)
        self.load_documents()

        self.log("Loaded " + str(self.documents_count) + " documents.")

        # Creating folder for models
        model_path = os.path.join(
            settings.DATA_DIR, "datasets", self.text_id, "models")
        if not os.path.exists(model_path):
            os.makedirs(model_path)

        self.log("Dataset " + self.text_id + " loaded.")
        self.creation_time = datetime.now()
        self.status = 0
        self.save()