data_io.py 文件源码

def __init__(self,
                 path: str,
                 vocab: Optional[Dict[str, int]],
                 add_bos: bool = False,
                 limit: Optional[int] = None) -> None:
        self.path = path
        self.vocab = vocab
        self.bos_id = None
        if vocab is not None:
            assert C.UNK_SYMBOL in vocab
            assert vocab[C.PAD_SYMBOL] == C.PAD_ID
            assert C.BOS_SYMBOL in vocab
            assert C.EOS_SYMBOL in vocab
            self.bos_id = vocab[C.BOS_SYMBOL]
        else:
            check_condition(not add_bos, "Adding a BOS symbol requires a vocabulary")
        self.add_bos = add_bos
        self.limit = limit

        self._iter = None  # type: Optional[Iterator]
        self._iterated_once = False
        self.count = 0
        self._next = None