util.py 文件源码

def iterate_from(self, start_tok):
        piecenum = bisect.bisect_right(self._offsets, start_tok)-1

        while piecenum < len(self._pieces):
            offset = self._offsets[piecenum]
            piece = self._pieces[piecenum]

            # If we've got another piece open, close it first.
            if self._open_piece is not piece:
                if self._open_piece is not None:
                    self._open_piece.close()
                self._open_piece = piece

            # Get everything we can from this piece.
            for tok in piece.iterate_from(max(0, start_tok-offset)):
                yield tok

            # Update the offset table.
            if piecenum+1 == len(self._offsets):
                self._offsets.append(self._offsets[-1] + len(piece))

            # Move on to the next piece.
            piecenum += 1