utils.py 文件源码

def encode(self, text):
        """Support batch or single str.

        Args:
            text (str or list of str): texts to convert.

        Returns:
            torch.IntTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
            torch.IntTensor [n]: length of each text.
        """
        length = []
        result = []
        for item in text:            
            if self.is_chinese(item):          
               item = unicode(item,'utf-8')
            length.append(len(item))
            for char in item:               
                index = self.dict[char]                
                result.append(index)
        text = result
        return (torch.IntTensor(text), torch.IntTensor(length))