htmlcleaner.py 文件源码

def cleanUnicode(string):   
    try:
        try:
            #string = str(string)
            if isinstance(string, unicode):
                unicode_replaced_str = string.decode('utf-8')
            elif isinstance(string, str):
                unicode_replaced_str = string.decode('utf-8')
            import unidecode
            unicode_replaced_str = unidecode.unidecode(unicode_replaced_str)
            string = unicode_replaced_str

        except:
            pass

        fixed_string = unicodedata.normalize('NFKD', string).encode('ascii', 'ignore' )    
        return fixed_string
    except:
        return string

#interface: