extractFeatures_org02.py 文件源码

def create_word_scores(posWords,negWords):
    # import all yuliao
    import itertools 
    from nltk.probability import FreqDist, ConditionalFreqDist 
    def count_fd(valueWords,tag):   
        Words = list(itertools.chain(*valueWords)) #????????????
        word_fd = FreqDist() #?????????
        cond_word_fd = ConditionalFreqDist() #????????????????????
        for word in Words:
            word_fd[word] += 1#word_fd.inc(word)
            cond_word_fd[tag][word]+= 1#cond_word_fd['pos'].inc(word)        
        word_count = cond_word_fd[tag].N() #???
        return word_fd,cond_word_fd,tag,word_count
    """
    def count_fd(valueWords,tag):    
        Words[0] = list(itertools.chain(*valueWords)) #????????????
        word_fd = FreqDist() #?????????
        cond_word_fd = ConditionalFreqDist() #????????????????????
        for word in Words[0]:
            word_fd[word] += 1#word_fd.inc(word)
            cond_word_fd[tag[0]][word]+= 1#cond_word_fd['pos'].inc(word)
        for word in Words[1]:
            word_fd[word] += 1#word_fd.inc(word)
            cond_word_fd[tag[1]][word]+= 1#cond_word_fd['pos'].inc(word)            
        word_count[0] = cond_word_fd[tag[0]].N() #???
        word_count[1] = cond_word_fd[tag[1]].N() #???
        return word_fd,cond_word_fd,tag,word_count[0],word_count[1]
    """
    total_word_count = count_fd(posWords,'pos')[3]+count_fd(negWords,'neg')[3]
    # get words_scores
    def all_word_scores(total_word_count,*args):#word_fd,cond_word_fd,tag,word_count):  
        #print args#count_fd(posWords,'pos')[0]
        word_fd,cond_word_fd,tag,word_count=args[0][0],args[0][1],args[0][2],args[0][3]
        #print word_fd,cond_word_fd,tag,word_count
        word_score = []
        for word, freq in word_fd.iteritems():
            score = BigramAssocMeasures.chi_sq(cond_word_fd[tag][word], (freq, word_count), total_word_count) #????????????????????????????
            word_score.append((word,score))
        return word_score
    word_scores={}
    for word_score in all_word_scores(total_word_count,count_fd(posWords,'pos')):
        word_scores.setdefault(word_score[0],word_score[1])
    for word_score in all_word_scores(total_word_count,count_fd(posWords,'neg')):
        word_scores.setdefault(word_score[0],word_score[1]) 
    return word_scores #??????????????    
    #word_scores[word] = pos_score + neg_score #?????????????????????????
#????????????????????????