python比较两个文本的相似性

使用余弦定理来实现对两个文本相似性的比较：seo中的应用之一判断采集内容的重复度，决定是否入库上线；seo应用二涨工资；seo应用三装逼！
余弦公式

python代码：

#!/usr/bin/env python
# -*- coding: utf-8 -*
 
import re
from math import sqrt
 
def file_reader(filename,filename2):
    file_words = {}
    ignore_list = [u'的',u'了',u'和',u'呢',u'啊',u'哦',u'恩',u'嗯',u'吧'];
    #ignore_list = [];
    accepted_chars = re.compile(ur"[\u4E00-\u9FA5]+")
 
    file_object = open(filename)
 
    try:
        all_the_text = file_object.read()
        for s in all_the_text.decode('gb2312'):
            if accepted_chars.match(s) and s not in ignore_list:
                if s not in file_words.keys():
                    file_words[s] = [1,0]
                else:
                    file_words[s][0] += 1
    finally:
        file_object.close()
 
    file_object2 = open(filename2)
 
    try:
        all_the_text = file_object2.read()
        for s in all_the_text.decode('gb2312'):
            if accepted_chars.match(s) and s not in ignore_list:
                if s not in file_words.keys():
                    file_words[s] = [0,1]
                else:
                    file_words[s][1] += 1
    finally:
        file_object2.close()
 
    sum_2 = 0
    sum_file1 = 0
    sum_file2 = 0
    for word in file_words.values():
        sum_2 += word[0]*word[1]
        sum_file1 += word[0]**2
        sum_file2 += word[1]**2
 
    rate = sum_2/(sqrt(sum_file1*sum_file2))
    print 'rate: '
    print rate
 
file_reader('thefile.txt','thefile2.txt')

此算法非常简单，但精确度不够高。但也可以通过简单的优化得到更好的结果，如以上代码就简单的加入了过渡功能。若再加入一个分词库，通过分词来比较中文的相似性，那就更好了。下面代码就用到了分词库实现文本的比较，效果提升明显。

使用分词以及余弦定理来比较两份文档的相似性的代码：

#!/usr/bin/env python
# -*- coding: utf-8 -*
 
import re
from math import sqrt
#You have to install the python lib
import jieba
 
def file_reader(filename,filename2):
    file_words = {}
    ignore_list = [u'的',u'了',u'和',u'呢',u'啊',u'哦',u'恩',u'嗯',u'吧'];
    accepted_chars = re.compile(ur"[\u4E00-\u9FA5]+")
 
    file_object = open(filename)
 
    try:
        all_the_text = file_object.read()
        seg_list = jieba.cut(all_the_text, cut_all=True)
        #print "/ ".join(seg_list)
        for s in seg_list:
            if accepted_chars.match(s) and s not in ignore_list:
                if s not in file_words.keys():
                    file_words[s] = [1,0]
                else:
                    file_words[s][0] += 1
    finally:
        file_object.close()
 
    file_object2 = open(filename2)
 
    try:
        all_the_text = file_object2.read()
        seg_list = jieba.cut(all_the_text, cut_all=True)
        for s in seg_list:
            if accepted_chars.match(s) and s not in ignore_list:
                if s not in file_words.keys():
                    file_words[s] = [0,1]
                else:
                    file_words[s][1] += 1
    finally:
        file_object2.close()
 
    sum_2 = 0
    sum_file1 = 0
    sum_file2 = 0
    for word in file_words.values():
        sum_2 += word[0]*word[1]
        sum_file1 += word[0]**2
        sum_file2 += word[1]**2
 
    rate = sum_2/(sqrt(sum_file1*sum_file2))
    print 'rate: '
    print rate
 
 
 
file_reader('thefile.txt','thefile2.txt')

原理解释参考此文章：https://my.oschina.net/BreathL/blog/42477

其它：https://pypi.python.org/pypi/NearDuplicatesDetection/0.2.0

这个python库号称可以比较两文本的相似度，但本人使用中文文件测试结果很不理想。不信大家可以试试。

本文：余争 https://my.oschina.net/dancing/home