Python多线程抓取验证HTTP代理

python
Python多线程抓取验证HTTP代理:这个脚本来自于互联网,原帖不知道是谁,如果你是告诉我一声,带上链接!脚本做了一下修改,并全部加上代码作用注释,这两点加起来应该就有唯一性了,应该没有比蔸雨这个脚本更详细的注释了吧,而且目标网站是kuaidaili.com,结果发现这个站有专门做防采集的技术,悲剧了,能采集一点点吧,看来还是得找另外提供proxy站点,篼雨就不找免费的了,土豪们直接花钱买吧!

#!/usr/bin/env python
#coding:utf-8
import urllib2
import re
import threading
import time
import MySQLdb
import requests
rawProxyList = []
checkedProxyList = []
#抓取代理网站
targets = []
for i in xrange(1,50):
        # target = r"https://www.proxy.com.ru/list_%d.html" % i    #自己可换网址
        target = r"https://www.kuaidaili.com/free/inha/%d/" % i
        targets.append(target)
#抓取代理服务器正则
# p = re.compile(r'''<tr><b><td>(\d+)</td><td>(.+?)</td><td>(\d+)</td><td>(.+?)</td><td>(.+?)</td></b></tr>''')   #换网址就要换正则
p = re.compile(r'<tr>[\s|\S]*?<td>(.+?)</td>[\s|\S]*?<td>(.+?)</td>[\s|\S]*?<td>高匿名</td>[\s|\S]*?<td>.*?</td>')
#获取代理的类
class ProxyGet(threading.Thread):            #继承threading.Thread类
    def __init__(self,target):
        threading.Thread.__init__(self)      #语法就这么写的,不用想太多
        self.target = target

    def getProxy(self):
        print "代理服务器目标网站: " + self.target
        headers={
            'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Encoding':'gzip, deflate',
            'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
            'Cache-Control':'max-age=0',
            'Connection':'keep-alive',
            'Host':'www.kuaidaili.com',
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0'
        }
        req = requests.post(url=self.target,headers=headers)     
        result = req.content                  #上下两句最基础的爬虫了吧
        #print chardet.detect(result)
        matchs = p.findall(result)
#       print matchs
        for row in matchs:
            ip=row[0]
            port =row[1]
            # addr = row[4].decode("cp936").encode("utf-8")
            proxy = [ip,port]
            print proxy
            rawProxyList.append(proxy)         #把用正则提取到的内容append到列表
    def run(self):
        self.getProxy()
#检验代理的类
class ProxyCheck(threading.Thread):            #继承threading.Thread类
    def __init__(self,proxyList):
        threading.Thread.__init__(self)        #语法就这么写的,不用想太多
        self.proxyList = proxyList             #初始化
        self.timeout = 5
        self.testUrl = "https://www.baidu.com/"
        self.testStr = "030173"
    def checkProxy(self):
        #利用urllib2库的HTTPCookieProcessor对象来创建cookie处理器
        cookies = urllib2.HTTPCookieProcessor()
        for proxy in self.proxyList:
            #设置ProxyHandler代理
            proxyHandler = urllib2.ProxyHandler({"http" : r'https://%s:%s' %(proxy[0],proxy[1])}) 
            #print r'https://%s:%s' %(proxy[0],proxy[1])
            #通过cookies和proxyHandler来构建opener,顺便说下urlopen不支持cookie,http代理等功能所以、、、
            opener = urllib2.build_opener(cookies,proxyHandler)     
            #构造头部信息中的ua
            opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:22.0) Gecko/20100101 Firefox/22.0')]
            #urllib2.install_opener(opener)
            #记录下时间
            t1 = time.time()
            try:
                #req = urllib2.urlopen("https://www.baidu.com", timeout=self.timeout)
                #创建一个请求,原理同urllib2的urlopen
                req = opener.open(self.testUrl, timeout=self.timeout)
                #print "urlopen is ok...."
                result = req.read()
                #print "read html...."
                #统计出代理请求连接花费时间timeused
                timeused = time.time() - t1
                pos = result.find(self.testStr)
                #print "pos is %s" %pos
                #判断是否正常打开百度且连接速度在要求范围内
                if (pos > -1) and (timeused < 5) and baidu_title == '百度一下,你就知道':
                    checkedProxyList.append((proxy[0],proxy[1],proxy[2],timeused))
                    #print "ok ip: %s %s %s %s" %(proxy[0],proxy[1],proxy[2],timeused)
                else:
                     continue
            except Exception,e:
                #print e.message  #可以打印异常
                continue
    def run(self):
        self.checkProxy()
if __name__ == "__main__":
    getThreads = []
    checkThreads = []
    #对每个目标网站开启一个线程负责抓取代理,这段更多的是固定语法可套用
    for i in range(len(targets)):
        t = ProxyGet(targets[i])
        getThreads.append(t)
    for i in range(len(getThreads)):
        getThreads[i].start()
    for i in range(len(getThreads)):
        getThreads[i].join()
    print '.'*10+"总共抓取了%s个代理" %len(rawProxyList) +'.'*10

    #先举个简单的例子,单独copy出来打印结果一下就懂,不懂可以直接套就是
    #这段代码相当于把l这个列表分成5份
    # l=range(1,100)
    # for i in range(5):
    #     print l[((len(l)+4)/5) * i:((len(l)+4)/5) * (i+1)]

    #开启20个线程负责校验,将抓取到的代理分成20份,每个线程校验一份
    for i in range(20):
        t = ProxyCheck(rawProxyList[((len(rawProxyList)+19)/20) * i:((len(rawProxyList)+19)/20) * (i+1)])
        checkThreads.append(t)
    for i in range(len(checkThreads)):
        checkThreads[i].start()
    for i in range(len(checkThreads)):
        checkThreads[i].join()
    print '.'*10+"总共有%s个代理通过校验" %len(checkedProxyList) +'.'*10
    #插入数据库,表结构自己创建,四个字段ip,port,speed,address
    #下面其实更多是mysql的东西了,直接导出到txt或者csv都行,下面导出到txt格式为例
    f= open("hege_daili.txt",'w+')
    for proxy in checkedProxyList:
        print "qualified: %s\t%s" % (proxy[0],proxy[1])
        f.write(proxy[0]+"\n")   #换行写入
    f.close()
# def db_insert(insert_list):
#     try:
#         conn = MySQLdb.connect(host="localhost", user="root", passwd="admin",db="m_common",charset='utf8')
#         cursor = conn.cursor()   
#         cursor.execute('delete from proxy')                  #删除表中的数据
#         cursor.execute('alter table proxy AUTO_INCREMENT=1') #加入一条新数据时,id从1开始
#         #批量插入到数据库
#         cursor.executemany("INSERT INTO proxy(ip,port,speed,address) VALUES (%s,%s,%s,%s)",insert_list)
#         conn.commit()
#         cursor.close()
#         conn.close()
#     except MySQLdb.Error,e:
#         print "Mysql Error %d: %s" % (e.args[0], e.args[1])
# #代理排序持久化
# proxy_ok = []
# f= open("proxy_list.txt",'w+')
# for proxy in sorted(checkedProxyList,cmp=lambda x,y:cmp(x[3],y[3])):
#     if proxy[3] < 8:
#         #print "checked proxy is: %s:%s\t%s\t%s" %(proxy[0],proxy[1],proxy[2],proxy[3])
#         proxy_ok.append((proxy[0],proxy[1],proxy[3],proxy[2]))
#         f.write("%s:%s\t%s\t%s\n"%(proxy[0],proxy[1],proxy[2],proxy[3]))
# f.close()
# db_insert(proxy_ok)

Leave a Comment