Python批量采集360关键词指数

python+requests
Python批量查360好搜关键词指数,requests模块让爬虫更简单!脚本基础的功能都有,轮换UA,IP,verfied_ip.txt里的IP先提前用脚本验证过能正常使用!接下来爆爆爆!自己调整代码或者可以加入多线程,都比较easy,另外整体代码也比较简单,不作多说明了!

#encoding=utf-8
#@by--bigwayseo
#http://bigwayseo.com/
import requests
# import json
# import urllib
import random
# import linecache
import re
import time
#提取360关键词数据的正则表达式
recon=re.compile('{"query":"(.*?)","data":{.*?"week_index":([0-9]+),"month_index":([0-9]+)}}')
old_ip=[]

class sodata():
    def __init__(self):
        self.old_ip=[]
        self.recon=re.compile('{"query":"(.*?)","data":{.*?"week_index":([0-9]+),"month_index":([0-9]+)}}')
    #随机轮换user agent    
    def getua(self):
        head=[
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
        "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre",
        "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
        "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",
        "Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-CN) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; zh-CN) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; zh-CN) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1",
        "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8",
        "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; fr-fr) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3",
        "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418.8 (KHTML, like Gecko) Safari/419.3",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Camino/2.2.1",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0b6pre) Gecko/20100907 Firefox/4.0b6pre Camino/2.2a1pre",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20120813 Firefox/16.0",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; zh-CN) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0.112941",
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; zh-CN) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0"
        ]
        random_headers=random.choice(head)
        return random_headers
    #随机轮换ip
    def getip(self):
        for ip in open('verfied_ip.txt'):
            if ip.strip() in self.old_ip:
                continue
            else:
                ip=ip.strip()
                self.old_ip.append(ip)    
        newip=random.choice(self.old_ip)
        return newip

    def myproxies(self):       
        newip=self.getip()
        proxies={
        "http": "http://%s"%newip.strip()
        # "http": "http://182.90.30.245:80",
        # "https": "http://119.188.94.145:80",
        # "https": "http://119.97.164.48:8085",
        # "https": "http://180.153.46.112:80",
        # "http": "http://219.139.159.252:80",
        # "http": "http://42.51.155.42:80",
        # "http": "http://182.90.68.106:80"
        }
        return proxies

    def mainpass(self):  
        for word in open('360word.txt'):
            keyword=str(word)
            self.url='http://index.haosou.com/index.php?a=overviewJson&q='+keyword+'&area=全国'
            random_headers=self.getua()        
            headers={'User-Agent':'%s'%random_headers}
            # data={
            # 'a':'soIndexJson',
            # 'q':'word',
            # 'area':'全国'
            # }
            # time.sleep(1)
            #http://bigwayseo.com/
            t=1
            while t:
                try:
                    pro=self.myproxies()
                    print pro
                    html=requests.post(url=self.url,headers=headers,proxies=pro,timeout=10).content
                    if '360指数_访问异常出错' or 'verify' in html:
                        print '360指数_访问异常出错' 
                        # time.sleep(600)
                        continue
                    else:
                        t=0
                        r=re.findall(self.recon,html)
                        for i in r:
                            print '%s,周平均指数%s,月平均指数%s'%(keyword,i[1],i[2])
                except Exception,e:
                    print e
                    continue
if __name__ == '__main__':
    p=sodata()
    p.mainpass()     
  • 通过抓包工具,或者浏览器自身的F12开发者工具可以捕获:
    http://index.haosou.com/index.php?a=overviewJson&q='+keyword+'&area=全国
  • 看看这个链接对应的reponse选项,可以发现对应的json数据
  • 通过requests模块post数据到这个链接即可返回一个关于关键词的json格式数据
  • 通过json的方法提取或者正则提取等,按自己的顺手的方法来即可
  • 逻辑很简单,百度指数,因为已经加密,所以要复杂许多,但是python批查询360指数已全部实现
  • 随着时间推移,360会改变,会导致脚本需做出相应的修改,特备注脚本开发于2016年1月29日
  • 发表评论

    电子邮件地址不会被公开。