Python批量查360好搜关键词指数，requests模块让爬虫更简单！脚本基础的功能都有，轮换UA，IP，verfied_ip.txt里的IP先提前用脚本验证过能正常使用！接下来爆爆爆！自己调整代码或者可以加入多线程，都比较easy，另外整体代码也比较简单，不作多说明了！
#encoding=utf-8
#@by--bigwayseo
#https://bigwayseo.com/
import requests
# import json
# import urllib
import random
# import linecache
import re
import time
#提取360关键词数据的正则表达式
recon=re.compile('{"query":"(.*?)","data":{.*?"week_index":([0-9]+),"month_index":([0-9]+)}}')
old_ip=[]
class sodata():
def __init__(self):
self.old_ip=[]
self.recon=re.compile('{"query":"(.*?)","data":{.*?"week_index":([0-9]+),"month_index":([0-9]+)}}')
#随机轮换user agent
def getua(self):
head=[
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b4pre) Gecko/20100815 Minefield/4.0b4pre",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12",
"Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-CN) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; zh-CN) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; zh-CN) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; fr-fr) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418.8 (KHTML, like Gecko) Safari/419.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Camino/2.2.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0b6pre) Gecko/20100907 Firefox/4.0b6pre Camino/2.2a1pre",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20120813 Firefox/16.0",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; zh-CN) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0.112941",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; zh-CN) AppleWebKit/528.16 (KHTML, like Gecko, Safari/528.16) OmniWeb/v622.8.0"
]
random_headers=random.choice(head)
return random_headers
#随机轮换ip
def getip(self):
for ip in open('verfied_ip.txt'):
if ip.strip() in self.old_ip:
continue
else:
ip=ip.strip()
self.old_ip.append(ip)
newip=random.choice(self.old_ip)
return newip
def myproxies(self):
newip=self.getip()
proxies={
"http": "https://%s"%newip.strip()
# "http": "https://182.90.30.245:80",
# "https": "https://119.188.94.145:80",
# "https": "https://119.97.164.48:8085",
# "https": "https://180.153.46.112:80",
# "http": "https://219.139.159.252:80",
# "http": "https://42.51.155.42:80",
# "http": "https://182.90.68.106:80"
}
return proxies
def mainpass(self):
for word in open('360word.txt'):
keyword=str(word)
self.url='https://index.haosou.com/index.php?a=overviewJson&q='+keyword+'&area=全国'
random_headers=self.getua()
headers={'User-Agent':'%s'%random_headers}
# data={
# 'a':'soIndexJson',
# 'q':'word',
# 'area':'全国'
# }
# time.sleep(1)
#https://bigwayseo.com/
t=1
while t:
try:
pro=self.myproxies()
print pro
html=requests.post(url=self.url,headers=headers,proxies=pro,timeout=10).content
if '360指数_访问异常出错' or 'verify' in html:
print '360指数_访问异常出错'
# time.sleep(600)
continue
else:
t=0
r=re.findall(self.recon,html)
for i in r:
print '%s,周平均指数%s，月平均指数%s'%(keyword,i[1],i[2])
except Exception,e:
print e
continue
if __name__ == '__main__':
p=sodata()
p.mainpass()
https://index.haosou.com/index.php?a=overviewJson&q='+keyword+'&area=全国