目录
新发地http://www.xinfadi.com.cn/priceDetail.html
https://www.zbj.com/search/f?type=new&kw=saas
selenium——利用鼠标移动操作巧妙爬取highcharts动态图表
简介
爬虫在使用场景中的分类
反爬机制
反反爬策略
robots.txt协议
基本概念
http协议
常用请求头信息
常用响应头信息
https协议
加密方式
requests模块
作用
如何使用
环境安装
pip install requests
实战编码
爬取搜狗首页的页面数据
import requests
if __name__=="__main__":
#指定url
url='https://www.sogou.com/'
#发起请求
response=requests.get(url=url)
#获取响应数据,text返回的是字符串形式的响应数据
page_text=response.text
print(page_text)
#持久化存储
with open('./sogou.html','w',encoding='utf-8') as fp:
fp.write(page_text)简易网页采集器
import requests
#UA检测：门户网站的服务器会检测对应请求的载体身份标识，如果检测到请求的载体身份标识为某一款浏览器
#说明该请求是一个正常的请求，但是，如果检测到请求的载体身份标识不是基于某一款浏览器的，则表示爬虫
#UA:User-Agent(请求载体的身份标识)
if __name__=="__main__":
headers={
"User-Agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54'
}
url='https://www.sogou.com/web'
#处理url携带的参数：封装到字典中
kw=input('enter a word')
param={
'query':kw
}
#对指定的url发起的请求对应的url是携带参数的，并且请求过程中处理了参数
response=requests.get(url=url,params=param,headers=headers)
page_text=response.text
fileName=kw+'.html'
with open(fileName,'w',encoding='utf-8') as fp:
fp.write(page_text)
print(fileName,'保存成功')破解百度翻译
import requests,json
#UA检测：门户网站的服务器会检测对应请求的载体身份标识，如果检测到请求的载体身份标识为某一款浏览器
#说明该请求是一个正常的请求，但是，如果检测到请求的载体身份标识不是基于某一款浏览器的，则表示爬虫
#UA:User-Agent(请求载体的身份标识)
if __name__=="__main__":
headers={
"User-Agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54'}
post_url="https://fanyi.baidu.com/sug"
#post请求参数处理（同get请求一致）
word=input('enter a word')
data={
'kw':word
}
response=requests.post(url=post_url,data=data,headers=headers)
#获取响应数据：json()方法返回的是obj（如果确认响应数据是json类型的，才可以使用json（））
dic_obj=response.json()
print(response.text)
print(dic_obj)
#持久化存储
fileName=word+'.json'
with open(fileName,'w',encoding='utf-8') as fp:
json.dump(dic_obj, fp=fp, ensure_ascii=False) # 中文不能使用ascii编码
print("OVER!!")爬取豆瓣电影分类排行榜
import requests
import json
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54'}
url='https://movie.douban.com/j/chart/top_list?'
param={
'type':'24',
'interval_id':'100:90',
'action': '',
'start':'40',#从库中的第几部电影去取
'limit':'20',#一次取出的个数
}
response=requests.get(url=url,params=param,headers=headers)
response_data=response.json()
print(response_data)
with open("./小说详情页",'w',encoding='utf-8') as fp:
json.dump(response_data,fp=fp,ensure_ascii=False)
爬取肯德基餐厅查询
肯德基餐厅信息查询 (kfc.com.cn)
import requests,json
headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54"}
url="http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword"
param={
"cname":"",
"pid":"",
"keyword": "北京",
"pageIndex":"1",
"pageSize": "10"
}
response=requests.post(url=url,data=param,headers=headers)
page_text=response.text
print(page_text)
with open("./肯德基.txt",'w',encoding='utf-8') as fp:
fp.write(page_text)聚焦爬虫
简介
数据解析分类：
数据解析原理概述
正则表达式
豆瓣电影Top250
https://movie.douban.com/top250
import requests
import re
url="https://movie.douban.com/top250"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54"}
response=requests.get(url=url,headers=headers)
content=response.text
# print(content)
#解析数据
#预加载正则表达式
#re.S用法 re.S的作用： 不使用re.S时，则只在每一行内进行匹配，如果存在一行没有，就换下一行重新开始，使用re.S参数以后，正则表达式会将这个字符串看做整体，在整体中进行匹配，一般在爬虫项目中会经常用到。
obj=re.compile(r'<li>.*?<div class="item">.*?<span class="title">(?P<name>.*?)</span>',re.S)
result=obj.finditer(content)
for i in result:
print(i.group("name"))import requests
import re
import csv
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54"}
print("请输入您要爬取的页数")
num=input()
num=int(num)
for i in range(num):
i=i*25
print(i)
url = "https://movie.douban.com/top250?start={}".format(i)
print(url)
response=requests.get(url=url,headers=headers)
content=response.text
# print(content)
#解析数据
#预加载正则表达式
#re.S用法 re.S的作用： 不使用re.S时，则只在每一行内进行匹配，如果存在一行没有，就换下一行重新开始，使用re.S参数以后，正则表达式会将这个字符串看做整体，在整体中进行匹配，一般在爬虫项目中会经常用到。
obj=re.compile(r'<li>.*?<div class="item">.*?<span class="title">(?P<name>.*?)'
r'</span>.*?<p class="">.*?<br>(?P<year>.*?)
r'.*?<span class="rating_num" property="v:average">(?P<average>.*?)</span>',re.S)
result=obj.finditer(content)
#for i in result:
# print(i.group("name"))
# print(i.group("year").strip())
# print(i.group("average"))
# dic=i.groupdict()
# dic['year']=dic['year'].strip()
with open("data.csv","a",encoding="utf-8") as f:
writer=csv.writer(f)
for i in result:
dic=i.groupdict()
dic['year']=dic['year'].strip()
writer.writerow(dic.values())
电影天堂https://www.dytt89.com/
#1.定位到2023必看片
#2.从2023必看篇中提取到页面的链接地址
#3.请求子页面的链接地址，拿到我们想要的下载地址
import re
import requests
domain="https://www.dytt89.com/"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54"}
resp=requests.get(url=domain,headers=headers,verify=False)#去掉安全验证
resp.encoding="gb2312"#指定字符集
obj1=re.compile(r"2023必看热片.*?<ul>(?P<ul>.*?)</ul>",re.S)
result1=obj1.finditer(resp.text)
obj2=re.compile(r"<li><a href='(?P<li>.*?)'.*?</li>",re.S)
for it in result1:
ul=it.group('ul')
# print(ul)
#提取子链接
result2=obj2.finditer(ul)
for i in result2:
li=i.group('li')
print(li)
Bs4解析
新发地http://www.xinfadi.com.cn/priceDetail.html
唯美壁纸
https://www.umei.cc/bizhitupian/weimeibizhi/
#1.拿到主页面的源代码，然后提取到子页面的链接地址，href
#2.通过href拿到子页面的内容，从子页面中找到图片的下载地址 img->src
#3.下载地址
import requests
from bs4 import BeautifulSoup
url="https://www.umei.cc/bizhitupian/weimeibizhi"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.54"}
resp=requests.get(url=url,headers=headers)
resp.encoding='utf-8'
page=resp.text
# print(page)
main_page = BeautifulSoup(page, "html.parser")
div_list = main_page.find_all("div", class_="img")
visited_links = []
for div in div_list:
alist = div.find_all("a")
for a in alist:
href="https://www.umei.cc/"+a.get('href')
if href not in visited_links: # 检查是否已经访问过
visited_links.append(href)
# print(href)
#拿到子页面的源代码
child_page_resp=requests.get(url=href,headers=headers)
child_page_resp.encoding='utf-8'
child_page_text=child_page_resp.text
#从子页面中拿到图片的下载路径
second_page=BeautifulSoup(child_page_text,"html.parser")
div_list1=second_page.find("div",class_="big-pic").find("img")
src=div_list1.get('src')
#下载图片
img_resp=requests.get(src)
img_name=src.split("/")[-1]#拿到url中的最后一个/以后的内容
# img_resp.content
with open(img_name,mode="wb") as f:
f.write(img_resp.content)#图片内容写入文件
print("over!!!",img_name)
print("all over")
Xpath
猪八戒网
https://www.zbj.com/search/f?type=new&kw=saas
#拿到页面源代码
#提取和解析数据
import requests
from lxml import etree
url="https://beijing.zbj.com/search/f?type=newsaas13374"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.39"}
resp=requests.get(url=url,headers=headers)
page_content=resp.text
# print(page_content)
#解析
html=etree.HTML(page_content)
#拿到每一个服务商的div
divs=html.xpath('//*[@id="__layout"]/div/div[3]/div/div[4]/div/div[2]/div[1]/div')
for div in divs:
price_list = div.xpath('./div/div[3]/div[1]/span/text()')
if price_list:
price = price_list[0].strip("¥")
else:
print("Unable to extract price information for this provider.")
continue
title = div.xpath('./div/div[3]/div[2]/a/text()')
print(price)
print(title)我遇到的问题：
requests进阶概述
处理cookie登录小说网
https://www.17k.com/
https://passport.17k.com/ck/user/login
https://user.17k.com/ck/author/shelf?page=1&appKey=2406394919
import requests
#会话
session=requests.session()
#登录
url="https://passport.17k.com/ck/user/login"
data = {
'loginName': '小木aaaaaa',
'password': '15175857475a'
}
session.post(url,data=data)
# print(resp.text)
# print(resp.cookies)
#2.拿到书架上的数据
resp=session.get('https://user.17k.com/ck/author/shelf?page=12406394919')
print(resp.json())防盗链抓取梨视频
https://www.pearvideo/
import requests
#1.拿到contId
#2.拿到videoStatus返回的json.->srcURL
#3.srcURL里面的内容进行修正
#4.下载视频
url="https://www.pearvideo.com/video_1741775"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.39",
#防盗链:溯源，当前请求的上一级是谁
"Referer":url
}
contId=url.split("_")[1]
videoStatueUrl=f"https://www.pearvideo.com/videoStatus.jsp?contId={contId}0.7473506266397887"
resp=requests.get(url=videoStatueUrl,headers=headers)
dic=resp.json()
srcUrl=dic['videoInfo']['videos']["srcUrl"]
systemTime=dic['systemTime']
srcUrl=srcUrl.replace(systemTime,f"cont-{contId}")
print(srcUrl)
#下载视频
with open("a.mp4",mode="wb") as f:
f.write(requests.get(srcUrl).content)
#真实链接：https://video.pearvideo.com/mp4/adshort/20210916/cont-1741775-15768660_adpkg-ad_hd.mp4
#得到链接：https://video.pearvideo.com/mp4/adshort/20210916/1681457911134-15768660_adpkg-ad_hd.mp4代理
https://www.zdaye.com/Free/
import requests
#61.160.212.99:8111
proxies={
"https":"https://223.108.49.50:3128"
}
resp=requests.get("https://www.baidu.com",proxies=proxies)
resp.encoding="utf-8"
print(resp.text)遇到的问题：代理不能用
抓取网易云音乐评论信息
https://music.163.com/#/playlist?id=2821115454
import requests, json
from Crypto.Cipher import AES
from base64 import b64encode
url = "https://music.163.com/weapi/comment/resource/comments/get?csrf_token="
# 请求方式post
data = {
"csrf_token": "",
"cursor": "-1",
"offset": "0",
"orderType": "1",
"pageNo": "1",
"pageSize": "20",
"rid": "A_PL_0_2821115454",
"threadId": "A_PL_0_2821115454"
}
# 处理加密过程
"""" function a(a) {#随机的16位字符串
var d, e, b = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", c = "";
for (d = 0; a > d; d += 1)#循环16次
e = Math.random() * b.length,#随机数
e = Math.floor(e),#取整
c += b.charAt(e);#取字符串中的xxx位
return c
}
function b(a, b) {
var c = CryptoJS.enc.Utf8.parse(b)
, d = CryptoJS.enc.Utf8.parse("0102030405060708")
, e = CryptoJS.enc.Utf8.parse(a)
, f = CryptoJS.AES.encrypt(e, c, {
iv: d,
mode: CryptoJS.mode.CBC
});
return f.toString()
}
function c(a, b, c) {#c里面不产生随机数
var d, e;
return setMaxDigits(131),
d = new RSAKeyPair(b,"",c),
e = encryptedString(d, a)
}
function d(d, e, f, g) {
#确定里面的每个参数
#d:数据 e:010001
#f:'00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7'
#g:'0CoJUm6Qyw8W8jud'
var h = {}
, i = a(16);#i就是一个16位的随机值,把
return h.encText = b(d, g),
h.encText = b(h.encText, i),#返回的就是params
h.encSecKey = c(i, e, f),#得到的是encSecKey,
h"""
encSecKey = "7b6b13e32326f18111136e910ac9162d0bbe63af52dcaf11f9052085a58aaa5f4313d178736dc0934d04c806e73436f11c9fa3a9f6858bc86eae5c71d39347c2d7745ff30e3af7e5ae947605e4ec80131e0ed48e7a6464b1f55f274b7131ece470f5b0b454939a044020e918f140f17c490b0646365e65c33e21be6a08a6a37f"
f = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7'
g = '0CoJUm6Qyw8W8jud'
e = "010001 "
i = "RGjaSLQLvktUK761"
def get_encSecKey():
return encSecKey
def get_params(data):
first = enc_params(data, g)
second = enc_params(first, i)
return second
def to_16(data):
pad=16-len(data)%16
data+=chr(pad)*pad
return data
def enc_params(data, key):
iv = "0102030405060708"
data=to_16(data)
aes = AES.new(key=key.encode("utf-8"), IV=iv.encode('utf-8'), mode=AES.MODE_CBC)
bs = aes.encrypt(data.encode("utf-8"))#加密，加密的内容的长度必须是16的倍数
return str(b64encode(bs), "utf-8") # 转化成字符串返回
# 处理加密过程
resp = requests.post(url, data={
"params": get_params(json.dumps(data)),
"encSecKey": get_encSecKey()
})
print(resp.text)给爬虫提速
多线程
多进程
线程池
#线程池：一次性开辟一些线程，我们用户直接给线程池子提交任务，线程任务的调度交给线程池来完成
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
def fn(name):
for i in range(1000):
print(name,i)
if __name__=='__main__':
#创建线程池
with ThreadPoolExecutor(50) as t:
for i in range(100):
t.submit(fn,name=f"线程{i}")
#等待线程池中的任务全部执行完毕，才继续执行（守护）
print("123")新发地
http://www.xinfadi.com.cn/priceDetail.html
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
import requests,json
#1.如何提取单个页面的数据
#2.上线程池，多个页面同时抓取
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.39",
}
def download_one_page(url,i):
#拿到页面源代码
data={
"limit": 20,
"current": i,
"pubDateStartTime":"",
"pubDateEndTime":"",
"prodPcatid":"",
"prodCatid":"",
"prodName":"",
}
resp=requests.post(url,data=data,headers=headers)
content=resp.json()
print(content)
if __name__ == '__main__':
url="http://www.xinfadi.com.cn/getPriceData.html"
with ThreadPoolExecutor(50) as t:
for i in range(1,6):
t.submit(download_one_page,url,i)
协程
#python编写协程的程序
import asyncio,time
# async def func():
# print("hello world")
# if __name__=='__main__':
# g=func()
# asyncio.run(g)#协程程序运行需要asyncio模块的支持
async def func1():
print("你好啊，我叫小猪佩奇")
await asyncio.sleep(3)
print("你好啊，我叫小猪佩奇")
async def func2():
print("你好啊，我叫卡布达")
# time.sleep(2)#当程序出现了同步操作的时候，异步就中断了
await asyncio.sleep(2)#异步操作代码
print("你好啊，我叫卡布达")
async def func3():
print("你好啊，我叫葫芦娃")
await asyncio.sleep(4)
print("你好啊，我叫葫芦娃")
if __name__=='__main__':
f1=func1()
f2=func2()
f3=func3()
tasks=[
f1,f2,f3
]
t1=time.time()
#一次性启动多个任务（协程）
asyncio.run(asyncio.wait(tasks))
t2=time.time()
print(t2-t1)import asyncio,time
async def func1():
print("你好啊，我叫小猪佩奇")
await asyncio.sleep(3)
print("你好啊，我叫小猪佩奇")
async def func2():
print("你好啊，我叫卡布达")
# time.sleep(2)#当程序出现了同步操作的时候，异步就中断了
await asyncio.sleep(2)#异步操作代码
print("你好啊，我叫卡布达")
async def func3():
print("你好啊，我叫葫芦娃")
await asyncio.sleep(4)
print("你好啊，我叫葫芦娃")
async def main():
#第一种写法
#f1=func1()
#await f1 一般await挂起操作放在协程对象前面
#第二种写法（推荐）
tasks=[
asyncio.create_task(func1()),
asyncio.create_task(func2()),
asyncio.create_task(func3()),
]
await asyncio.wait(tasks)
if __name__ == '__main__':
t1=time.time()
asyncio.run(main())
t2=time.time()
print(t2-t1)异步协程aiohttp模块
#requests.get() 同步的代码->异步操作aiohttp
#pip install aiohttp
import asyncio
import aiohttp
urls=[
"http://kr.shanghai-jiuxin.com/file/bizhi/20220928/by12mnrxl5j.jpg",
"http://kr.shanghai-jiuxin.com/file/bizhi/20220928/pckme50aodn.jpg",
"http://kr.shanghai-jiuxin.com/file/bizhi/20220928/ultvo5cq2uw.jpg"
]
async def aiodownload(url):
#s=aiohttp.ClientSession() #相当与requests
#发送请求
#得到图片的内容
#保存到文件
name=url.rsplit("/",1)[1] #切取一次取第0个
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
#写入文件
with open(name,mode="wb") as f:
f.write(await resp.content.read())#读取内容是异步的，需要await挂起，rsp.text()
async def main():
tasks=[]
for url in urls:
tasks.append(aiodownload(url))
await asyncio.wait(tasks)
if __name__ == '__main__':
asyncio.run(main())扒光一部小说
西游记
综合训练：抓取一部视频
#一般的视频网站是怎么做的？
#用户上传->转码（把视频做处理）->切片处理（把单个文件进行拆分）
#用户在进行拉动进度条的时候
#需要一个文件记录：1.视频播放顺序，2.视频存放路径
"""想要抓取一个视频：
1.找到m3u8(各种手段)
2.通过m3u8下载ts文件
3.可以通过各种手段（不仅是编程手段），把ts文件合并成一个mp4文件
"""Selenium
概述
步骤
打开和关闭浏览器
browser=selenium.webdriver.Chrome()#打开谷歌浏览器
browser.close()获取网址
browser.get(url)寻找节点
#寻找单节点
node = browser.find_element_by_xpath("//span[@style='user-select;auto']")
#寻找多节点
node = browser.find_elements_by_xpath("//span[@style='user-select;auto']")输入信息
input=browser.find_element_by_xpath("//input")
input.send_keys("青")点击按钮
button=browser.find_element_by_xpath('')
button.click()selenium——以LoL数据网站opgg为例
下载xpath helper插件
https://www.op.gg/champions/darius/top/counters?region=global&tier=platinum_plus
from selenium import webdriver
import selenium
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support import ui
from selenium.webdriver.chrome.options import Options
chrome_options=Options()
#增加无头
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
#防止被网站识别
chrome_options.add_experimental_option('excludeSwitches',['enable-automation'])
wait_time=5
#用户输入
position=input("请输入您想查询的位置")
hero=input("请输入您要查询的英雄")
#构造网址
url=f"https://www.op.gg/champions/{hero}/{position}/counters"
#打开浏览器
browser=webdriver.Chrome(chrome_options=chrome_options,executable_path=r"phantomjs-2.1.1-windows\bin")
#browser.maximize_window() 最大化
browser.get(url)
browser.implicitly_wait(10)
#寻找按钮多节点
wait=ui.WebDriverWait(browser,wait_time)
try:
wait.until(lambda driver:driver.find_elements(By.XPATH,r'//*[@id="content-container"]/aside/div[1]/div[2]/div/table/tbody/tr/td[2]/div/div'))
except Exception as error1:
print(error1)
buttons=browser.find_elements(By.XPATH,r'//*[@id="content-container"]/aside/div[1]/div[2]/div/table/tbody/tr/td[2]/div/div')
#遍历每一个按钮节点
for button in buttons:
# 点击按钮
browser.execute_script("arguments[0].click();", button)
# 寻找表格的最左列
time.sleep(5)
col = browser.find_elements(By.XPATH, r'//*[@id="content-container"]/main/div[1]/ul/li/div/div/div[1]/div')
print(button.text, col[0].text, col[5].text)
可执行程序
selenium——利用鼠标移动操作巧妙爬取highcharts动态图表
https://robo.datayes.com/v2/stock/002475/information#STOCK_POPULAR_FEELINGS
问题：大部分情况卡在登录页面，然后报错（偶尔成功）
如何操作鼠标
from selenium.wevdriver.common.action_chains import ActionChains
browser=webdriver.Chrome()
#进行链式鼠标操作
action=ActionChains(browser)
action.(各种操作)
action.perform有关移动操作
from selenium.wevdriver.common.action_chains import ActionChains
browser=webdriver.Chrome()
#进行链式鼠标操作
action=ActionChains(browser)
action.move_by_offset(xasis,yasis)#第一次是绝对坐标，第二次是相对坐标from selenium import webdriver
import selenium
from selenium.webdriver.common.action_chains import ActionChains
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support import ui
from id_and_key import zhanghao,key
from selenium.webdriver.chrome.options import Options
chrome_options=Options()
#防止被网站识别
chrome_options.add_experimental_option('excludeSwitches',['enable-automation'])
wait_time=3
runtime=300
#1.打开浏览器，输入网址
browser=webdriver.Chrome(chrome_options=chrome_options)
browser.maximize_window()
browser.get("https://robo.datayes.com/v2/stock/002475/information#STOCK_POPULAR_FEELINGS")
browser.implicitly_wait(10)#加入隐式等待
time.sleep(5)
# # 寻找立即登录
# button = browser.find_element(By.XPATH, '//*[@id="_0d1ab255-c166-4ff8-8005-57b44fc2ea2e"]/div[1]/div[2]/div/p[2]/button')
# browser.execute_script("arguments[0].click();", button)
# 寻找账号登录在这里
button1 = browser.find_element(By.XPATH, '/html/body/div[5]/div/div')
browser.execute_script("arguments[0].click();", button1)
time.sleep(2)
#寻找账号输入框
input_id=browser.find_element(By.XPATH,'//*[@id="username"]')
input_id.send_keys(zhanghao)
#寻找密码输入框
input_key=browser.find_element(By.XPATH,'//*[@id="password"]')
input_key.send_keys(key)
#寻找点击登录
button2=browser.find_element(By.XPATH,'//*[@id="dyc-login-register"]/div[2]/div[2]/div[1]/div[3]/div[1]/form/div[3]/div/div/span/button')
browser.execute_script("arguments[0].click();", button2)
time.sleep(5)
#
days=eval(input("请调节搜索范围"))
#定义步长
step=300/days
if step<1:
step=1
# 鼠标的起始点
action = ActionChains(browser)
action.move_by_offset(350, 250).perform()
while 1:
try:
action=ActionChains(browser)
action.move_by_offset(10,0).perform()
except Exception:
break
#寻找span节点
try:
wait=ui.WebDriverWait(browser,runtime)
wait.until(lambda driver: driver.find_elements(By.XPATH,'//span'))
except Exception as error1:
print(error1)
try:
wait.until(lambda driver: driver.find_elements(By.XPATH,'//span'))
except Exception as error2:
print(error2)
time.sleep(2)
# time.sleep(1)
spans=browser.find_elements(By.XPATH,"//span")
print(spans[25].text)
# for i in range(len(spans)):
# print(i)
# print(spans[i].text)
# print(len(spans))