1.爬取电子出版社的生活类书本数据
通过ajax查找实现
# -- coding: utf-8 --
# 目标:生活板块的书籍书名、价格和作者
import json
import jsonpath
import requests
headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
'cookie':'acw_tc1a0c639f17292171657431353e00d3333404ced367ab5ea36be81ca90c3609;JSESSIONID=A8143E0428FC4DA3171E8560C47966E8'}
# response = requests.get('https://www.ptpress.com.cn/recommendBook/getRecommendBookListForPortal?bookTagId=d5cbb56d-09ef-41f5-9110-ced741048f5f',headers=headers)
# dict_data = json.loads(response.content)
# print(jsonpath.jsonpath(dict_data,'$..bookName'))
2.爬取详细信息
# 目标:书籍书名、价格和作者
# 1.获取书名id的ajax
# 2.根据id获取单本书籍信息的ajax
# 3.根据data得到想要的内容
import requests # 导入请求库,用于发送 HTTP 请求
import json # 导入 JSON 库,用于处理 JSON 数据
# 定义获取推荐书籍列表的 URL
url = 'https://www.ptpress.com.cn/recommendBook/getRecommendBookListForPortal?bookTagId=d5cbb56d-09ef-41f5-9110-ced741048f5f'
# 设置请求头,模拟浏览器请求
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
'cookie': 'acw_tc1a0c639f17292171657431353e00d3333404ced367ab5ea36be81ca90c3609; JSESSIONID=A8143E0428FC4DA3171E8560C47966E8'
}
# 发送 GET 请求以获取推荐书籍列表
response = requests.get(url=url, headers=headers)
books = response.json()['data']
def output_to_console(books):
# 输出标题
print('书名\t作者\t价格')
# 遍历返回的数据中的每本书
for book in books:
book_id = book['bookId']
detail = get_book_details(book_id) # 获取书籍详细信息
if detail:
# 提取书名、作者和折后价格
book_name = detail['bookName']
author = detail['author']
discount_price = detail['discountPrice']
print(f"{book_name}\t{author}\t{discount_price}")
def get_book_details(book_id):
url = 'https://www.ptpress.com.cn/bookinfo/getBookDetailsById'
params = {'bookId': book_id}
response = requests.post(url=url, headers=headers, params=params)
return response.json()['data'] if response.json().get('success') else None
# 调用函数输出书籍信息
output_to_console(books)
3.爬取当当网书本信息(静态网页练习)
# -- coding: utf-8 --
# 爬取目标:排名、书名、图片地址、作者、推荐指数、五星评分次数、价格
import requests
import time
from lxml import etree
headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
'cookie':'ddscreen=2; dest_area=country_id%3D9000%26province_id%3D111%26city_id%20%3D0%26district_id%3D0%26town_id%3D0; __permanent_id=20241017154613694111150420084440082; __visit_id=20241017154613696429022245816885774; __out_refer=; __trace_id=20241017154613696426813165695783602'}
response = requests.get('http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-1',headers=headers)
num=1
while num<10:
url = 'http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-'+str(num)
response = requests.get(url,headers=headers)
# print(response.status_code)
# print(response.text)
time.sleep(3)
html = etree.HTML(response.text)
# el_list = html.xpath('/html/body/div[3]/div[3]/div[2]/ul/li') # 已经确定访问成功且能读取数据,静态网页换xpath定位方法
el_list = html.xpath('//ul[@class="bang_list clearfix bang_list_mode"]/li')
print(len(el_list))
for li in el_list:
paiming = li.xpath('./div[1]/text()')
shuming = li.xpath('./div[3]/a/@title')
tupiandizhi = li.xpath('./div[2]/a/img/@src')
zuozhe = li.xpath('./div[5]/a/text()') if len(li.xpath('./div[5]/a/text()')) > 0 else "未知"
tuijian = li.xpath('./div[4]/span[2]/text()')
cishu = li.xpath('./div[7]/span/text()')
jiage = li.xpath('./div[8]/p[1]/span[1]/text()')
dict = {
'排名': paiming,
'书名': shuming,
'图片地址': tupiandizhi,
'作者': zuozhe,
'推荐指数': tuijian,
'五星评分次数': cishu,
'价格': jiage
}
print(dict)
num+=1