初步实现效果
实现代码
import requests
from lxml import etree
import os
import sys
def getData(wd):
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
}
target_url = "https://www.baidu.com/s?wd="+str(wd)
data = requests.get(target_url, headers = headers)
data_etree = etree.HTML(data.content)
content_list = data_etree.xpath('//div[@id="content_left"]/div[contains(@class, "result c-container")]')
result = ""
for content in content_list:
result_title = "<标题> "
bd_title = content.xpath('.//h3/a')
for bd_t in bd_title:
result_title += bd_t.xpath('string(.)')
result_content = "<内容> "
bd_content = content.xpath('.//div[@class="c-abstract"]')
for bd_c in bd_content:
result_content += bd_c.xpath('string(.)')
result_link = "<链接> "+str(list(content.xpath('.//div[@class="f13"]/a[@class="c-showurl"]/@href'))[0])
result_list = [result_title, "\n" , result_content , "\n", result_link, "\n", "\n"]
for result_l in result_list:
result += str(result_l)
return result
def saveDataToFile(file_name, data):
if os.path.exists("./data/"):
pass
else:
os.makedirs("./data/")
with open("./data/"+file_name+".txt", "w+") as f:
f.write(data)
def main():
wd = ""
try:
wd = sys.argv[1]
except:
pass
if (len(wd) == 0):
wd = "火影"
str_data = getData(wd)
print(str_data)
saveDataToFile(wd, str_data)
if __name__ == '__main__':
main()