hdwan.net 大佬有话说 :
隔壁看到的爬虫下小说[Python]
import requests #导入HTTP请求库
from pyquery import PyQuery #PyQuery，原生css选择器
”’获取单章节内容”’
def get_one_chapter(chapter_url = None, name = None):
response = requests.get(url = chapter_url)
doc = PyQuery(response.text)
title = doc("h1").text() #“h1”各网站标签不尽相同，请自行分析网站源码配置
content = doc("#content").text() #“#content”各网站小说内容关键词，请自行分析网站源码配置
print(title,content)
with open(file = name + ".txt", mode = "a+", encoding = "utf-8") as f:
f.write(title + "nn" + content)
”’解析一本书所有章节的url和书名”
def get_index(book_url):
index_url = book_url
text = requests.get(url = index_url).text
doc = PyQuery(text)
links = doc(‘#list a’) #links 获取所有链接标签
name = doc(‘h1’).text()
for link in list(links.items()): #一般目录网页都会有限显示最新更新章节，更具实际情况跳过章节
chapter_url = ‘http://www.XXXXXX’ + link.attr.href #防止有人说我打广告，网址自己配置
get_one_chapter(chapter_url = chapter_url, name = name)
”’解析全站书籍url”’
def get_all_book_url():
all_book_url = "http://www.XXXXXX"
response = requests.get(url = all_book_url).text
doc = PyQuery(response)
links = doc(‘#main a’)
for link in list(links.items()):
book_url = ‘http://www.XXXXXX’ + link.attr.href
get_index(book_url = book_url)
get_all_book_url()
https://www.vxotu.com/u/20200416/1524233.png
tir 大佬有话说 :
看起来不错，但是现在不需要爬，有现成的规则