python 线程池爬取段子,解析库是BeautifulSoup,数据库是mongdb

使用线程池，爬取段子，并将段子存入数据库。 -程序中使用了两个类，发送请求类和获取数据类，随机生成User_agent.看代码 import random import re import time from multiprocessing.dummy import Pool # 线程池 from queue import Queue # 线程间的通信 import requests from pymongo import MongoClient from bs4 import BeautifulSoup client = MongoClient(host="localhost",port=27017) col = client["artical"]["joke"] class ParseUrl(object): """发送请求获取相应""" def __init__(self): pass def parse(self,url,header=None): if not header: header = {"User-Agent":self.User_agent()} header["Referer"] = url response=requests.get(url,headers=header) bs = BeautifulSoup(response.text,features="lxml") return response,bs def User_agent(self): # 随机的User_agent first_num = random.randint(55, 62) third_num = random.randint(0, 3200) fourth_num = random.randint(0, 140) os_type = [ '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)', '(Macintosh; Intel Mac OS X 10_12_6)' ] chrome_version = 'Chrome/{}.0.{}.{}'.format(first_num, third_num, fourth_num) user_agent = ' '.join(['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36', '(KHTML, like Gecko)', chrome_version, 'Safari/537.36'] ) return user_agent class Joke_sipder(ParseUrl): """获取数据及保存"""" def __init__(self): ParseUrl.__init__(self) self.url = "http://duanziwang.com/" self.base_url = "http://duanziwang.com/page/{}/" self.pool = Pool(5) self.is_running = True self.total_requests_num = 0 self.total_response_num=0 self.queue = Queue() def get_parse(self,url): # 解析 _,bs=self.parse(url) article_list = bs.find_all("article",{"class":"post"}) for art in article_list: item = dict() item["title"]=art.find("h1",class_="post-title").get_text() item["time"] = art.find("time",datetime=re.compile(r"T")).get_text() item["hot"] = art.find("time",text=re.compile(r"°C")).get_text() item["praise"] = art.find("a",class_="post-like").get_text() item["content"] = art.find("div",class_="post-content").p.get_text() if art.find("div",class_="post-content").p else None self.save_content(item) self.total_response_num+=1 def get_next(self): # 获取url for i in range(1,16): self.queue.put(self.base_url.format(i)) self.total_requests_num += 1 def save_content(self,data): """保存数据""" col.insert_one(data) def exetute_save(self): url = self.queue.get() self.get_parse(url) def _callback(self,temp): if self.is_running: self.pool.apply_async(self.exetute_save,callback=self._callback) def main(self): self.get_next() for i in range(5): # 控制并发量 self.pool.apply_async(self.exetute_save,callback=self._callback) # 异步请求 while True: time.sleep(0.0001) if self.total_response_num >= self.total_requests_num: self.is_running=False break self.pool.close() # 关闭线程池，防止新的开启 if __name__ == '__main__': jkp=Joke_sipder() jkp.main()