1、药网数据爬取： url:https://www.111.com.cn/categories/953710?tp=10-1 要求：抓取50页 字段：总价，描述，评论数量，详情页链接 用正则爬取。 代码如下
import re,requests,json base_url='https://www.111.com.cn/categories/953710?-j%s.html' # https://www.111.com.cn/categories/953710-j7.html headers={ 'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36' } info=[] for i in range(1,51): item={} # 总价，描述，评论数量，详情页链接 response = requests.get(base_url %i, headers=headers) # print(response.text) ul_pattern=re.compile('<ul id="itemSearchList" class="itemSearchList">(.*?)</ul>',re.S) ul_content=ul_pattern.search(response.text) if ul_content: ul_content=ul_content.group(1) # print(ul_content) li_pattern =re.compile(r'<li id="producteg_(.*?)</li>',re.S) li_content =li_pattern.findall(ul_content) # print(li_content) # print(type(li_content)) for li in li_content: #爬取总价 price_pattern = re.compile(r'<span>(.*?)</span>',re.S) price_content= price_pattern.search(li) if price_content: price_content = price_content.group(1).strip() # print(price_content) #爬取描述 detail_pattern = re.compile(r'<span class=" list_lable_self"></span>(.*?)</a>',re.S) detail_content = detail_pattern.search(li) if detail_content: detail_content=detail_content.group(1).strip() # print(detail_content) #爬取评论数量 commit_pattern = re.compile(r'评论 <em>(.*?)</em>条</a>',re.S) commit_content = commit_pattern.search(li) if commit_content: commit_content = commit_content.group(1).strip() # print(commit_content) #爬取详情连接 xq_pattern = re.compile('href="//(.*?)" target="_blank"',re.S) xq_content = xq_pattern.search(li) if xq_content: xq_content = xq_content.group(1).strip() # print(xq_content) item['price_content']=price_content item['detail_content']=detail_content item['commit_content']=commit_content item['xq_content']=xq_content info.append(item) with open('haha.json','w',encoding='utf-8') as fp: json.dump(info,fp)经验总结：在使用正则时尽量多匹配字符，这样能够精确匹配，否则数据容易混淆
