上篇文章写了一下如何验证代理ip 是否有效,这一篇实现可以爬取代理ip网站的IP,验证有效并写入我的本地文件
import time
from selenium import webdriver
import requests
ip_file = r'C:\Users\Administrator\Desktop\代理IP.txt'
ip_url = "https://www.zdaye.com/dayProxy/2019/11/1.html"
browser = webdriver.Chrome()
browser.implicitly_wait(10)
browser.get(ip_url)
divs = browser.find_elements_by_xpath('//div[@class="thread_posts_list"]//div[@class="thread_item"]')
time.sleep(2)
divs[0].find_element_by_class_name('thread_tags').click()
time.sleep(3)
result = browser.find_element_by_class_name('cont').text
for i in result.split('\n'):
ip_one = i.split('@')[0]
try:
proxies = {'http':ip_one}
res = requests.get("http://icanhazip.com/",proxies=proxies,timeout=6)
# print(ip_one.split(':')[0])
# print(res.text)
if ip_one.split(':')[0] == res.text.strip():
print('该ip地址{}是correct啊, 老大你可以保存一下'.format(ip_one))
with open(ip_file,'a+',encoding='utf-8') as f:
f.write(ip_one+'\n')
else:
print('这个ip地址{}不合法啊, 老大'.format(ip_one))
except Exception as e:
print('请求都出现错误,这个ip地址{}真的是垃圾啊,老大'.format(ip_one))
查看结果:
