Java爬虫

mac2022-06-30 28

1. HttpClient的使用

1.1 依赖

<dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.5.6</version> </dependency>

1.2 JavaApi

package com.sun.httpclient; import com.sun.javafx.fxml.builder.URLBuilder; import org.apache.http.NameValuePair; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; import java.util.ArrayList; import java.util.List; /** * @author sun * @date 2019/10/02 21:06 */ public class HttpTest { public static void main(String[] args) throws Exception { httpPostDataTest(); } /** * 发送一个Get / Post 请求 */ public static void httpGetTest() { CloseableHttpClient httpClient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet("http://www.baidu.com"); // 可直接替换成HttpPost try (CloseableHttpResponse response = httpClient.execute(httpGet)) { if (response.getStatusLine().getStatusCode() == 200) { String result = EntityUtils.toString(response.getEntity(), "UTF-8"); System.out.println(result); } }catch (IOException e){} } /** * 带参发送GET请求 */ public static void httpGetParamTest() throws Exception { CloseableHttpClient httpClient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(new URIBuilder("http://yun.itheima.com/search") .setParameter("keys", "Java") .build()); try(CloseableHttpResponse response = httpClient.execute(httpGet)){ // 如果请求成功的话 if(response.getStatusLine().getStatusCode() == 200){ String result = EntityUtils.toString(response.getEntity(), "UTF-8"); System.out.println(result); } } } /** * Http发送Post 带数据 */ public static void httpPostDataTest() throws IOException { CloseableHttpClient httpClient = HttpClients.createDefault(); HttpPost httpPost = new HttpPost("http://yun.itheima.com/search"); List<NameValuePair> data = new ArrayList<>(); data.add(new BasicNameValuePair("keys","Java")); UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(data,"UTF-8"); // 带参数提交 httpPost.setEntity(urlEncodedFormEntity); CloseableHttpResponse response = httpClient.execute(httpPost); // 如果请求成功 if(response.getStatusLine().getStatusCode() == 200){ String result = EntityUtils.toString(response.getEntity(), "UTF-8"); System.out.println(result); } } /** * 使用连接池 */ public static void useHttpClientPool(){ // 应该注入为一个bean PoolingHttpClientConnectionManager httpPool = new PoolingHttpClientConnectionManager(); // 使用 CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(httpPool).build(); } /** * 给请求添加配置 */ public static void httpConfigTest(){ RequestConfig config = RequestConfig.custom() .setConnectionRequestTimeout(4000) //请求连接数时间 .setConnectTimeout(4000) //链接超时时间其它的配置记得看源码 .build(); // 怎么使用 ? HttpGet httpGet = new HttpGet("http://www.baidu.com"); httpGet.setConfig(config); } }

2.Jsoup

Java HTML解析器

2.1 依赖

<dependency> <groupId>org.jsoup</groupId> <artifactId>jsoup</artifactId> <version>1.11.3</version> </dependency> /** * use jsoup to parse a basic website , here are a simple example */ public static void testJsoup() throws IOException { Document document = Jsoup.parse(new URL("http://www.itcast.cn/"), 4000); // 1.dom api way String result = document.getElementsByTag("title").first().text(); // 2.css selector way , more see css selector eg: [abc] it mean to find element is 'abc' 's item // and [name = 516] , it mean to find attribute named `name` 's value is 516 's element // here also write mix selector Elements elements = document.select("#id"); }

3. HttpClient 整合 Jsoup

代码我只写了大概思路 , 大家可以自行细化 , 分而治之

package com.sun.httpclient; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; /** * @author sun * @date 2019/10/02 22:28 */ @SuppressWarnings("all") public class JDSpider { public static void main(String[] args) throws IOException { String url = "https://item.jd.com/100005853638.html"; // 应该注入为一个bean PoolingHttpClientConnectionManager httpPool = new PoolingHttpClientConnectionManager(); // 使用 CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(httpPool).build(); HttpGet httpGet = new HttpGet(url); CloseableHttpResponse response = httpClient.execute(httpGet); if(response.getStatusLine().getStatusCode() == 200){ String html = EntityUtils.toString(response.getEntity(), "UTF-8"); Document document = Jsoup.parse(html); // 获取标题 String title = document.getElementsByClass("sku-name").text(); // 获取价格 String price = document.getElementsByClass("p-price").next().first().text(); // 获取图片 String img = document.select(".img-hover img").attr("src"); } } }

4.辅助工具

ip代理 https://proxy.mimvp.com/freesecret.php去重算法 https://github.com/yanyiwu/simhash

5.学习心得

以上的知识是我通过晚上一到两个小时内学到的 , 我敢说自己完全掌握以上记得笔记 . 不只是复制粘贴对于api , 方法我希望大家不要去死记硬背了 , 之前我刚开始学Java的时候就是背代码 , 因为当时我的脑海中还没有一个代码执行的流程 , 不懂这些套路, 所以需要背代码 , 这样才是正确的 , 不然总是不能写功能现在 , 当我学的越多后 , 心中有了这样的一个总体的流程后 , 我只需要记得这个类记得这个功能 , 具体细节我不需要记那么清楚我做了笔记 , 一是加深了映象二是忘记时快速找到觉解决问题

还有没事了多翻源码 , 这样你就知道这个类还有哪些其他的方法 , 多看源码 , 多看注释 , 被虐几遍后你就会发现如此简单我分享下我是如何看源码的今天我看了RestTemplate类的源码

首先我去看这个类(RestTemplate)上的注释 , 了解这个类的大体作用其次我会看这个类的父类的源码并做笔记,看他有哪些方法并做笔记我遵从首先广度优先 , 先在脑海大体构建这个类的结构 , 然后再深度优先 , 学习细节如何实现 , 有什么作用

看源码的过程我觉得并不难 , 只要多被打击几次 , 再来的时候就会游刃有余, 独领风骚特别强调的是多看注释

还有一点的是 , 大家学习的时候一定要挑重点看 , 对于老师开始废话的时候就快进 , 要高效的学习 , 不要浪费时间 , 直击重点, 弄清问题的本质, 因为你可能看的可是面向0基础等等的 , 那么在你会的地方的时候就跳过 , 对于高校学习这点 , 我做的总是不好 , 要么就是碰到的教程无法适应我的高度 , 要么不符合我现在的时间要求 , 需要太多时间成本学习 , 所以大家如果遇到好的学习教程的话, 可以给我分享一下吗 ~

最新回复(0)