package main
import (
"bufio"
"fmt"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/transform"
"io"
"io/ioutil"
"net/http"
)
func determineEncoding(r io.Reader) encoding.Encoding {
//编码判断
bytes, err := bufio.NewReader(r).Peek(1024)
if err != nil {
panic(err)
}
e, _, _ := charset.DetermineEncoding(bytes, ",")
return e
}
func main() {
resp, err := http.Get("http://www.baidu.com")
if err != nil {
panic(err)
}
defer resp.Body.Close() //最后关闭，先进后出
if resp.StatusCode != http.StatusOK {
fmt.Println("Error:status code", resp.StatusCode)
return
}
e := determineEncoding(resp.Body)
utf8Reader := transform.NewReader(resp.Body, e.NewDecoder())
all, err := ioutil.ReadAll(utf8Reader)
//utf8Reader:=transform.NewReader(resp.Body,simplifiedchinese.GBK.NewDecoder())
//下面是不加编码判断的
//all, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Printf("%s\n", all)
}
转载于:https://www.cnblogs.com/c-x-a/p/10707958.html
相关资源：go语言实现的简单爬虫来爬取博文
