本文以 https://i.pximg.net/img-master/img/2022/01/30/00/50/14/95863886_p0_master1200.jpg
的下载作为例子
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
| func GetHeader(image string) (http.Header, error) { client := &http.Client{ Transport: &http.Transport{ Dial: func(network, addr string) (net.Conn, error) { return net.Dial("tcp", "210.140.92.142:443") }, TLSClientConfig: &tls.Config{ ServerName: "-", InsecureSkipVerify: true, }, }, } req, err := http.NewRequest("HEAD", image, nil) if err != nil { return nil, err } req.Header.Set("Host", "i.pximg.net") req.Header.Set("Referer", "https://www.pixiv.net/") resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() return resp.Header, nil }
|
执行 GetHeader("https://i.pximg.net/img-master/img/2022/01/30/00/50/14/95863886_p0_master1200.jpg")
,我们将返回的 Header
打印得到
HEADER1 2 3 4 5 6 7 8 9 10 11
| Cache-Control: max-age=31536000 Expires: Sun, 29 Jan 2023 15:50:15 GMT Last-Modified: Sat, 29 Jan 2022 15:50:14 GMT X-Content-Type-Options: nosniff Server: nginx Content-Type: image/jpeg Age: 237119 Via: http/1.1 f001 (second) Accept-Ranges: bytes Date: Tue, 01 Feb 2022 09:45:03 GMT Content-Length: 1093164
|
其中, Content-Length: 1093164
为该资源的大小, Accept-Ranges: bytes
表示该资源是可断点续传的,因而我们可以实现并发下载资源。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
| func DownSlice(image string, start, end int) ([]byte, error) { client := &http.Client{ Transport: &http.Transport{ Dial: func(network, addr string) (net.Conn, error) { return net.Dial("tcp", "210.140.92.142:443") }, TLSClientConfig: &tls.Config{ ServerName: "-", InsecureSkipVerify: true, }, }, } req, err := http.NewRequest("GET", image, nil) if err != nil { return nil, err } req.Header.Set("Host", "i.pximg.net") req.Header.Set("Referer", "https://www.pixiv.net/") req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end-1)) resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() b, _ := ioutil.ReadAll(resp.Body) return b, nil }
|
执行 DownSlice("https://i.pximg.net/img-master/img/2022/01/30/00/50/14/95863886_p0_master1200.jpg", 0, 1024)
,即可下载资源的 0-1024 的部分,知道原理后我们利用协程即可对资源进行并发下载。