{"id":159814,"date":"2020-09-23T00:02:28","date_gmt":"2020-09-22T16:02:28","guid":{"rendered":"http:\/\/4563.org\/?p=159814"},"modified":"2020-09-23T00:02:28","modified_gmt":"2020-09-22T16:02:28","slug":"%e8%af%b7%e6%95%99%e5%85%b3%e4%ba%8e-python-%e5%a4%9a%e7%ba%bf%e7%a8%8b%e4%b8%8b%e8%bd%bd%e5%99%a8%e9%97%ae%e9%a2%98","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=159814","title":{"rendered":"\u8bf7\u6559\u5173\u4e8e Python \u591a\u7ebf\u7a0b\u4e0b\u8f7d\u5668\u95ee\u9898"},"content":{"rendered":"
\n
\n
\n

\u8bf7\u6559\u5173\u4e8e Python \u591a\u7ebf\u7a0b\u4e0b\u8f7d\u5668\u95ee\u9898 <\/h1>\n

<\/p>\n

\n
\u8cc7\u6df1\u5927\u4f6c : Chan775 <\/span> <\/i> 2<\/span> <\/div>\n
<\/div>\n<\/p><\/div>\n<\/p><\/div>\n<\/p><\/div>\n
<\/p>\n

\u521a\u5b66\u5b8c python \u4e0d\u4e45\uff0c\u60f3\u505a\u4e2a\u5c0f\u722c\u866b\u7ec3\u7ec3\u624b\uff0c\u4e8e\u662f\u6574\u4e86\u4e2a\u5c0f\u722c\u866b\u5728 vps \u6301\u7eed\u722c\u53d6\u67d0\u4e2a\u5c0f\u89c6\u9891\u7f51\u7ad9\u7684\u53d1\u5e03\u7684\u5c0f\u89c6\u9891\uff0c\u5c0f\u89c6\u9891\u7684\u5927\u5c0f\u4e3a 1-200M\uff0c\u5c0f\u722c\u866b\u8d1f\u8d23\u4e0b\u8f7d\u89c6\u9891\u7684\u90e8\u5206\u5f88\u7b80\u5355\uff0c\u76f4\u63a5 r = requests.get(url),\u7136\u540e\u628a r.content \u5199\u5165\u6587\u4ef6\u3002<\/p>\n

\u540e\u6765\u89c9\u5f97\u4e0b\u8f7d\u6548\u7387\u597d\u50cf\u4e0d\u884c\uff0c\u4e8e\u662f\u4e0a\u7f51\u641c\u4e86\u4e00\u4e0b\u591a\u7ebf\u7a0b\u4e0b\u8f7d\u5668\uff0c\u4fbf\u5f00\u59cb\u91cd\u5199\u4e0b\u8f7d\u89c6\u9891\u90e8\u5206\u7684\u4ee3\u7801\u4e86\uff0c\u6210\u54c1\u5982\u4e0b\uff0c\u5728\u6211\u7684\u7535\u8111\u4e0a\u80fd\u6b63\u5e38\u8fd0\u884c\uff0c\u4f46\u5728 vps \u4e0a\u6301\u7eed\u8fd0\u884c\u7684\u8bdd\uff0c\u6709\u4e0d\u5c11\u95ee\u9898\uff0c\u4e0d\u4ec5\u5185\u5b58\u5360\u7528\u5927(\u5360\u7528 200 \u591a M \u7684\u5185\u5b58)\uff0c\u800c\u4e14\u65f6\u4e0d\u65f6\u51fa\u73b0Max retries exceeded with url<\/code>\u7684\u9519\u8bef\u3002<\/p>\n

from threading import Thread  import requests   class Download:     def __init__(self, url):         self.url = url         self.ua = {'User-Agent': 'Mozilla\/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko\/20100101 Firefox\/80.0'}         r = requests.head(self.url, headers=self.ua)         # \u5faa\u73af\u5bfb\u5740         while r.status_code == 301 or r.status_code == 302:             self.url = r.headers['Location']             r = requests.head(self.url, headers=self.ua)         self.name = self.url.split('?')[0].split('\/')[-1]         self.size = int(r.headers['Content-Length'])         # \u521b\u5efa\u7b49\u5927\u7a7a\u6587\u4ef6         f = open(self.name, \"wb\")         f.truncate(self.size)         f.close()         # \u6839\u636e\u6587\u4ef6\u5927\u5c0f\u5206\u914d\u7ebf\u7a0b         if self.size < 5 * 1024 * 1024:             self.thread_num = 1         elif self.size < 10 * 1024 * 1024:             self.thread_num = 2         elif self.size < 20 * 1024 * 1024:             self.thread_num = 4         elif self.size < 40 * 1024 * 1024:             self.thread_num = 8         else:             self.thread_num = 16         # \u786e\u5b9a\u6587\u4ef6\u5757\u5927\u5c0f         self.part = self.size \/\/ self.thread_num      def dl(self, start, end):         header = {'Range': 'bytes={}-{}'.format(start, end)}         header.update(self.ua)         with requests.get(self.url, headers=header, stream=True) as r:             with open(self.name, 'rb+') as f:                 f.seek(start)                 f.write(r.content)      @staticmethod     def unit_conversion(byte):         byte = int(byte)         if byte > 1024:             res = byte \/ 1024             if res < 1024:                 res = float('%.2f' % res)                 return str(res) + 'KB'             elif res < 1024 * 1024:                 res = res \/ 1024                 res = float('%.2f' % res)                 return str(res) + 'MB'             else:                 res = res \/ (1024 * 1024)                 res = float('%.2f' % res)                 return str(res) + 'GB'         else:             return str(byte) + 'B'      def run(self):         thread_list = []         for i in range(self.thread_num - 1):             start = i * self.part             end = (i + 1) * self.part             t = Thread(target=self.dl, args=(start, end))             thread_list.append(t)         # \u6700\u540e\u4e00\u90e8\u5206         start = (self.thread_num - 1) * self.part         end = self.size         t = Thread(target=self.dl, args=(start, end))         thread_list.append(t)         # \u542f\u52a8\u6240\u6709\u5b50\u7ebf\u7a0b         for t in thread_list:             t.start()         # \u5b50\u8fdb\u7a0b\u5408\u5e76\u5230\u4e3b\u7ebf\u7a0b         for t in thread_list:             t.join()         print(f'{Download.unit_conversion(self.size)} {self.name}\u4e0b\u8f7d\u5b8c\u6210\uff01')         return self.name, self.size   if __name__ == '__main__':     t = Download('http:\/\/www.baidu.com').run()     print(t)  <\/code><\/pre>\n
\u8bf7\u6559\u5927\u5bb6\u51e0\u4e2a\u95ee\u9898\uff1a 1 \u3001\u591a\u7ebf\u7a0b\u4e0b\u8f7d\u7684\u65f6\u5019\uff0c\u591a\u4e2a\u7ebf\u7a0b\u8bfb\u53d6\u540c\u4e00\u4e2a\u6587\u4ef6\uff0c\u4e0d\u540c\u7684\u7ebf\u7a0b\u5728\u6587\u4ef6\u7684\u4e0d\u540c\u4f4d\u7f6e\u5199\u5165\u5185\u5bb9\uff0c\u9700\u8981\u52a0\u5165\u7ebf\u7a0b\u9501\u5417\uff1f\u6211\u611f\u89c9\u5b83\u4eec\u5199\u5165\u7684\u90e8\u5206\u4e0d\u540c\uff0c\u597d\u50cf\u6ca1\u6709\u51b2\u7a81\u554a<\/p>\n
2 \u3001\u6211\u5728\u7f51\u4e0a\u67e5\u5230Max retries exceeded with url<\/code>\u7684\u9519\u8bef\u662f\u7531\u4e8e http \u8fde\u63a5\u8fc7\u591a\u5f15\u8d77\u7684\uff0c\u7f51\u4e0a\u7ed9\u51fa\u7684\u65b9\u6cd5\u6709\u7528 with \u8bed\u53e5\u6253\u5f00 requests.get(url)\u6765\u786e\u4fdd\u8fde\u63a5\u4f1a\u88ab\u5173\u95ed\uff0c\u8fd8\u6709\u7684\u662f\u5efa\u8bae\u76f4\u63a5\u52a0 response.close()\u5173\u95ed\uff0c\u8fd8\u6709\u7684\u7528with closing(requests.get(img_url, stream=True)) as r:<\/code>,\u54ea\u79cd\u65b9\u6cd5\u6bd4\u8f83\u9760\u8c31\u5462\uff1f<\/p>\n
3 \u3001\u4ffa\u8fd8\u6709\u5565\u53ef\u4ee5\u4f18\u5316\u6539\u8fdb\u7684\u5730\u65b9\uff1f<\/p>\n<\/p><\/div>\n
 \u5927\u4f6c\u6709\u8a71\u8aaa<\/b> (5<\/span>)        <\/div>\n
 <\/div>\n<\/p><\/div>\n<\/p><\/div>\n
    \n
  • \n
    \n
    \n
     \u8cc7\u6df1\u5927\u4f6c : ysc3839 <\/span>  <\/div>\n
     <\/i>            <\/span> <\/div>\n<\/p><\/div>\n
                                                                 \u76f4\u63a5\u8c03\u7528 aria2 \u53bb\u4e0b\u8f7d                                                            <\/div>\n<\/p><\/div>\n<\/li>\n
  • \n
    \n
    \n
     \u8cc7\u6df1\u5927\u4f6c : virtuemartx <\/span>  <\/div>\n
     <\/i>            <\/span> <\/div>\n<\/p><\/div>\n
                                                                 \u591a\u7ebf\u7a0b\u4e0d\u52a0\u9501\u4e5f\u884c \u662f\u4e0d\u662f\u56e0\u4e3a python gil \uff1f \u5bfc\u81f4\u591a\u7ebf\u7a0b\u4e5f\u8ddf nodejs \u5f02\u6b65\u5dee\u4e0d\u591a\uff1f                                                            <\/div>\n<\/p><\/div>\n<\/li>\n
  • \n
    \n
    \n
     \u8cc7\u6df1\u5927\u4f6c : laminux29 <\/span>  <\/div>\n
     <\/i>            <\/span> <\/div>\n<\/p><\/div>\n
                                                                 1.\u6210\u4eba\u7f51\u7ad9\u5bf9\u4e8e\u56fa\u5b9a IP\uff0c\u6bcf\u5929\u4f1a\u6709\u56fa\u5b9a\u89c2\u770b\u989d\u5ea6\uff0c\u4f60\u4e00\u4e2a\u6216 2 \u4e2a\u8fdb\u7a0b\u8fdb\u884c\u4e0b\u8f7d\uff0c\u5c31\u4f1a\u6d88\u8017\u6389\u4e00\u4e2a\u989d\u5ea6\u3002\u8fd9\u662f Max retries exceeded with url \u7684\u4ea7\u751f\u539f\u56e0\u3002\u8d2d\u4e70\u4ed8\u8d39\u4f1a\u5458\uff0c\u989d\u5ea6\u4f1a\u63d0\u9ad8\uff0c\u4f46\u6700\u9ad8\u4ed8\u8d39\u8d26\u53f7\u7684\u989d\u5ea6\uff0c\u4e5f\u7ecf\u4e0d\u8d77\u722c\u866b\u8fd9\u79cd\u904d\u5386\u5f0f\u7684\u6d88\u8017\u3002<\/p>\n
    2.\u6210\u4eba\u7f51\u7ad9\u4e00\u822c\u670d\u52a1\u5668\u90fd\u5728\u5883\u5916\uff0c\u800c\u4e14\u7531\u4e8e\u4eba\u591a\uff0c\u5355\u7ebf\u7a0b\u4e0b\u8f7d\u7684\u901f\u5ea6\u90fd\u4e0d\u884c\u3002\u53ef\u4ee5\u8d2d\u4e70\u4ed8\u8d39\u4f1a\u5458\uff0c\u4f1a\u6709\u4e13\u95e8\u7684\u63a7\u5236\u4eba\u6570\u7684\u9ad8\u901f\u901a\u9053\uff0c\u8fd9\u6837\u5355\u7ebf\u7a0b\u4e0b\u8f7d\u901f\u5ea6\u5c31\u5feb\u4e86\u3002<\/p>\n
    3.\u591a\u7ebf\u7a0b\u4e0b\u8f7d\u7684\u8f6f\u4ef6\uff0c\u4e00\u822c\u4e00\u4e2a\u7ebf\u7a0b\u662f 10kb block\uff0c\u6700\u591a\u4e5f\u5c31 1024kb block\uff0c\u4f60\u8fd9\u4e0b\u8f7d\u901f\u5ea6\u6162\uff0c\u7136\u540e\u4e00\u4e2a\u7ebf\u7a0b\u8fd8\u8dd1\u4e86 5MB block\uff0c\u800c\u4e14\u8fd8\u662f\u5c0f\u5185\u5b58 VPS \u5728\u8dd1\u3002<\/p>\n
    \u5efa\u8bae\uff1a<\/p>\n
    \u8981\u7ec3\u624b\u7684\u8bdd\uff0c\u4e70\u4e2a\u7a33\u5b9a\u7684\u673a\u573a\uff0c\u53bb xxxxhub \u7ec3\u624b\uff0c\u6bcf\u4e2a\u7ebf\u7a0b\u6539\u4e3a 32kb block\uff0c\u6bcf\u6761\u89c6\u9891\u8dd1\u51e0\u4e2a\u7ebf\u7a0b\uff0c\u4ee5\u53ca\u540c\u65f6\u8dd1\u591a\u5c11\u6761\u89c6\u9891\uff0c\u8981\u6839\u636e\u4f60\u7684 VPS \u5185\u5b58\u4e0e\u5e26\u5bbd\u6765\u5b9a\u3002                                                            <\/p><\/div>\n<\/p><\/div>\n<\/li>\n
  • \n
    \n
    \n
     \u8cc7\u6df1\u5927\u4f6c : fasionchan <\/span>  <\/div>\n
     <\/i>            <\/span> <\/div>\n<\/p><\/div>\n
                                                                 r.content \u5c06 response body \u6574\u4f53\u8bfb\u5230\u5185\u5b58\uff0c\u5e94\u8be5\u662f\u9020\u6210\u5185\u5b58\u5360\u7528\u5927\u7684\u5173\u952e\u56e0\u7d20\u3002\u53ef\u4ee5\u5206\u914d\u56fa\u5b9a\u7684\u7f13\u51b2\u533a\uff0c\u5206\u6bb5\u8bfb\u5199\u3002                                                            <\/div>\n<\/p><\/div>\n<\/li>\n
  • \n
    \n
    \n
     \u4e3b<\/span> \u8cc7\u6df1\u5927\u4f6c : Chan775 <\/span>  <\/div>\n
     <\/i>            <\/span> <\/div>\n<\/p><\/div>\n
                                                                 \u611f\u8c22\u56de\u590d\uff0c\u4e0d\u5f97\u4e0d\u8bf4\uff0c\u8001\u54e5\u4f60\u597d\u6709\u7ecf\u9a8c\u3002<\/p>\n
    1. \u786e\u5b9e\u662f\u5b58\u5728\u89c2\u770b\u9650\u5236\uff0c\u4e0d\u8fc7\u53ef\u4ee5\u7ed5\u8fc7\u5b83\uff0c\u901a\u8fc7\u5b83\u7684\u5206\u4eab\u9875\u9762\u4e0b\u8f7d\u6216\u8005\u7528 x-forward \u53c2\u6570\u4f2a\u9020 ip \u6216\u8005\u76f4\u63a5\u6784\u9020\u4e0b\u8f7d\u76f4\u94fe\u3002<\/p>\n
    2. \u6211\u7684 vps \u4e5f\u662f\u5728\u5883\u5916\uff0c1.5g \u5185\u5b58\u3002\u89c6\u9891\u4e0b\u8f7d\u901f\u5ea6\u5012\u662f\u4e0d\u5f88\u6162\uff0c\u5c31\u662f\u5927\u6587\u4ef6\u7684\u8bdd\uff0c\u7528 r.content \u4e00\u6b21\u8bfb\u53d6\u5230\u5185\u5b58\uff0c\u6709\u65f6\u4f1a\u4e2d\u65ad\uff0c\u6240\u6709\u60f3\u6574\u591a\u7ebf\u7a0b\u4e0b\u8f7d\uff0c\u53e6\u5916\u4e5f\u80fd\u5b9e\u8df5\u4e00\u4e0b\u591a\u7ebf\u7a0b\u7684\u77e5\u8bc6\u3002<\/p>\n
    3. \u6700\u540e\u591a\u7ebf\u7a0b\u8fd9\u4e2a\uff0c\u6211\u786e\u5b9e\u4e0d\u592a\u61c2\u8be5\u600e\u4e48\u5206\u914d\u7ebf\u7a0b\uff0c\u7b2c\u4e00\u6b21\u5199\u591a\u7ebf\u7a0b\u7684\u4ee3\u7801\u3002\u6211\u770b aria2 \u6700\u591a\u5206\u914d 16 \u7ebf\u7a0b\uff0c\u6240\u4ee5\u6574\u4e86\u4e2a\u6700\u5927 16 \u7ebf\u7a0b\u7684\u3002                                                            <\/p><\/div>\n<\/p><\/div>\n<\/li>\n
  • \n","protected":false},"excerpt":{"rendered":"
    \u8bf7\u6559\u5173\u4e8e Python \u591a\u7ebf\u7a0b\u4e0b\u8f7d…<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/159814"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=159814"}],"version-history":[{"count":0,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/159814\/revisions"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=159814"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=159814"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=159814"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}