{"id":140202,"date":"2020-07-22T03:57:09","date_gmt":"2020-07-21T19:57:09","guid":{"rendered":"http:\/\/4563.org\/?p=140202"},"modified":"2020-07-22T03:57:09","modified_gmt":"2020-07-21T19:57:09","slug":"%e5%85%b3%e4%ba%8e-scrapy-%e7%9a%84%e5%87%a0%e4%b8%aa%e9%97%ae%e9%a2%98","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=140202","title":{"rendered":"\u5173\u4e8e scrapy \u7684\u51e0\u4e2a\u95ee\u9898"},"content":{"rendered":"
\n
\n
\n

\u5173\u4e8e scrapy \u7684\u51e0\u4e2a\u95ee\u9898 <\/h1>\n

<\/p>\n

\n
\u8cc7\u6df1\u5927\u4f6c : 13936 <\/span> <\/i> 3<\/span> <\/div>\n
<\/div>\n<\/p><\/div>\n<\/p><\/div>\n<\/p><\/div>\n
<\/p>\n

\u4e4b\u524d\u4e00\u76f4\u90fd\u5728\u7528 requests, \u4e0a\u624b\u662f\u5f88\u7b80\u5355\uff0c\u4f46\u662f\u91cd\u65b0\u5199\u722c\u866b\u7684\u65f6\u5019\u6709\u4e0d\u5c11\u5185\u5bb9\u662f\u91cd\u590d\u7684\u3002\u5bfc\u81f4\u65b0\u5199\u722c\u866b\u65f6\u90fd\u4f1a\u4ece\u4e0a\u4e00\u4e2a\u9879\u76ee\u90a3\u62f7\u8d1d\u4e0d\u5c11\u4ee3\u7801\u8fc7\u6765\u3002<\/p>\n

\u6700\u8fd1\u4e0a\u624b\u4e86 scrapy\uff0c\u53d1\u73b0\u4e0a\u624b\u8fd8\u6709\u70b9\u9ebb\u70e6\u3002\u524d\u671f\u51e0\u4e2a\u5c0f\u95ee\u9898\u56e0\u4e3a\u6211\u559c\u6b22\u6b7b\u78d5\u7684\u4e2a\u6027\u8017\u8d39\u4e86\u4e0d\u5c11\u65f6\u95f4\u3002 \u6700\u8fd1\u4e3b\u8981\u6709\u51e0\u4e2a\u95ee\u9898\u60f3\u8bf7\u6559\u4e0b\u5927\u5bb6\u3002<\/p>\n

\u6211\u7684\u722c\u53d6\u903b\u8f91\u662f\u5148\u722c\u53d6\u597d\u4e94\u516d\u5343\u6761\u6570\u636e\u653e\u8868\u4e00\u91cc\uff0c\u7136\u540e\u518d\u6839\u636e\u8fd9\u51e0\u5343\u6761\u6570\u636e\u7684 id \u53bb\u722c\u53d6\u65b0\u7684\u5185\u5bb9\u3002\u4e00\u822c\u80fd\u6839\u636e\u6bcf\u4e2a id \u80fd\u722c\u6765 100 \u6761\u6570\u636e\u653e\u8868\u4e8c\u91cc\u3002\u6240\u4ee5\u8868\u4e8c\u722c\u5b8c\u540e\u5927\u6982\u6709\u4e94\u5341\u4e07\u7684\u6570\u636e\u5728\u91cc\u9762\u3002<\/p>\n

\u95ee\u9898\u662f\u6211\u4e4b\u540e\u8fd8\u4f1a\u7ee7\u7eed\u722c\uff0c\u8fd8\u6709\u516d\u4e03\u4e07\u7684\u9700\u8981\u653e\u81f3\u8868\u4e00\u91cc\u7684\u6570\u636e\u6ca1\u6709\u722c\uff0c\u5982\u679c\u8fd8\u6309\u7167\u540c\u4e00\u5957\u5b58\u50a8\u903b\u8f91\u6765\uff0c\u8868\u4e8c\u5927\u6982\u5f97\u6709\u4e94\u516d\u767e\u4e07\u6761\u6570\u636e\uff0c\u4e00\u822c\u8fd9\u79cd\u60c5\u51b5\u9700\u8981\u505a\u5206\u8868\u5904\u7406\u5417\uff1f<\/p>\n

\u8fd8\u6709\uff0c\u6709\u65f6\u5019\u4f1a\u9047\u5230\u53cd\u722c\u63aa\u65bd 403 \u9519\u8bef\uff0c\u6211\u5728\u4e2d\u95f4\u4ef6\u91cc\u8bbe\u7f6e\u4e86\u5f53\u9047\u5230\u975e 200 \u54cd\u5e94\u7684\u65f6\u5019\uff0c\u5c31\u4f1a raise closespider,\u4f46\u662f\u597d\u50cf\u505c\u6b62\u4e0d\u4e86\u722c\u866b\u554a\uff0c\u53ea\u662f\u4e0d\u505c\u5730\u722c\u53d6\u65b0\u7684\u9875\u9762\u7136\u540e\u4e0d\u505c\u5730\u5347\u8d77\u65b0\u7684 closespider \u9519\u8bef\uff0c\u4f46\u8fd8\u662f\u4f1a\u7ee7\u7eed\u722c\u3002\u662f\u6211\u7684\u722c\u53d6\u903b\u8f91\u5bfc\u81f4\u7684\u5417\uff1f\u8fd8\u662f\u8bf4\u6211\u7684 closespider \u4ee3\u7801\u4e0d\u591f\u5b8c\u6574\uff1f<\/p>\n

\u63d0\u524d\u8c22\u8c22\u5404\u4f4d\u4e86\uff01<\/p>\n<\/p><\/div>\n

\u5927\u4f6c\u6709\u8a71\u8aaa<\/b> (3<\/span>) <\/div>\n
<\/div>\n<\/p><\/div>\n<\/p><\/div>\n
    \n
  • \n
    \n
    \n
    \u8cc7\u6df1\u5927\u4f6c : ooh <\/span> <\/div>\n
    <\/i> <\/span> <\/div>\n<\/p><\/div>\n
    \u51e0\u767e\u4e07\u6761\u6570\u636e\uff0c\u4f60\u67e5\u8be2\u505a\u597d\u7d22\u5f15\u6ca1\u95ee\u9898\uff0c
    Scapy \u9ed8\u8ba4\u53ea\u5904\u7406\u6210\u529f\u7684\u8bf7\u6c42(200-300)\uff0c\u5982\u679c\u8981\u5904\u7406\u9519\u8bef\u8bf7\u6c42\u8981\u653e\u5230 handle_httpstatus_list = [403] \u91cc\u9762
    \u505c\u6b62\u8c03\u7528 self.crawler.stop() <\/div>\n<\/p><\/div>\n<\/li>\n
  • \n
    \n
    \n
    \u8cc7\u6df1\u5927\u4f6c : peonone <\/span> <\/div>\n
    <\/i> <\/span> <\/div>\n<\/p><\/div>\n
    \u51e0\u767e\u4e07\u5230\u4e0d\u4e86\u5206\u8868\u7684\u7a0b\u5ea6<\/p>\n

    \u4f60\u5728 middleware \u91cc\u9762 raise closespider \u662f\u4e0d\u5bf9\u7684\uff0cclosespider \u662f\u7528\u4e8e\u5728\u56de\u8c03\u51fd\u6570\u91cc\u9762\u629b\u51fa\u7684\u3002
    downloader middleware \u7684 process_response \u53ea\u63a5\u53d7 Request \/ Response \u8fd4\u56de\u503c\u6216\u8005 IgnoreRequest \u5f02\u5e38\u3002
    \u50cf\u4e00\u8bf4\u7684\uff0c\u4f60\u9700\u8981\u5148\u4fdd\u8bc1 403 \u8bf7\u6c42\u88ab spider \u7684\u56de\u8c03\u5904\u7406\uff0c\u7136\u540e\u5728\u56de\u8c03\u91cc\u9762\u629b\u51fa closespider
    \u9047\u5230\u8fd9\u79cd\u95ee\u9898\u5e94\u8be5\u5148\u7ffb\u7ffb\u6587\u6863<\/p>\n

    https:\/\/docs.scrapy.org\/en\/latest\/topics\/exceptions.html#scrapy.exceptions.CloseSpider
    https:\/\/docs.scrapy.org\/en\/latest\/topics\/downloader-middleware.html#scrapy.downloadermiddlewares.DownloaderMiddleware.process_response <\/div>\n<\/p><\/div>\n<\/li>\n

  • \n
    \n
    \n
    \u4e3b<\/span> \u8cc7\u6df1\u5927\u4f6c : 13936 <\/span> <\/div>\n
    <\/i> <\/span> <\/div>\n<\/p><\/div>\n
    @peonone \u8c22\u8c22\uff0c\u6587\u6863\u662f\u7ffb\u4e86\u7684\u3002\u4f46\u662f\u76ee\u524d\u4e3b\u8981\u9700\u8981\u89e3\u51b3\u6389\u95ee\u9898\uff0c\u4e00\u70b9\u4e00\u70b9\u4ece\u5934\u7406\u89e3\u900f\u5f7b scrapy \u592a\u8017\u65f6\u95f4\u4e86\u3002\u8c22\u8c22\u7b54\u590d\u5566\u3002 <\/div>\n<\/p><\/div>\n<\/li>\n
  • \n","protected":false},"excerpt":{"rendered":"

    \u5173\u4e8e scrapy \u7684\u51e0\u4e2a\u95ee\u9898 \u8cc7…<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/140202"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=140202"}],"version-history":[{"count":0,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/140202\/revisions"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=140202"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=140202"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=140202"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}