{"id":85700,"date":"2020-05-13T11:29:06","date_gmt":"2020-05-13T03:29:06","guid":{"rendered":"http:\/\/4563.org\/?p=85700"},"modified":"2020-05-13T11:29:06","modified_gmt":"2020-05-13T03:29:06","slug":"python-asyncio-%e9%97%ae%e9%a2%98","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=85700","title":{"rendered":"Python asyncio \u95ee\u9898"},"content":{"rendered":"<div>\n<div>\n<div>\n<h1>                  Python asyncio \u95ee\u9898               <\/h1>\n<p> <\/p>\n<div>\n<div> <span>\u8cc7\u6df1\u5927\u4f6c : fangwenxue <\/span>  <span><i><\/i> 3<\/span> <\/div>\n<div> <\/div>\n<\/p><\/div>\n<\/p><\/div>\n<\/p><\/div>\n<div isfirst=\"1\"> <\/p>\n<pre><code>    def crawl(self, tasks: list, task_handler=None, callback=None, semaphore_count=100):         if not task_handler:             task_handler = self.page_handler          main_loop = asyncio.new_event_loop()         asyncio.set_event_loop(main_loop)         semaphore = asyncio.Semaphore(semaphore_count)         n = len(tasks)          asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())          new_tasks = []          async def _run():             async with semaphore:                 async with ClientSession() as session:                     for i, task in enumerate(tasks):                         future = asyncio.ensure_future(task_handler(task, i=i + 1, n=n, session=session))                         if callback:                             future.add_done_callback(callback)                         new_tasks.append(future)                     return await asyncio.gather(*new_tasks)          try:             result = main_loop.run_until_complete(_run())             return [x for j in result for x in j]         except Exception as e:             logging.exception(e)         finally:             if not main_loop.is_closed():                 main_loop.close()             return []                  async def page_handler(self, task, session, **kwargs):         ''' \u7701\u7565\u90e8\u5206\u4ee3\u7801 '''         result = self.crawl(book_item_url_list, self.item_handler)         return result  <\/code><\/pre>\n<ul>\n<li>\n<p>\u600e\u6837\u521b\u5efa\u4e00\u4e2a\u552f\u4e00 ClientSession()<\/p>\n<ul>\n<li>session = ClientSession() \u8fd9\u6837\u4e0d\u597d\u4f7f<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u5728 page_handler \u518d\u6b21\u6761\u7528 crawl \u51fa\u73b0<\/p>\n<ul>\n<li>Cannot run the event loop while another loop is running<\/li>\n<li>crawl \u91cc\u9762\u7684\u4efb\u52a1\u4f1a\u5728\u4ea7\u751f\u5b50\u4efb\u52a1\uff0c \u8fd9\u4e2a\u5b50\u4efb\u52a1\u5982\u4f55\u7528 loop \u52a0\u5165\u4efb\u52a1\u5217\u8868<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u6211\u4f20\u4e86 callback \u53c8\u7528\u4e86 return \u8fd9\u662f\u4e0d\u662f\u53c8\u70b9\u77db\u76fe<\/p>\n<\/li>\n<li>\n<p>\u6211\u6253\u5370 i\/n \u8fd9\u4e2a i \u662f\u4e71\u5e8f \u600e\u6837\u4efb\u52a1\u5217\u8868\u987a\u5e8f\u6267\u884c<\/p>\n<\/li>\n<\/ul><\/div>\n<div> <b>\u5927\u4f6c\u6709\u8a71\u8aaa<\/b> (<span>1<\/span>)        <\/div>\n<div> <\/div>\n<\/p><\/div>\n<\/p><\/div>\n<ul>\n<li data-pid=\"1499090\" data-uid=\"2\">\n<div>\n<div>\n<div> <span>\u8cc7\u6df1\u5927\u4f6c : sikong31 <\/span>  <\/div>\n<div> <i title=\"\u5f15\u7528\"><\/i>  <span>          <\/span> <\/div>\n<\/p><\/div>\n<div>                                                             1 \u552f\u4e00\u7684 ClientSession\uff0c\u5c31\u81ea\u5df1\u5b9a\u4e49\u4e00\u4e2a AppSession \u7c7b\uff0c\u8fd4\u56de\u4e00\u4e2a\u7c7b\u5c5e\u6027\u5c31\u662f\u4e86<br \/>2 \u518d\u8c03\u7528\u5c31\u76f8\u5f53\u4e8e\u963b\u585e\u4e86\uff0c\u4f60\u53ef\u4ee5\u53c2\u8003\u4e0b asyncio \u52a8\u6001\u6dfb\u52a0\u4efb\u52a1<br \/>4 \u5982\u679c\u4e0d\u662f\u7b49\u6700\u540e\u4e00\u8d77\u5904\u7406\uff0c\u5217\u8868\u662f\u6ca1\u529e\u6cd5\u987a\u5e8f\u8fd4\u56de\u7684\uff0c\u56e0\u4e3a\u54ea\u4e2a\u8bf7\u6c42\u5148\u8fd4\u56de\u5e76\u4e0d\u662f\u4f60\u8fd9\u8fb9\u63a7\u5236\u7684\uff0c\u53ea\u6709\u7ed3\u679c\u5168\u90e8\u8fd4\u56de\u4e86\u624d\u662f\u987a\u5e8f\u7684                                                            <\/div>\n<\/p><\/div>\n<\/li>\n<li>\n","protected":false},"excerpt":{"rendered":"<p>Python asyncio \u95ee\u9898&hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/85700"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=85700"}],"version-history":[{"count":0,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/85700\/revisions"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=85700"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=85700"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=85700"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}