{"id":99588,"date":"2020-04-16T09:24:34","date_gmt":"2020-04-16T01:24:34","guid":{"rendered":"http:\/\/4563.org\/?p=99588"},"modified":"2020-04-16T09:24:34","modified_gmt":"2020-04-16T01:24:34","slug":"pyspark-%e8%bf%9e%e6%8e%a5-spark-%e9%9b%86%e7%be%a4%e9%83%a8%e7%bd%b2%e9%97%ae%e9%a2%98","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=99588","title":{"rendered":"pyspark \u8fde\u63a5 spark \u96c6\u7fa4\u90e8\u7f72\u95ee\u9898"},"content":{"rendered":"<div>\n<div>\n<div>\n<h1>                  pyspark \u8fde\u63a5 spark \u96c6\u7fa4\u90e8\u7f72\u95ee\u9898               <\/h1>\n<p> <\/p>\n<div>\n<div> <span>\u8cc7\u6df1\u5927\u4f6c : marin1995 <\/span>  <span><i><\/i> 54<\/span> <\/div>\n<div> <\/div>\n<\/p><\/div>\n<\/p><\/div>\n<\/p><\/div>\n<div isfirst=\"1\">                        \u8bf7\u6559\u4e00\u4e0b\u5404\u4f4d\u5927\u4f6c\uff0c\u6211\u73b0\u5728\u4f7f\u7528 pysaprk \u505a sparkstreaming \u5904\u7406 kafka \u7684\u6570\u636e\u3002\u7a0b\u5e8f\u5355\u673a\u8dd1\u7684\u65f6\u5019\u6ca1\u6709\u95ee\u9898\uff0c\u8fde\u63a5\u96c6\u7fa4\u7684\u65f6\u5019\u5c31\u4f1a\u62a5\u9519 no module named &#8220;xxxx&#8221;\uff0c\u8fde\u63a5\u65b9\u5f0f\u662f\u5728\u7a0b\u5e8f\u662f\u8bbe\u7f6e setMaster(&#8220;&#8221;spark:\/\/host:port)\u3002\u662f\u4e0d\u662f\u8fd8\u8981\u914d\u7f6e\u5176\u4ed6\u73af\u5883\u53d8\u91cf\u5462\uff1f\u9ebb\u70e6\u5927\u4f6c\u6307\u70b9\u4e00\u4e0b\uff0c\u6709\u507f~      <\/div>\n<div> <b>\u5927\u4f6c\u6709\u8a71\u8aaa<\/b> (<span>3<\/span>)        <\/div>\n<div> <\/div>\n<\/p><\/div>\n<\/p><\/div>\n<ul>\n<li data-pid=\"753419\" data-uid=\"2\">\n<div>\n<div>\n<div> <span>\u8cc7\u6df1\u5927\u4f6c : besttime <\/span>  <\/div>\n<div> <i title=\"\u5f15\u7528\"><\/i>  <span>          <\/span> <\/div>\n<\/p><\/div>\n<div>                                                             pyspark \u7684 job \u63d0\u4ea4\u5230 spark \u540e\uff0c\u4f1a\u8c03\u7528 spark \u6240\u5728\u73af\u5883\u7684 python \u89e3\u91ca\u5668\uff0c\u800c\u8fd9\u4e2a\u62a5\u9519\u7684\u95ee\u9898\u5c31\u662f\u5728\u4e8e spark \u6240\u5728\u7684\u73af\u5883\u7684 python \u89e3\u91ca\u5668\u7f3a\u4e4f\u4f60\u4f7f\u7528\u7684\u7c7b\u5e93\u3002                                                            <\/div>\n<\/p><\/div>\n<\/li>\n<li data-pid=\"753420\" data-uid=\"2\">\n<div>\n<div>\n<div> <span>\u8cc7\u6df1\u5927\u4f6c : fzy0728 <\/span>  <\/div>\n<div> <i title=\"\u5f15\u7528\"><\/i>  <span>          <\/span> <\/div>\n<\/p><\/div>\n<div>                                                             https:\/\/www.jianshu.com\/p\/df0a189ff28b                                                            <\/div>\n<\/p><\/div>\n<\/li>\n<li data-pid=\"753421\" data-uid=\"2\">\n<div>\n<div>\n<div> <span>\u8cc7\u6df1\u5927\u4f6c : xiaoxuxu <\/span>  <\/div>\n<div> <i title=\"\u5f15\u7528\"><\/i>  <span>          <\/span> <\/div>\n<\/p><\/div>\n<div>                                                             \u53ef\u4ee5\u7528 pyenv \u5efa\u4e2a Python \u73af\u5883\uff0c\u7136\u540e\u6253\u6210\u538b\u7f29\u5305\u3002\u8fd0\u884c spark-submit \u7684\u65f6\u5019\u7528&#8211;archives \u4f20\u8fd9\u4e2a\u538b\u7f29\u5305\uff0c\u5e76\u4e14\u8bbe\u7f6e PYSPARK_DRIVER_PYTHON\u3001PYSPARK_PYTHON \u4e3a\u538b\u7f29\u5305\u5185 Python \u53ef\u6267\u884c\u6587\u4ef6\u7684\u76f8\u5bf9\u8def\u5f84\uff0c\u5927\u6982\u662f\u8fd9\u6837\uff1a<\/p>\n<p>PYSPARK_PYTHON=.\/python-env\/bin\/python PYSPARK_DRIVER_PYTHON=.\/python-env\/bin\/python spark-submit &#8211;master yarn &#8211;deploy-mode cluster &#8211;archives python-env.zip#python-env                                                            <\/p><\/div>\n<\/p><\/div>\n<\/li>\n<li>\n","protected":false},"excerpt":{"rendered":"<p>pyspark \u8fde\u63a5 spark &hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/99588"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=99588"}],"version-history":[{"count":0,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/99588\/revisions"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=99588"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=99588"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=99588"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}