{"id":131894,"date":"2020-07-06T07:57:04","date_gmt":"2020-07-05T23:57:04","guid":{"rendered":"http:\/\/4563.org\/?p=131894"},"modified":"2020-07-06T07:57:04","modified_gmt":"2020-07-05T23:57:04","slug":"pyspark-%e8%ae%be%e7%bd%aepython%e7%9a%84%e7%89%88%e6%9c%ac","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=131894","title":{"rendered":"PySpark \u8bbe\u7f6epython\u7684\u7248\u672c"},"content":{"rendered":"<p>PySpark \u5728\u5de5\u4f5c\u7a0b\u5e8f\u4e2d\u90fd\u9700\u8981\u4f7f\u7528\u7684\u4e3a\u9ed8\u8ba4\u73af\u5883\u7684python\u7248\u672c\uff0c\u600e\u6837\u628apython\u7684\u7248\u672c\u5207\u6362\u62103\u7684\u7248\u672c\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7 <code>PYSPARK_PYTHON<\/code> \u6307\u5b9a\u8981\u4f7f\u7528\u7684Python\u7248\u672c\u3002<\/p>\n<p>\u5177\u4f53\u6b65\u9aa4\u5982\u4e0b\uff1a<\/p>\n<ul>\n<li>\n<p>python3 \u73af\u5883\u9700\u8981\u63d0\u524d\u5b89\u88c5\u597d\uff0c\u5982\u679c\u6ca1\u6309\u7167\u53ef\u4ee5\u53c2\u8003<a href=\"https:\/\/www.studytime.xin\/python\/2020\/01\/20\/python-centos7-install.html\">centos7 \u4e0bpython2\u4e0epython3\u5171\u5b58<\/a><\/p>\n<\/li>\n<li>\n<p>\u4fee\u6539spark-env.sh\u6587\u4ef6, \u5728\u672b\u5c3e\u6dfb\u52a0<code>export PYSPARK_PYTHON=\/usr\/bin\/python3<\/code><\/p>\n<\/li>\n<li>\n<p>\u82e5\u4e3a\u96c6\u7fa4\u73af\u5883\uff0c\u9700\u8981\u628a\u4fee\u6539\u540e\u7684 spark-env.sh \u5206\u53d1\u5230\u5176\u4ed6\u5b50\u8282\u70b9\u7684spark\u5b89\u88c5\u5305\u4e0b\u7684conf\u76ee\u5f55\u4e0b<\/p>\n<\/li>\n<li>\n<p>\u4fee\u6539spark\u5b89\u88c5\u5305bin\u76ee\u5f55\u4e0b\u7684pyspark\uff0c\u4fee\u6539\u4e0b\u56fe\u7ea2\u8272\u65b9\u6846\u7684\u4f4d\u7f6e\uff0c\u5c06\u539f\u6765PYSPARK_PYTHON=python\u6539\u6210PYSPARK_PYTHON=python3\uff0c\u540c\u6837\u7684\uff0c\u5176\u4ed6\u5b50\u8282\u70b9\u4e5f\u90fd\u9700\u8981\u4fee\u6539<\/p>\n<\/li>\n<\/ul>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/07\/20200706_5f0355df3c562.png\" alt=\"PySpark \u8bbe\u7f6epython\u7684\u7248\u672c\"><\/p>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>if [[ -z &quot;$PYSPARK_PYTHON&quot; ]]; then<\/span><br><span>  if [[ $PYSPARK_DRIVER_PYTHON &#x3D;&#x3D; *ipython* &amp;&amp; ! $WORKS_WITH_IPYTHON ]]; then<\/span><br><span>    echo &quot;IPython requires Python 2.7+; please install python2.7 or set PYSPARK_PYTHON&quot; 1&gt;&amp;2<\/span><br><span>    exit 1<\/span><br><span>  else<\/span><br><span>    PYSPARK_PYTHON&#x3D;python3<\/span><br><span>  fi<\/span><br><span>fi<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<ul>\n<li>\u91cd\u542fSpark\uff0c\u542f\u52a8pyspark,\u53ef\u53d1\u73b0python\u7684\u7248\u672c\u5df2\u5207\u6362\u62103.6.4\u7684\u7248\u672c<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>PySpark \u5728\u5de5\u4f5c\u7a0b\u5e8f\u4e2d\u90fd\u9700\u8981&hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/131894"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=131894"}],"version-history":[{"count":0,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/131894\/revisions"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=131894"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=131894"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=131894"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}