{"id":142022,"date":"2020-08-02T22:51:39","date_gmt":"2020-08-02T14:51:39","guid":{"rendered":"http:\/\/4563.org\/?p=142022"},"modified":"2020-08-02T22:57:18","modified_gmt":"2020-08-02T14:57:18","slug":"spark%e7%b3%bb%e5%88%97%ef%bc%88%e4%b8%89%ef%bc%89spark%e9%9b%86%e7%be%a4%e5%ae%89%e8%a3%85%e9%83%a8%e7%bd%b2","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=142022","title":{"rendered":"Spark\u7cfb\u5217\uff08\u4e09\uff09:Spark\u96c6\u7fa4\u5b89\u88c5\u90e8\u7f72"},"content":{"rendered":"<h2><a href=\"https:\/\/www.studytime.xin\/article\/#\u5b89\u88c5\u57fa\u7840\" title=\"\u5b89\u88c5\u57fa\u7840\"><\/a>\u5b89\u88c5\u57fa\u7840<\/h2>\n<ul>\n<li>Java8\u5b89\u88c5\u6210\u529f<\/li>\n<li>zookeeper\u5b89\u88c5\u6210\u529f<\/li>\n<\/ul>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u4e0b\u8f7d\u5b89\u88c5\u5305\" title=\"\u4e0b\u8f7d\u5b89\u88c5\u5305\"><\/a>\u4e0b\u8f7d\u5b89\u88c5\u5305<\/h3>\n<ul>\n<li><a href=\"https:\/\/archive.apache.org\/dist\/spark\/spark-2.3.3\/spark-2.3.3-bin-hadoop2.7.tgz\">spark\u5b98\u7f51\u4e0b\u8f7d\u94fe\u63a5<\/a><\/li>\n<li><a href=\"https:\/\/pan.baidu.com\/s\/1ytjRn231Gx3RFDSncrj5qQ\">\u4f5c\u8005\u767e\u5ea6\u7f51\u76d8\u94fe\u63a5: https:\/\/pan.baidu.com\/s\/1ytjRn231Gx3RFDSncrj5qQ  \u5bc6\u7801: 77tm<\/a><\/li>\n<\/ul>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u4e0a\u4f20\u5b89\u88c5\u5305\u5230\u670d\u52a1\u5668\" title=\"\u4e0a\u4f20\u5b89\u88c5\u5305\u5230\u670d\u52a1\u5668\"><\/a>\u4e0a\u4f20\u5b89\u88c5\u5305\u5230\u670d\u52a1\u5668<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>cd &#x2F;opt&#x2F;software&#x2F;<\/span><br><span><\/span><br><span>[hadoop@node01 software]$ ls<\/span><br><span>3.51.0.tar.gz                                          clickhouse-server-19.16.10.44-1.el7.x86_64.rpm         spark-2.3.3-bin-hadoop2.7.tgz<\/span><br><span>clickhouse-client-19.16.10.44-1.el7.x86_64.rpm         clickhouse-server-common-19.16.10.44-1.el7.x86_64.rpm  zookeeper-3.4.5-cdh5.14.2.tar.gz<\/span><br><span>clickhouse-common-static-19.16.10.44-1.el7.x86_64.rpm  hbase-1.2.0-cdh5.14.2.tar.gz<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u89e3\u538b\u5b89\u88c5\u5305\u5230\u6307\u5b9a\u76ee\u5f55\" title=\"\u89e3\u538b\u5b89\u88c5\u5305\u5230\u6307\u5b9a\u76ee\u5f55\"><\/a>\u89e3\u538b\u5b89\u88c5\u5305\u5230\u6307\u5b9a\u76ee\u5f55<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>tar -zxvf spark-2.3.3-bin-hadoop2.7.tgz -C &#x2F;opt&#x2F;module&#x2F;<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\" title=\"\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\"><\/a>\u4fee\u6539\u914d\u7f6e\u6587\u4ef6<\/h3>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u4fee\u6539spark-env-sh\" title=\"\u4fee\u6539spark-env.sh\"><\/a>\u4fee\u6539spark-env.sh<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>cd &#x2F;opt&#x2F;module&#x2F;spark-2.3.3-bin-hadoop2.7&#x2F;conf&#x2F;<\/span><br><span><\/span><br><span>cp spark-env.sh.template spark-env.sh<\/span><br><span>vim spark-env.sh<\/span><br><span><\/span><br><span>#\u914d\u7f6ejava\u7684\u73af\u5883\u53d8\u91cf<\/span><br><span>export JAVA_HOME&#x3D;&#x2F;usr&#x2F;java&#x2F;jdk1.8.0_211-amd64<\/span><br><span>#\u914d\u7f6ezk\u76f8\u5173\u4fe1\u606f<\/span><br><span>export SPARK_DAEMON_JAVA_OPTS&#x3D;&quot;-Dspark.deploy.recoveryMode&#x3D;ZOOKEEPER  -Dspark.deploy.zookeeper.url&#x3D;node01:2181,node02:2181,node03:2181  -Dspark.deploy.zookeeper.dir&#x3D;&#x2F;spark&quot;<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u4fee\u6539slave\" title=\"\u4fee\u6539slave\"><\/a>\u4fee\u6539slave<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>mv slaves.template slaves<\/span><br><span><\/span><br><span>#\u6307\u5b9aspark\u96c6\u7fa4\u7684worker\u8282\u70b9<\/span><br><span>node02<\/span><br><span>node03<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u5206\u53d1\u5b89\u88c5\u76ee\u5f55\u5230\u96c6\u7fa4\u5176\u4ed6\u673a\u5668\" title=\"\u5206\u53d1\u5b89\u88c5\u76ee\u5f55\u5230\u96c6\u7fa4\u5176\u4ed6\u673a\u5668\"><\/a>\u5206\u53d1\u5b89\u88c5\u76ee\u5f55\u5230\u96c6\u7fa4\u5176\u4ed6\u673a\u5668<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>scp -r &#x2F;opt&#x2F;module&#x2F;spark-2.3.3-bin-hadoop2.7 node02:&#x2F;opt&#x2F;module<\/span><br><span>scp -r &#x2F;opt&#x2F;module&#x2F;spark-2.3.3-bin-hadoop2.7 node03:&#x2F;opt&#x2F;module<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u4fee\u6539\u96c6\u7fa4spark\u73af\u5883\u53d8\u91cf\uff0c\u6bcf\u53f0\u673a\u5668\u90fd\u6267\u884c\" title=\"\u4fee\u6539\u96c6\u7fa4spark\u73af\u5883\u53d8\u91cf\uff0c\u6bcf\u53f0\u673a\u5668\u90fd\u6267\u884c\"><\/a>\u4fee\u6539\u96c6\u7fa4spark\u73af\u5883\u53d8\u91cf\uff0c\u6bcf\u53f0\u673a\u5668\u90fd\u6267\u884c<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>vim ~&#x2F;.bash_profile<\/span><br><span><\/span><br><span># spark<\/span><br><span>export SPARK_HOME&#x3D;&#x2F;opt&#x2F;module&#x2F;spark-2.3.3-bin-hadoop2.7<\/span><br><span>export PATH&#x3D;$PATH:$SPARK_HOME&#x2F;bin:$SPARK_HOME&#x2F;sbin<\/span><br><span>source  ~&#x2F;.bash_profile<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u542f\u52a8\u96c6\u7fa4\" title=\"\u542f\u52a8\u96c6\u7fa4\"><\/a>\u542f\u52a8\u96c6\u7fa4<\/h3>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u542f\u52a8zookeeper\u96c6\u7fa4\uff0c\u6bcf\u53f0\u673a\u5668\u6267\u884c\u6b64\u547d\u4ee4\" title=\"\u542f\u52a8zookeeper\u96c6\u7fa4\uff0c\u6bcf\u53f0\u673a\u5668\u6267\u884c\u6b64\u547d\u4ee4\"><\/a>\u542f\u52a8zookeeper\u96c6\u7fa4\uff0c\u6bcf\u53f0\u673a\u5668\u6267\u884c\u6b64\u547d\u4ee4<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>zkServer.sh start<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u542f\u52a8spark\u96c6\u7fa4\" title=\"\u542f\u52a8spark\u96c6\u7fa4\"><\/a>\u542f\u52a8spark\u96c6\u7fa4<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>$SPARK_HOME&#x2F;sbin&#x2F;start-all.sh<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u7279\u6b8a\u8bf4\u660e\uff1a\" title=\"\u7279\u6b8a\u8bf4\u660e\uff1a\"><\/a>\u7279\u6b8a\u8bf4\u660e\uff1a<\/h4>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u5728\u54ea\u91cc\u542f\u52a8\u8fd9\u4e2a\u811a\u672c\uff0c\u5c31\u5728\u5f53\u524d\u8be5\u673a\u5668\u542f\u52a8\u4e00\u4e2aMaster\u8fdb\u7a0b\uff0c\u6574\u4e2a\u96c6\u7fa4\u7684worker\u8fdb\u7a0b\u7684\u542f\u52a8\u7531slaves\u6587\u4ef6\u3002\" title=\"\u5728\u54ea\u91cc\u542f\u52a8\u8fd9\u4e2a\u811a\u672c\uff0c\u5c31\u5728\u5f53\u524d\u8be5\u673a\u5668\u542f\u52a8\u4e00\u4e2aMaster\u8fdb\u7a0b\uff0c\u6574\u4e2a\u96c6\u7fa4\u7684worker\u8fdb\u7a0b\u7684\u542f\u52a8\u7531slaves\u6587\u4ef6\u3002\"><\/a>\u5728\u54ea\u91cc\u542f\u52a8\u8fd9\u4e2a\u811a\u672c\uff0c\u5c31\u5728\u5f53\u524d\u8be5\u673a\u5668\u542f\u52a8\u4e00\u4e2aMaster\u8fdb\u7a0b\uff0c\u6574\u4e2a\u96c6\u7fa4\u7684worker\u8fdb\u7a0b\u7684\u542f\u52a8\u7531slaves\u6587\u4ef6\u3002<\/h4>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u540e\u671f\u53ef\u4ee5\u5728\u5176\u4ed6\u673a\u5668\u5355\u72ec\u5728\u542f\u52a8master\" title=\"\u540e\u671f\u53ef\u4ee5\u5728\u5176\u4ed6\u673a\u5668\u5355\u72ec\u5728\u542f\u52a8master\"><\/a>\u540e\u671f\u53ef\u4ee5\u5728\u5176\u4ed6\u673a\u5668\u5355\u72ec\u5728\u542f\u52a8master<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>$SPARK_HOME&#x2F;sbin&#x2F;start-master.sh<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u505c\u6b62\u96c6\u7fa4\" title=\"\u505c\u6b62\u96c6\u7fa4\"><\/a>\u505c\u6b62\u96c6\u7fa4<\/h3>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u5728\u5904\u4e8eactive-Master\u4e3b\u8282\u70b9\u6267\u884c\uff0c\u53ef\u4ee5\u5173\u95ed\u96c6\u7fa4\" title=\"\u5728\u5904\u4e8eactive Master\u4e3b\u8282\u70b9\u6267\u884c\uff0c\u53ef\u4ee5\u5173\u95ed\u96c6\u7fa4\"><\/a>\u5728\u5904\u4e8eactive Master\u4e3b\u8282\u70b9\u6267\u884c\uff0c\u53ef\u4ee5\u5173\u95ed\u96c6\u7fa4<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>$SPARK_HOME&#x2F;sbin&#x2F;stop-all.sh<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u5728\u5904\u4e8estandBy-Master\u4e3b\u8282\u70b9\u6267\u884c\" title=\"\u5728\u5904\u4e8estandBy Master\u4e3b\u8282\u70b9\u6267\u884c\"><\/a>\u5728\u5904\u4e8estandBy Master\u4e3b\u8282\u70b9\u6267\u884c<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>$SPARK_HOME&#x2F;sbin&#x2F;stop-master.sh<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u67e5\u770b\u8fdb\u7a0b\" title=\"\u67e5\u770b\u8fdb\u7a0b\"><\/a>\u67e5\u770b\u8fdb\u7a0b<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><span>10<\/span><br><span>11<\/span><br><span>12<\/span><br><span>13<\/span><br><span>14<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>[hadoop@node01 software]$ xcall jps<\/span><br><span>--------- node01 ----------<\/span><br><span>24070 Jps<\/span><br><span>2601 JobHistoryServer<\/span><br><span>23530 QuorumPeerMain<\/span><br><span>23612 Master<\/span><br><span>--------- node02 ----------<\/span><br><span>7073 Worker<\/span><br><span>7011 QuorumPeerMain<\/span><br><span>7464 Jps<\/span><br><span>--------- node03 ----------<\/span><br><span>1394 Jps<\/span><br><span>1061 Worker<\/span><br><span>1002 QuorumPeerMain<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#spark\u96c6\u7fa4\u7684web\u7ba1\u7406\u754c\u9762\" title=\"spark\u96c6\u7fa4\u7684web\u7ba1\u7406\u754c\u9762\"><\/a>spark\u96c6\u7fa4\u7684web\u7ba1\u7406\u754c\u9762<\/h3>\n<p>\u5f53\u542f\u52a8\u597dspark\u96c6\u7fa4\u4e4b\u540e\uff0c\u53ef\u4ee5\u8bbf\u95ee\u8fd9\u6837\u4e00\u4e2a\u5730\u5740\uff0c<a href=\"http:\/\/node01:8080\">http:\/\/node01:8080<\/a>\u3002<\/p>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/08\/20200802191751.png\" alt=\"Spark\u7cfb\u5217\uff08\u4e09\uff09:Spark\u96c6\u7fa4\u5b89\u88c5\u90e8\u7f72\"><br \/>\u53ef\u4ee5\u901a\u8fc7\u8fd9\u4e2aweb\u754c\u9762\u89c2\u5bdf\u5230\u5f88\u591a\u4fe1\u606f<\/p>\n<ul>\n<li>\u6574\u4e2aspark\u96c6\u7fa4\u7684\u8be6\u7ec6\u4fe1\u606f<\/li>\n<li>\u6574\u4e2aspark\u96c6\u7fa4\u603b\u7684\u8d44\u6e90\u4fe1\u606f<\/li>\n<li>\u6574\u4e2aspark\u96c6\u7fa4\u5df2\u7ecf\u4f7f\u7528\u7684\u8d44\u6e90\u4fe1\u606f<\/li>\n<li>\u6574\u4e2aspark\u96c6\u7fa4\u8fd8\u5269\u7684\u8d44\u6e90\u4fe1\u606f<\/li>\n<li>\u6574\u4e2aspark\u96c6\u7fa4\u6b63\u5728\u8fd0\u884c\u7684\u4efb\u52a1\u4fe1\u606f<\/li>\n<li>\u6574\u4e2aspark\u96c6\u7fa4\u5df2\u7ecf\u5b8c\u6210\u7684\u4efb\u52a1\u4fe1\u606f<\/li>\n<\/ul>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u9ad8\u53ef\u7528\u4f7f\u7528\u9a8c\u8bc1\" title=\"\u9ad8\u53ef\u7528\u4f7f\u7528\u9a8c\u8bc1\"><\/a>\u9ad8\u53ef\u7528\u4f7f\u7528\u9a8c\u8bc1<\/h3>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u542f\u52a8master\uff0c\u4f5c\u4e3astandby\" title=\"\u542f\u52a8master\uff0c\u4f5c\u4e3astandby\"><\/a>\u542f\u52a8master\uff0c\u4f5c\u4e3astandby<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>ssh node02<\/span><br><span>$SPARK_HOME&#x2F;sbin&#x2F;start-master.sh<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/08\/20200802_5f26d373524ec.png\" alt=\"Spark\u7cfb\u5217\uff08\u4e09\uff09:Spark\u96c6\u7fa4\u5b89\u88c5\u90e8\u7f72\"><\/p>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u624b\u52a8\u6740\u6b7bnode01\u4e0a\u9762\u7684Master\u8fdb\u7a0b\uff0c\u89c2\u5bdf\u662f\u5426\u4f1a\u81ea\u52a8\u8fdb\u884c\u5207\u6362\" title=\"\u624b\u52a8\u6740\u6b7bnode01\u4e0a\u9762\u7684Master\u8fdb\u7a0b\uff0c\u89c2\u5bdf\u662f\u5426\u4f1a\u81ea\u52a8\u8fdb\u884c\u5207\u6362\"><\/a>\u624b\u52a8\u6740\u6b7bnode01\u4e0a\u9762\u7684Master\u8fdb\u7a0b\uff0c\u89c2\u5bdf\u662f\u5426\u4f1a\u81ea\u52a8\u8fdb\u884c\u5207\u6362<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>[hadoop@node01 ~]$ jps<\/span><br><span>24263 Jps<\/span><br><span>2601 JobHistoryServer<\/span><br><span>23530 QuorumPeerMain<\/span><br><span>23612 Master<\/span><br><span>[hadoop@node01 ~]$ kill -9 23612<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p><strong>node01\u8282\u70b9<\/strong>\uff0c\u7531\u4e8eMaster\u8fdb\u7a0b\u88ab\u6740\u6b7b\uff0c\u6240\u4ee5\u754c\u9762\u65e0\u6cd5\u8bbf\u95ee<br \/><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/08\/20200802_5f26d3920fea1.png\" alt=\"Spark\u7cfb\u5217\uff08\u4e09\uff09:Spark\u96c6\u7fa4\u5b89\u88c5\u90e8\u7f72\"><\/p>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#node02\u8282\u70b9\uff0cMaster\u88ab\u5e72\u6389\u4e4b\u540e\uff0cnode02\u8282\u70b9\u4e0a\u7684Master\u6210\u529f\u7be1\u4f4d\u6210\u529f\uff0c\u6210\u4e3aALIVE\u72b6\u6001\" title=\"node02\u8282\u70b9\uff0cMaster\u88ab\u5e72\u6389\u4e4b\u540e\uff0cnode02\u8282\u70b9\u4e0a\u7684Master\u6210\u529f\u7be1\u4f4d\u6210\u529f\uff0c\u6210\u4e3aALIVE\u72b6\u6001\"><\/a><strong>node02\u8282\u70b9<\/strong>\uff0cMaster\u88ab\u5e72\u6389\u4e4b\u540e\uff0cnode02\u8282\u70b9\u4e0a\u7684Master\u6210\u529f\u7be1\u4f4d\u6210\u529f\uff0c\u6210\u4e3aALIVE\u72b6\u6001<\/h3>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/08\/20200802192008.png\" alt=\"Spark\u7cfb\u5217\uff08\u4e09\uff09:Spark\u96c6\u7fa4\u5b89\u88c5\u90e8\u7f72\"><\/p>\n<h2><a href=\"https:\/\/www.studytime.xin\/article\/#\u6267\u884cSpark\u7a0b\u5e8fon-standalone\" title=\"\u6267\u884cSpark\u7a0b\u5e8fon standalone\"><\/a>\u6267\u884cSpark\u7a0b\u5e8fon standalone<\/h2>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><span>10<\/span><br><span>11<\/span><br><span>12<\/span><br><span>13<\/span><br><span>14<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>bin&#x2F;spark-submit <\/span><br><span>--class org.apache.spark.examples.SparkPi <\/span><br><span>--master spark:&#x2F;&#x2F;node01:7077 <\/span><br><span>--executor-memory 1G <\/span><br><span>--total-executor-cores 2 <\/span><br><span>examples&#x2F;jars&#x2F;spark-examples_2.11-2.3.3.jar <\/span><br><span>10<\/span><br><span><\/span><br><span><\/span><br><span>####\u53c2\u6570\u8bf4\u660e<\/span><br><span>--class\uff1a\u6307\u5b9a\u5305\u542bmain\u65b9\u6cd5\u7684\u4e3b\u7c7b<\/span><br><span>--master\uff1a\u6307\u5b9aspark\u96c6\u7fa4master\u5730\u5740<\/span><br><span>--executor-memory\uff1a\u6307\u5b9a\u4efb\u52a1\u5728\u8fd0\u884c\u7684\u65f6\u5019\u9700\u8981\u7684\u6bcf\u4e00\u4e2aexecutor\u5185\u5b58\u5927\u5c0f<\/span><br><span>--total-executor-cores\uff1a \u6307\u5b9a\u4efb\u52a1\u5728\u8fd0\u884c\u7684\u65f6\u5019\u9700\u8981\u603b\u7684cpu\u6838\u6570<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/08\/20200802_5f26d41d3b64f.png\" alt=\"Spark\u7cfb\u5217\uff08\u4e09\uff09:Spark\u96c6\u7fa4\u5b89\u88c5\u90e8\u7f72\"><br \/><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/08\/20200802_5f26d44295f26.png\" alt=\"Spark\u7cfb\u5217\uff08\u4e09\uff09:Spark\u96c6\u7fa4\u5b89\u88c5\u90e8\u7f72\"><\/p>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#spark-shell\u542f\u52a8\u4ee5\u53ca\u4f7f\u7528\" title=\"spark shell\u542f\u52a8\u4ee5\u53ca\u4f7f\u7528\"><\/a>spark shell\u542f\u52a8\u4ee5\u53ca\u4f7f\u7528<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>bin&#x2F;spark-shell <\/span><br><span>--master spark:&#x2F;&#x2F;node01:7077 <\/span><br><span>--executor-memory 1G <\/span><br><span>--total-executor-cores 2 <\/span><br><span><\/span><br><span># \u53c2\u6570\u8bf4\u660e\uff1a<\/span><br><span>--master spark:&#x2F;&#x2F;node01:7077 \u6307\u5b9aMaster\u7684\u5730\u5740<\/span><br><span>--executor-memory 500m:\u6307\u5b9a\u6bcf\u4e2aworker\u53ef\u7528\u5185\u5b58\u4e3a1G<\/span><br><span>--total-executor-cores 2: \u6307\u5b9a\u6574\u4e2a\u96c6\u7fa4\u4f7f\u7528\u7684cup\u6838\u6570\u4e3a2\u4e2a<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p>\u7279\u8bf4\u8bf4\u660e\uff1a<br \/>\u5982\u679c\u542f\u52a8spark shell\u65f6\u6ca1\u6709\u6307\u5b9amaster\u5730\u5740\uff0c\u4f46\u662f\u4e5f\u53ef\u4ee5\u6b63\u5e38\u542f\u52a8spark shell\u548c\u6267\u884cspark shell\u4e2d\u7684\u7a0b\u5e8f\uff0c\u5176\u5b9e\u662f\u542f\u52a8\u4e86spark\u7684local\u6a21\u5f0f\uff0c\u8be5\u6a21\u5f0f\u4ec5\u5728\u672c\u673a\u542f\u52a8\u4e00\u4e2a\u8fdb\u7a0b\uff0c\u6ca1\u6709\u4e0e\u96c6\u7fa4\u5efa\u7acb\u8054\u7cfb\u3002<br \/>Spark Shell\u4e2d\u5df2\u7ecf\u9ed8\u8ba4\u5c06SparkContext\u7c7b\u521d\u59cb\u5316\u4e3a\u5bf9\u8c61sc\u3002\u7528\u6237\u4ee3\u7801\u5982\u679c\u9700\u8981\u7528\u5230\uff0c\u5219\u76f4\u63a5\u5e94\u7528sc\u5373\u53ef<br \/>Spark Shell\u4e2d\u5df2\u7ecf\u9ed8\u8ba4\u5c06SparkSQl\u7c7b\u521d\u59cb\u5316\u4e3a\u5bf9\u8c61spark\u3002\u7528\u6237\u4ee3\u7801\u5982\u679c\u9700\u8981\u7528\u5230\uff0c\u5219\u76f4\u63a5\u5e94\u7528spark\u5373\u53ef<\/p>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#spark-shell-\u2013master-local-N-\u8bfb\u53d6HDFS\u4e0a\u6587\u4ef6\u8fdb\u884c\u5355\u8bcd\u7edf\u8ba1\" title=\"spark-shell \u2013master local[N] \u8bfb\u53d6HDFS\u4e0a\u6587\u4ef6\u8fdb\u884c\u5355\u8bcd\u7edf\u8ba1\"><\/a>spark-shell \u2013master local[N] \u8bfb\u53d6HDFS\u4e0a\u6587\u4ef6\u8fdb\u884c\u5355\u8bcd\u7edf\u8ba1<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><span>10<\/span><br><span>11<\/span><br><span>12<\/span><br><span>13<\/span><br><span>14<\/span><br><span>15<\/span><br><span>16<\/span><br><span>17<\/span><br><span>18<\/span><br><span>19<\/span><br><span>20<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>cd &#x2F;home&#x2F;hadoop<\/span><br><span><\/span><br><span>vim hello.txt<\/span><br><span><\/span><br><span>you,jump<\/span><br><span>i,jump<\/span><br><span>you,jump<\/span><br><span>i,jump<\/span><br><span>jump<\/span><br><span><\/span><br><span>hdfs dfs -mkdir -p &#x2F;spark<\/span><br><span>hdfs dfs -put hello.txt &#x2F;spark<\/span><br><span><\/span><br><span># \u8fdb\u5165spark shell<\/span><br><span><\/span><br><span>scala&gt; sc.textFile(&quot;file:&#x2F;&#x2F;&#x2F;home&#x2F;hadoop&#x2F;hello.txt&quot;).flatMap(x&#x3D;&gt;x.split(&quot; &quot;)).map(x&#x3D;&gt;(x,1)).reduceByKey((x,y)&#x3D;&gt;x+y).collect<\/span><br><span>res7: Array[(String, Int)] &#x3D; Array((wq,1), (rt,1), (ew,1), (&quot;&quot;,2), (ewr,1), (qwr,1), (qw,1), (e,1), (er,1), (qweqw,2))<\/span><br><span><\/span><br><span>scala&gt; sc.textFile(&quot;hdfs:&#x2F;&#x2F;node01:8020&#x2F;spark&#x2F;hello.txt&quot;).flatMap(x&#x3D;&gt;x.split(&quot; &quot;)).map(x&#x3D;&gt;(x,1)).reduceByKey((x,y)&#x3D;&gt;x+y).collect<\/span><br><span>res6: Array[(String, Int)] &#x3D; Array((jump,1), (you,jump,2), (i,jump,2))<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u6269\u5c55\u77e5\u8bc6\u70b9\" title=\"\u6269\u5c55\u77e5\u8bc6\u70b9\"><\/a>\u6269\u5c55\u77e5\u8bc6\u70b9<\/h3>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u5982\u4f55\u6062\u590d\u5230\u4e0a\u4e00\u6b21\u6d3b\u7740master\u6302\u6389\u4e4b\u524d\u7684\u72b6\u6001\" title=\"\u5982\u4f55\u6062\u590d\u5230\u4e0a\u4e00\u6b21\u6d3b\u7740master\u6302\u6389\u4e4b\u524d\u7684\u72b6\u6001?\"><\/a>\u5982\u4f55\u6062\u590d\u5230\u4e0a\u4e00\u6b21\u6d3b\u7740master\u6302\u6389\u4e4b\u524d\u7684\u72b6\u6001?<\/h3>\n<p>\u5728\u9ad8\u53ef\u7528\u6a21\u5f0f\u4e0b\uff0c\u6574\u4e2aspark\u96c6\u7fa4\u5c31\u6709\u5f88\u591a\u4e2amaster\uff0c\u5176\u4e2d\u53ea\u6709\u4e00\u4e2amaster\u88abzk\u9009\u4e3e\u6210\u6d3b\u7740\u7684master\uff0c\u5176\u4ed6\u7684\u591a\u4e2amaster\u90fd\u5904\u4e8estandby\uff0c\u540c\u65f6\u628a\u6574\u4e2aspark\u96c6\u7fa4\u7684\u5143\u6570\u636e\u4fe1\u606f\u901a\u8fc7zk\u4e2d\u8282\u70b9\u8fdb\u884c\u4fdd\u5b58\u3002<br \/>    \u540e\u671f\u5982\u679c\u6d3b\u7740\u7684master\u6302\u6389\u3002\u9996\u5148zk\u4f1a\u611f\u77e5\u5230\u6d3b\u7740\u7684master\u6302\u6389\uff0c\u4e0b\u9762\u5f00\u59cb\u5728\u591a\u4e2a\u5904\u4e8estandby\u4e2d\u7684master\u8fdb\u884c\u9009\u4e3e\uff0c\u518d\u6b21\u4ea7\u751f\u4e00\u4e2a\u6d3b\u7740\u7684master\uff0c\u8fd9\u4e2a\u6d3b\u7740\u7684master\u4f1a\u8bfb\u53d6\u4fdd\u5b58\u5728zk\u8282\u70b9\u4e2d\u7684spark\u96c6\u7fa4\u5143\u6570\u636e\u4fe1\u606f\uff0c\u6062\u590d\u5230\u4e0a\u4e00\u6b21master\u7684\u72b6\u6001\u3002\u6574\u4e2a\u8fc7\u7a0b\u5728\u6062\u590d\u7684\u65f6\u5019\u7ecf\u5386\u8fc7\u4e86\u5f88\u591a\u4e2a\u4e0d\u540c\u7684\u9636\u6bb5\uff0c\u6bcf\u4e2a\u9636\u6bb5\u90fd\u9700\u8981\u4e00\u5b9a\u65f6\u95f4\uff0c\u6700\u7ec8\u6062\u590d\u5230\u4e0a\u4e2a\u6d3b\u7740\u7684master\u7684\u8f6c\u6001\uff0c\u6574\u4e2a\u6062\u590d\u8fc7\u7a0b\u4e00\u822c\u9700\u89811-2\u5206\u949f\u3002<\/p>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#\u5728master\u7684\u6062\u590d\u9636\u6bb5\u5bf9\u4efb\u52a1\u7684\u5f71\u54cd\" title=\"\u5728master\u7684\u6062\u590d\u9636\u6bb5\u5bf9\u4efb\u52a1\u7684\u5f71\u54cd?\"><\/a>\u5728master\u7684\u6062\u590d\u9636\u6bb5\u5bf9\u4efb\u52a1\u7684\u5f71\u54cd?<\/h4>\n<p>\u5bf9\u5df2\u7ecf\u8fd0\u884c\u7684\u4efb\u52a1\u662f\u6ca1\u6709\u4efb\u4f55\u5f71\u54cd\uff0c\u7531\u4e8e\u8be5\u4efb\u52a1\u6b63\u5728\u8fd0\u884c\uff0c\u8bf4\u660e\u5b83\u5df2\u7ecf\u62ff\u5230\u4e86\u8ba1\u7b97\u8d44\u6e90\uff0c\u8fd9\u4e2a\u65f6\u5019\u5c31\u4e0d\u9700\u8981master\u3002<br \/>\u5bf9\u5373\u5c06\u8981\u63d0\u4ea4\u7684\u4efb\u52a1\u662f\u6709\u5f71\u54cd\uff0c\u7531\u4e8e\u8be5\u4efb\u52a1\u9700\u8981\u6709\u8ba1\u7b97\u8d44\u6e90\uff0c\u8fd9\u4e2a\u65f6\u5019\u4f1a\u627e\u6d3b\u7740\u7684master\u53bb\u7533\u8bf7\u8ba1\u7b97\u8d44\u6e90\uff0c\u7531\u4e8e\u6ca1\u6709\u4e00\u4e2a\u6d3b\u7740\u7684master,\u8be5\u4efb\u52a1\u662f\u83b7\u53d6\u4e0d\u5230\u8ba1\u7b97\u8d44\u6e90\uff0c\u4e5f\u5c31\u662f\u4efb\u52a1\u65e0\u6cd5\u8fd0\u884c\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u5b89\u88c5\u57fa\u7840 Java8\u5b89\u88c5\u6210\u529f zo&hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/142022"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=142022"}],"version-history":[{"count":1,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/142022\/revisions"}],"predecessor-version":[{"id":142025,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/142022\/revisions\/142025"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=142022"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=142022"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=142022"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}