{"id":131837,"date":"2020-07-05T10:02:07","date_gmt":"2020-07-05T02:02:07","guid":{"rendered":"http:\/\/4563.org\/?p=131837"},"modified":"2020-07-05T10:02:07","modified_gmt":"2020-07-05T02:02:07","slug":"spark%e5%ad%a6%e4%b9%a0%e4%b9%8b%e8%b7%af%ef%bc%88%e4%ba%8c%ef%bc%89spark%e4%bc%aa%e5%88%86%e5%b8%83%e5%bc%8f%e5%ae%89%e8%a3%85","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=131837","title":{"rendered":"Spark\u5b66\u4e60\u4e4b\u8def\uff08\u4e8c\uff09:Spark\u4f2a\u5206\u5e03\u5f0f\u5b89\u88c5"},"content":{"rendered":"<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u4e0b\u8f7d-Spark-\u5b89\u88c5\u5305\" title=\"\u4e0b\u8f7d Spark \u5b89\u88c5\u5305\"><\/a>\u4e0b\u8f7d Spark \u5b89\u88c5\u5305<\/h3>\n<p><a href=\"http:\/\/spark.apache.org\/downloads.html\">\u5b98\u7f51\u4e0b\u8f7d http:\/\/spark.apache.org\/downloads.html<\/a><\/p>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/07\/20200706_5f0354849b6f7.png\" alt=\"Spark\u5b66\u4e60\u4e4b\u8def\uff08\u4e8c\uff09:Spark\u4f2a\u5206\u5e03\u5f0f\u5b89\u88c5\"><\/p>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u5b89\u88c5\u524d\u51c6\u5907\" title=\"\u5b89\u88c5\u524d\u51c6\u5907\"><\/a>\u5b89\u88c5\u524d\u51c6\u5907<\/h3>\n<ul>\n<li>Java8 \u5df2\u5b89\u88c5<\/li>\n<li>hadoop2.7.5 \u5df2\u5b89\u88c5<\/li>\n<\/ul>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u4fee\u6539-Hadoop-\u914d\u7f6e\u6587\u4ef6\" title=\"\u4fee\u6539 Hadoop \u914d\u7f6e\u6587\u4ef6\"><\/a>\u4fee\u6539 Hadoop \u914d\u7f6e\u6587\u4ef6<\/h3>\n<p>\u4fee\u6539 Hadoop yarn-site.xml\u914d\u7f6e<\/p>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><span>10<\/span><br><span>11<\/span><br><span>12<\/span><br><span>13<\/span><br><span>14<\/span><br><span>15<\/span><br><span>16<\/span><br><span>17<\/span><br><span>18<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>vim ~&#x2F;App&#x2F;hadoop-2.7.3&#x2F;etc&#x2F;hadoop&#x2F;yarn-site.xml<\/span><br><span><\/span><br><span>&lt;property&gt;<\/span><br><span>    &lt;name&gt;yarn.log-aggregation-enable&lt;&#x2F;name&gt; <\/span><br><span>    &lt;value&gt;true&lt;&#x2F;value&gt; <\/span><br><span>&lt;&#x2F;property&gt; <\/span><br><span>&lt;property&gt; <\/span><br><span>    &lt;name&gt;yarn.log.server.url&lt;&#x2F;name&gt; <\/span><br><span>    &lt;value&gt;http:&#x2F;&#x2F;bigdata:19888&#x2F;jobhistory&#x2F;logs&lt;&#x2F;value&gt; <\/span><br><span>&lt;&#x2F;property&gt; <\/span><br><span>&lt;property&gt; <\/span><br><span>    &lt;name&gt;yarn.nodemanager.pmem-check-enabled&lt;&#x2F;name&gt; <\/span><br><span>    &lt;value&gt;false&lt;&#x2F;value&gt; <\/span><br><span>&lt;&#x2F;property&gt; <\/span><br><span>&lt;property&gt; <\/span><br><span>    &lt;name&gt;yarn.nodemanager.vmem-check-enabled&lt;&#x2F;name&gt; <\/span><br><span>    &lt;value&gt;false&lt;&#x2F;value&gt;<\/span><br><span>&lt;&#x2F;property&gt;<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p>\u91cd\u542fyarn\u670d\u52a1<\/p>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>stop-yarn.sh <\/span><br><span>start-yarn.sh<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p>\u542f\u52a8 MapReduce History Server<\/p>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>sbin&#x2F;mr-jobhistory-daemon.sh start historyserver<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p>\u5728\u6d4f\u89c8\u5668\u4e2d\u6253\u5f00 MapReduce history server \u5730\u5740 <\/p>\n<p><a href=\"http:\/\/bigdata:19888\">http:\/\/bigdata:19888<\/a><\/p>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#Spark-\u5b89\u88c5\u3001\u914d\u7f6e\" title=\"Spark \u5b89\u88c5\u3001\u914d\u7f6e\"><\/a>Spark \u5b89\u88c5\u3001\u914d\u7f6e<\/h3>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#1-\u89e3\u538b\u7f29-spark-2-1-0-bin-hadoop2-7-tgz\" title=\"1.\u89e3\u538b\u7f29 spark-2.1.0-bin-hadoop2.7.tgz\"><\/a>1.\u89e3\u538b\u7f29 spark-2.1.0-bin-hadoop2.7.tgz<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>tar -zxvf spark-2.1.1-bin-hadoop2.7.tar -C ~&#x2F;App<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#2-\u8fdb\u5165-conf-\u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\uff0c\u4fee\u6539-spark-env-sh\" title=\"2.\u8fdb\u5165 conf \u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\uff0c\u4fee\u6539 spark-env.sh\"><\/a>2.\u8fdb\u5165 conf \u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\uff0c\u4fee\u6539 spark-env.sh<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>cd ~&#x2F;App&#x2F;spark-2.1.1-bin-hadoop2.7&#x2F;conf<\/span><br><span>cp spark-env.sh.template spark-env.sh<\/span><br><span>vim spark-env.sh<\/span><br><span><\/span><br><span>export HADOOP_CONF_DIR&#x3D;~&#x2F;App&#x2F;hadoop-2.7.3&#x2F;etc&#x2F;hadoop<\/span><br><span>export HIVE_CONF_DIR&#x3D;~&#x2F;App&#x2F;apache-hive-2.1.1-bin&#x2F;conf<\/span><br><span>export SPARK_DIST_CLASSPATH&#x3D;$(~&#x2F;App&#x2F;hadoop-2.7.3&#x2F;bin&#x2F;hadoop classpath)<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#3-\u8fdb\u5165-conf-\u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\uff0c\u4fee\u6539-spark-defaults-conf\" title=\"3.\u8fdb\u5165 conf \u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\uff0c\u4fee\u6539 spark-defaults.conf\"><\/a>3.\u8fdb\u5165 conf \u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\uff0c\u4fee\u6539 spark-defaults.conf<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>cp spark-defaults.conf.template spark-defaults.conf<\/span><br><span>vim spark-defaults.conf<\/span><br><span><\/span><br><span>spark.master&#x3D;local<\/span><br><span>spark.yarn.historyServer.address&#x3D;bigdata:18080<\/span><br><span>spark.history.ui.port&#x3D;18080<\/span><br><span>spark.eventLog.enabled&#x3D;true<\/span><br><span>spark.eventLog.dir&#x3D;hdfs:&#x2F;&#x2F;&#x2F;tmp&#x2F;spark&#x2F;events<\/span><br><span>spark.history.fs.logDirectory&#x3D;hdfs:&#x2F;&#x2F;&#x2F;tmp&#x2F;spark&#x2F;events<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h4><a href=\"https:\/\/www.studytime.xin\/article\/#4-\u5728-hdfs-\u4e0a\u521b\u5efa\u76ee\u5f55-tmp-spark-events\" title=\"4.\u5728 hdfs \u4e0a\u521b\u5efa\u76ee\u5f55 \/tmp\/spark\/events\"><\/a>4.\u5728 hdfs \u4e0a\u521b\u5efa\u76ee\u5f55 \/tmp\/spark\/events<\/h4>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>hadoop fs \u2013mkdir \u2013p &#x2F;tmp&#x2F;spark&#x2F;events<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u914d\u7f6e\u73af\u5883\u53d8\u91cf\" title=\"\u914d\u7f6e\u73af\u5883\u53d8\u91cf\"><\/a>\u914d\u7f6e\u73af\u5883\u53d8\u91cf<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>vim ~&#x2F;.bash_profile<\/span><br><span><\/span><br><span>export SPARK_HOME&#x3D;&#x2F;Users&#x2F;baihe&#x2F;App&#x2F;spark-2.1.1-bin-hadoop2.7<\/span><br><span>export PATH&#x3D;$SPARK_HOME&#x2F;bin:$PATH<\/span><br><span><\/span><br><span>source ~&#x2F;.bash_profile<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u542f\u52a8\" title=\"\u542f\u52a8\"><\/a>\u542f\u52a8<\/h3>\n<ol>\n<li>\u542f\u52a8 Hdfs <code>start-dfs.sh<\/code><\/li>\n<li>\u542f\u52a8 Spark <code>~\/App\/spark-2.1.1-bin-hadoop2.7\/sbin\/start-all.sh<\/code><\/li>\n<li>\u542f\u52a8 Spark History Server <code>~\/App\/spark-2.1.1-bin-hadoop2.7\/sbin\/start-history-server.sh<\/code><\/li>\n<\/ol>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u67e5\u770b\u8fdb\u7a0b\" title=\"\u67e5\u770b\u8fdb\u7a0b\"><\/a>\u67e5\u770b\u8fdb\u7a0b<\/h3>\n<figure>\n<table>\n<tr>\n<td>\n<pre><span>1<\/span><br><span>2<\/span><br><span>3<\/span><br><span>4<\/span><br><span>5<\/span><br><span>6<\/span><br><span>7<\/span><br><span>8<\/span><br><span>9<\/span><br><span>10<\/span><br><\/pre>\n<\/td>\n<td>\n<pre><span>\u279c  spark-2.1.1-bin-hadoop2.7 jps<\/span><br><span>9008 NameNode<\/span><br><span>9185 SecondaryNameNode<\/span><br><span>10484 HistoryServer<\/span><br><span>10215 Master<\/span><br><span>10281 Worker<\/span><br><span>5514<\/span><br><span>10506 Jps<\/span><br><span>9323 ResourceManager<\/span><br><span>9407 NodeManager<\/span><br><\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/07\/20200706_5f03548aed7ea.png\" alt=\"Spark\u5b66\u4e60\u4e4b\u8def\uff08\u4e8c\uff09:Spark\u4f2a\u5206\u5e03\u5f0f\u5b89\u88c5\"><\/p>\n<h3><a href=\"https:\/\/www.studytime.xin\/article\/#\u67e5\u770bweb\u754c\u9762\" title=\"\u67e5\u770bweb\u754c\u9762\"><\/a>\u67e5\u770bweb\u754c\u9762<\/h3>\n<p><a href=\"http:\/\/bigdata:8080\/\">http:\/\/bigdata:8080\/<\/a><\/p>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/07\/20200706_5f035490ef0f8.png\" alt=\"Spark\u5b66\u4e60\u4e4b\u8def\uff08\u4e8c\uff09:Spark\u4f2a\u5206\u5e03\u5f0f\u5b89\u88c5\"><\/p>\n<p><a href=\"http:\/\/bigdata:18080\/\">spark history server <\/a><\/p>\n<p><img decoding=\"async\" src=\"http:\/\/4563.org\/wp-content\/uploads\/2020\/07\/20200706_5f035495d3db0.png\" alt=\"Spark\u5b66\u4e60\u4e4b\u8def\uff08\u4e8c\uff09:Spark\u4f2a\u5206\u5e03\u5f0f\u5b89\u88c5\"><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u4e0b\u8f7d Spark \u5b89\u88c5\u5305 \u5b98\u7f51\u4e0b\u8f7d&hellip;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/131837"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=131837"}],"version-history":[{"count":0,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/131837\/revisions"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=131837"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=131837"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=131837"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}