{"id":131709,"date":"2020-07-06T12:16:56","date_gmt":"2020-07-06T04:16:56","guid":{"rendered":"http:\/\/4563.org\/?p=131709"},"modified":"2020-07-06T12:16:56","modified_gmt":"2020-07-06T04:16:56","slug":"hadoop%e9%9b%86%e7%be%a4%e9%85%8d%e7%bd%aelzo%e5%8e%8b%e7%bc%a9%e4%bb%a5%e5%8f%8a%e6%94%af%e6%8c%81hive","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=131709","title":{"rendered":"hadoop\u96c6\u7fa4\u914d\u7f6eLZO\u538b\u7f29\u4ee5\u53ca\u652f\u6301Hive"},"content":{"rendered":"

<\/a>\u4e0b\u8f7d\u4ee5\u53ca\u7f16\u8bd1lzo\u6e90\u7801\u5305<\/h3>\n

LZO\u6e90\u7801\u5305\u5730\u5740<\/a><\/p>\n

<\/a>\u5c06\u4e0b\u8f7d\u7684\u6e90\u7801\u5305\u4f7f\u7528maven\u8fdb\u884c\u7f16\u8bd1<\/h4>\n

\u4f7f\u7528 mvn clean package<\/code>\uff0c\u7f16\u8bd1\u4e4b\u540e\u5c06\u751f\u6210adoop-lzo-0.4.20.jar\u7684jar\u5305<\/p>\n

\u7279\u6b8a\u8bf4\u660e\uff1a\u4e5f\u53ef\u4ee5\u4e0d\u8fdb\u884c\u7f16\u8bd1\uff0c\u76f4\u63a5\u4f7f\u7528\u4f5c\u8005\u7f16\u8bd1\u597d\u7684lzo jar\u5305\u3002\u94fe\u63a5: https:\/\/pan.baidu.com\/s\/13IjKDEokh_dqkGoDY1bIFA<\/a> \u5bc6\u7801: fbtf<\/p>\n

<\/a>\u5c06\u7f16\u8bd1\u4e4b\u540e\u7684jar\u5305\u4e0a\u4f20\u5230node01\u8282\u70b9\uff0c\u4e14\u540c\u6b65\u81f3\u5176\u4ed6\u673a\u5668<\/h3>\n
\n\n\n
\n
1<\/span>
2<\/span>
3<\/span>
4<\/span>
5<\/span>
6<\/span>
7<\/span>
8<\/span>
<\/pre>\n<\/td>\n
\n
# \u4e0a\u4f20<\/span>
cd /opt/module/hadoop-2.6.0-cdh5.14.2/share/hadoop/common<\/span>
rz<\/span>
...<\/span>
<\/span>
# \u540c\u6b65\u81f3\u5176\u4ed6\u673a\u5668<\/span>
scp hadoop-lzo-0.4.20.jar  node02:$PWD<\/span>
scp hadoop-lzo-0.4.20.jar  node03:$PWD<\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n
<\/a>\u4fee\u6539core-site.xml\uff0c\u914d\u7f6e\u538b\u7f29\u65b9\u5f0f<\/h3>\n
\n\n\n
\n
1<\/span>
2<\/span>
3<\/span>
4<\/span>
5<\/span>
6<\/span>
7<\/span>
8<\/span>
9<\/span>
10<\/span>
11<\/span>
12<\/span>
13<\/span>
14<\/span>
15<\/span>
16<\/span>
17<\/span>
18<\/span>
19<\/span>
<\/pre>\n<\/td>\n
\n
cd /opt/module/hadoop-2.6.0-cdh5.14.2/etc/hadoop<\/span>
vim core-site.xml<\/span>
<\/span>
<property><\/span>
        <name>io.compression.codecs</name><\/span>
        <value><\/span>
                org.apache.hadoop.io.compress.GzipCodec,<\/span>
                org.apache.hadoop.io.compress.DefaultCodec,<\/span>
                org.apache.hadoop.io.compress.BZip2Codec,<\/span>
                org.apache.hadoop.io.compress.SnappyCodec,<\/span>
                com.hadoop.compression.lzo.LzoCodec,<\/span>
                com.hadoop.compression.lzo.LzopCodec<\/span>
        </value><\/span>
</property><\/span>
<\/span>
<property><\/span>
    <name>io.compression.codec.lzo.class</name><\/span>
    <value>com.hadoop.compression.lzo.LzoCodec</value><\/span>
</property><\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n
<\/a>\u540c\u6b65core-site.xml\u914d\u7f6e\u5230\u5176\u4ed6\u673a\u5668<\/h3>\n
\n\n\n
\n
1<\/span>
2<\/span>
3<\/span>
<\/pre>\n<\/td>\n
\n
cd /opt/module/hadoop-2.6.0-cdh5.14.2/etc/hadoop<\/span>
scp core-site.xml  node02:$PWD<\/span>
scp core-site.xml  node03:$PWD<\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n
<\/a>\u914d\u7f6ehadoop-env.sh(\u53ef\u9009)<\/h3>\n
\n\n\n
\n
1<\/span>
<\/pre>\n<\/td>\n
\n
export LD_LIBRARY_PATH=/opt/module/hadoop-2.6.0-cdh5.14.2/share/hadoop/common/<\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n
<\/a>\u914d\u7f6emapred-site.xml(\u53ef\u9009)<\/h3>\n
\n\n\n
\n
1<\/span>
2<\/span>
3<\/span>
4<\/span>
5<\/span>
6<\/span>
7<\/span>
8<\/span>
9<\/span>
10<\/span>
11<\/span>
12<\/span>
13<\/span>
14<\/span>
15<\/span>
16<\/span>
17<\/span>
18<\/span>
19<\/span>
20<\/span>
21<\/span>
22<\/span>
23<\/span>
24<\/span>
25<\/span>
<\/pre>\n<\/td>\n
\n
<!--\u542f\u7528map\u4e2d\u95f4\u6587\u4ef6\u538b\u7f29--><\/span>
<property> <\/span>
    <name>mapreduce.map.output.compress</name> <\/span>
    <value>true</value> <\/span>
</property> <\/span>
<!--\u542f\u7528map\u4e2d\u95f4\u538b\u7f29\u7c7b--><\/span>
<property><\/span>
   <name>mapred.map.output.compression.codec</name><\/span>
   <value>com.hadoop.compression.lzo.LzopCodec</value><\/span>
</property><\/span>
<!--\u542f\u7528mapreduce\u6587\u4ef6\u538b\u7f29--><\/span>
<property><\/span>
    <name>mapreduce.output.fileoutputformat.compress</name><\/span>
    <value>true</value><\/span>
</property> <\/span>
<!--\u542f\u7528mapreduce\u538b\u7f29\u7c7b--><\/span>
<property><\/span>
   <name>mapreduce.output.fileoutputformat.compress.codec</name><\/span>
   <value>com.hadoop.compression.lzo.LzopCodec</value><\/span>
</property><\/span>
<!--\u914d\u7f6eJar\u5305--><\/span>
<property><\/span>
    <name>mapred.child.env</name><\/span>
    <value>LD_LIBRARY_PATH=/opt/module/hadoop-2.6.0-cdh5.14.2/share/hadoop/common/</value><\/span>
</property><\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n
<\/a>\u540c\u6b65hadoop-env.sh\u4ee5\u53camapred-site.xml\u914d\u7f6e\u5230\u5176\u4ed6\u673a\u5668(\u53ef\u9009)<\/h3>\n
\n\n\n
\n
1<\/span>
2<\/span>
3<\/span>
4<\/span>
5<\/span>
6<\/span>
<\/pre>\n<\/td>\n
\n
cd /opt/module/hadoop-2.6.0-cdh5.14.2/etc/hadoop<\/span>
scp hadoop-env.sh  node02:$PWD<\/span>
scp hadoop-env.sh  node03:$PWD<\/span>
<\/span>
scp mapred-site.xml  node02:$PWD<\/span>
scp mapred-site.xml  node03:$PWD<\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n
<\/a>\u91cd\u65b0\u542f\u52a8hdfs\u96c6\u7fa4<\/h3>\n
\n\n\n
\n
1<\/span>
2<\/span>
<\/pre>\n<\/td>\n
\n
stop-dfs.sh<\/span>
start-dfs.sh<\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n
<\/a>\u914d\u7f6ehive\u652f\u6301LZO\u538b\u7f29<\/h3>\n
\u76f4\u63a5\u5c06hadoop-lzo-0.20.jar\u8fd9\u4e2ajar\u5305\u62f7\u8d1d\u5230hive\u7684lib\u76ee\u5f55\u4e0b\u5373\u53ef\u3002<\/p>\n
\u6211\u7684hive\u670d\u52a1\u5668\u5b89\u88c5\u5728node03\u8282\u70b9\u4e0a\uff0c\u6240\u4ee5\u5728node03\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4\uff0c\u5c06jar\u5305\u62f7\u8d1d\u5230hive\u7684lib\u76ee\u5f55\u4e0b\u5373\u53ef\u3002<\/p>\n
\n\n\n
\n
1<\/span>
<\/pre>\n<\/td>\n
\n
cp /opt/module/hadoop-2.6.0-cdh5.14.2/share/hadoop/common/hadoop-lzo-0.4.20.jar  /opt/module/hive-1.1.0-cdh5.14.2/lib/<\/span>
<\/pre>\n<\/td>\n<\/tr>\n<\/table>\n<\/figure>\n","protected":false},"excerpt":{"rendered":"
\u4e0b\u8f7d\u4ee5\u53ca\u7f16\u8bd1lzo\u6e90\u7801\u5305 LZO\u6e90…<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/131709"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=131709"}],"version-history":[{"count":0,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/131709\/revisions"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=131709"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=131709"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=131709"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}