# vi /etc/profile
export ANT_HOME=/usr/local/apache-ant-1.8.2
export PATH=$PATH:$ANT_HOME/bin
source /etc/profile 使配置文件生效
lzo的安装
wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.04.tar.gz
tar -zxvf lzo-2.04.tar.gz
cd lzo-2.04
./configure --enable-shared
make && make install
<property>
<name>mapreduce.map.output.compress.codec</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
同步各节点的配置文件!
hadoop集群内测试lzo:
首先,安装lzop,生成一些lzo文件
wget http://www.lzop.org/download/lzop-1.03.tar.gz
tar -zxvf lzop-1.03
cd lzop-1.03
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
./configure
make && make install
压缩日志文件lzop -U -9 nginx.log
利用hive导入到hdfs中hive> load data local inpath '/home/log/nginx.log.lzo' overwrite into table nginxlog;
测试hive语句 hive>select count(1) from nginxlog;
如果出现错误Failed with exception java.io.IOException:java.lang.RuntimeException: native-lzo library not available
删除$HADOOP_HOME/lib目录下的hadoop-gpl-compression-0.1.0.jar
cd kevinweil-hadoop-lzo-23e8370/build/native/Linux-amd64-64/lib
cp * $HADOOP_HOME/lib/native/Linux-amd64-64
重启hadoop
测试hive语句即可成功执行