编译准备工作
[root@node1 ~]# mkdir -p /usr/
java/^C
[root@node1 ~]# tar xf jdk-7u79-linux-x64.tar.gz -C /usr/
java/^C
[root@node1 ~]# cat /etc/profile | grep JAVA
###JAVA_HOME
export JAVA_HOME=/usr/java/jdk1.8.0_92
export CLASSPATH=$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
su - hadoop
mkdir -p /home/hadoop/app
mkdir -p /home/hadoop/maven_repos
mkdir -p /home/hadoop/software
mkdir -p /home/hadoop/source
[hadoop@node1 ~]$ tar xf /home/hadoop/software/apache-maven-3.5.0-bin.zip -C /home/hadoop/app/
[hadoop@node1 ~]$ cd /home/hadoop/app/apache-maven-3.3.9/conf/
编辑 settings.xml
<localRepository>/home/hadoop/maven_repos</localRepository>
<mirror>
<id>nexus-aliyun</id>
<mirrorOf>*</mirrorOf>
<name>Nexus aliyun</name>
<url>http://maven.aliyun.com/nexus/content/groups/public</url>
</mirror>
[root@node1 ~]# mkdir -p /usr/java/^C
[root@node1 ~]# tar xf jdk-7u79-linux-x64.tar.gz -C /usr/java/^C
[root@node1 ~]# cat /etc/profile | grep JAVA
###JAVA_HOME
export JAVA_HOME=/usr/java/jdk1.8.0_92
export CLASSPATH=$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
接下来就是spark的编译工作
[hadoop@node1 ~]$ cd /home/hadoop/source/
tar xf spark-1.6.1.tar.gz
修改 make-distribution.sh 添加如下的文件
VERSION=1.6.1
SCALA_VERSION=2.10.4
SPARK_HADOOP_VERSION=2.5.0-cdh5.8.4
SPARK_HIVE=1
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m"
./make-distribution.sh --tgz \
-Phadoop-2.6 \
-Dhadoop.version=2.6.0-cdh5.7.0 \
-Pyarn \
-Phive -Phive-thriftserver
编译完成之后 就会生成 tgz 文件
spark-1.6.1-bin-2.6.0-cdh5.7.0.tgz
tar xf spark-1.6.1-bin-2.6.0-cdh5.7.0.tgz -C /home/hadoop/app
cd home/hadoop/app/spark-1.6.1-bin-2.6.0-cdh5.7.0/
#spark_home
export
SPARK_HOME=/home/hadoop/app/spark-1.6.1-bin-2.6.0-cdh5.7.0
export
PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin
重命名 slaves spark-env.sh 文件
slaves 添加如下的内容
node1.oracle.com
spark-env.sh 添加如下的内容
export SPARK_MASTER_IP=192.168.137.251
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_CORES=1
export SPARK_WORKER_INSTANCES=1
export SPARK_WORKER_MEMORY=512M