2025年一键部署hadoop集群

科技前沿 • 2025-02-23 16:01 • 阅读 108

一键部署hadoop集群1 背景因为要经常使用 hadoop 集群所以就写了 shell 脚本 2 具体脚本如下 2 1 需要 host 映射 bin bash author Cayon time 2021 04 15 1 使用统一的用户定义变量 ip1 1 hostname1 2 ip2 3

大家好，我是讯享网，很高兴认识大家。

1. 背景

因为要经常使用hadoop集群，所以就写了shell脚本

2. 具体脚本如下

2.1 需要host映射

#!/bin/bash
# author : Cayon
# time : 2021-04-15
#1. 使用统一的用户
#定义变量
ip1=$1
hostname1=$2
ip2=$3
hostname2=$4
ip3=$5
hostname3=$6
#2. 统一目录
setUnifiedDirectory()
{
   mkdir -p /export/servers

   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m创建安装目录失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m创建安装目录成功!\n \033[0m"
}
#"3. 安装统一的JDK"
setJDK()
{
cd ~
   echo "准备解压JDK····"
   tar -zxvf jdk-8u65-linux-x64.tar.gz -C /export/servers/
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m解压JDK失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m解压JDK成功!\n \033[0m"
   cd /export/servers
   echo "export JAVA_HOME=/export/servers/jdk1.8.0_65
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar" >> /etc/profile
   source /etc/profile
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m加载环境变量失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m加载环境变量成功!\n \033[0m"

   java -version
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31mJDK安装失败!\n\033[0m"
exit 2
    fi
echo -e "\033[0;32mJDK安装完成!\n \033[0m"
}
#"4. 关闭机器的防火墙和selinux"
setFilewalld()
{
   systemctl stop firewalld.service
   systemctl disable firewalld.service
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m关闭防火墙失败!\n\033[0m"
exit 2
    fi
echo -e "\033[0;32m关闭防火墙成功!\n \033[0m"
   sed -i '7c SELINUX=disabled' /etc/selinux/config
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m更改selinux失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m更改selinux成功!\n \033[0m"
}
setIP()
{
   echo -e "${ip1} ${hostname1} \n ${ip2} ${hostname2} \n ${ip3} ${hostname3} " >> /etc/hosts
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m更改host失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m更改host成功!\n \033[0m"
   #echo -e "${hostname1} \n ${hostname2} \n ${hostname3} " >> /etc/hostname
   #if [ $? -ne 0 ] ; then
# echo -e "\033[0;31m更改主机名失败!\n\033[0m"
# exit 2
#fi
#echo -e "\033[0;32m更改主机名成功!\n \033[0m"
}
#配置ssh免秘钥
setSSH()
{
   ssh-keygen -t rsa
   ssh-copy-id ${ip1}
   ssh-copy-id ${ip2}
   ssh-copy-id ${ip3}
}
#安装hadoop及配置信息
installHadoop()
{
   cd ~
ls -l | grep hadoop-2.6.0-cdh5.14.0.tar.gz
if [ $? -ne 0 ] ; then
echo -e "\033[0;34m没有这个文件\033[0m"
sleep 1.5
fi
tar -zxvf hadoop-2.6.0-cdh5.14.0.tar.gz -C /export/servers/
cd /export/servers/hadoop-2.6.0-cdh5.14.0/
#配置环境变量
echo '#HADOOP_HOME
export HADOOP_HOME=/export/servers/hadoop-2.6.0-cdh5.14.0
export PATH=:$PATH:$HADOOP_HOME/bin' >> /etc/profile
   source /etc/profile
   sleep 1
#这个目录用于存放Hadoop的临时数据
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/tempDatas
       #这两个目录是用于存储元数据文件的
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas1
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas2
       #这两个目录是存放hdfs数据的目录
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas1
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas2
       #secondary同步元数据用到的目录
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/nn/edits
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/snn/name
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/dfs/snn/edits
sleep 1.5
#配置hadoop
       echo -e "\033[0;34m正在配置Hadoop...\033[0m"
sleep 1.5
       cd /export/servers/hadoop-2.6.0-cdh5.14.0/etc/hadoop
       #修改hadoop-env.sh
       sed -i '25c export JAVA_HOME=/export/servers/jdk1.8.0_65' hadoop-env.sh
       #修改mapred-env.sh
       sed -i '17c export JAVA_HOME=/export/servers/jdk1.8.0_65' mapred-env.sh
       #修改hadoop-env.sh
       sed -i '26c JAVA_HOME=/export/servers/jdk1.8.0_65' yarn-env.sh

mv mapred-site.xml.template mapred-site.xml
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>




<configuration>


<property>
   <name>fs.defaultFS</name>
   <value>hdfs://${hostname1}:8020</value>
</property>

<property>
   <name>hadoop.tmp.dir</name>
   <value>/export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/tempDatas</value>
</property>
<property>
   <name>io.file.buffer.size</name>
   <value>4096</value>
</property>
<property>
   <name>fs.trash.interval</name>
   <value>10080</value>
</property>
</configuration>" > core-site.xml

   echo "<?xml version=\"1.0\"?>
<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>




<configuration>

<property>
   <name>mapreduce.job.ubertask.enable</name>
   <value>true</value>
</property>

<property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
</property>

</configuration>" > mapred-site.xml

   echo "<?xml version=\"1.0\"?>

<configuration>




<property>
   <name>yarn.resourcemanager.hostname</name>
   <value>${hostname3}</value>
</property>

<property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce_shuffle</value>
</property>
<property>
   <name>yarn.nodemanager.resource.memory-mb</name>
   <value>8192</value>
</property>
<property>
   <name>yarn.scheduler.minimum-allocation-mb</name>
   <value>1024</value>
</property>
<property>
   <name>yarn.nodemanager.vmem-pmem-ratio</name>
   <value>2.1</value>
</property>

</configuration>" > yarn-site.xml

   echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>




<configuration>


<property>
   <name>dfs.namenode.secondary.http-address</name>
   <value>${hostname1}:50090</value>
</property>

<property>
   <name>dfs.namenode.http-address</name>
   <value>${hostname1}:50070</value>
</property>

<property>
   <name>dfs.namenode.name.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas1,file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas2</value>
</property>

<property>
   <name>dfs.datanode.data.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas1,file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas2</value>
</property>

<property>
   <name>dfs.namenode.edits.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/nn/edits</value>
</property>
<property>
   <name>dfs.namenode.checkpoint.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/snn/name</value>
</property>
<property>
   <name>dfs.namenode.checkpoint.edits.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/dfs/snn/edits</value>
</property>

<property>
   <name>dfs.replication</name>
   <value>3</value>
</property>

<property>
   <name>dfs.permissions.enabled</name>
   <value>false</value>
</property>

<property>
   <name>dfs.blocksize</name>
   <value></value>
</property>

</configuration>" > hdfs-site.xml

   echo -e "${hostname1}\n${hostname2}\n${hostname3}" >slaves
# cd /export/servers/hadoop-2.6.0-cdh5.14.0
# echo -e "\033[0;34m正在执行 NameNode 的格式化...\033[0m"
#   hdfs namenode -format
# echo -e "\033[0;34m正在开启 NaneNode 和 DataNode 守护进程...\033[0m"
# sbin/start-dfs.sh #启动dfs
# echo -e "\033[0;34m正在启动YARN...\033[0m"
# sbin/start-yarn.sh #启动YARN
if [ $? -ne 0 ] ; then
echo -e "\033[0;32mHadoop配置失败 \033[0m"
exit 2;
fi
echo -e "\033[0;32mHadoop配置并启动成功请至http://${hostname1}:50070和http://${hostname3}:8088/cluster查看\033[0m"
sleep 1.5
}

main()
{ echo "请选择功能"
echo "1. 使用统一的用户"
echo "2. 统一目录"
echo "3. 安装统一的JDK"
echo "4. 关闭机器的防火墙和selinux"
echo "5. 配置好对应的ip以及主机名映射"
   echo "6. 配置机器的ssh免秘钥登录"
   echo "7. 配置机器的时钟同步"
   echo "8. Hadoop的分布式的部署"
echo "q. 退出本脚本"
read a
case $a in
2) setUnifiedDirectory;;
3) setJDK;;
4) setFilewalld;;
5) setIP;;
6) setSSH;;
       8) installHadoop;;
q) exit 0;;
*) echo -e "\033[0;31m没有此选项\033[0m";;

esac
}
main

2.2 不需要host映射

#!/bin/bash
# author : Cayon
# time : 2021-04-15
#1. 使用统一的用户
#定义变量
ip1=$1
ip2=$2
ip3=$3
#2. 统一目录
setUnifiedDirectory()
{
   mkdir -p /export/servers

   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m创建安装目录失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m创建安装目录成功!\n \033[0m"
}
#"3. 安装统一的JDK"
setJDK()
{
cd ~
   echo "准备解压JDK····"
   tar -zxvf jdk-8u65-linux-x64.tar.gz -C /export/servers/
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m解压JDK失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m解压JDK成功!\n \033[0m"
   cd /export/servers
   echo "export JAVA_HOME=/export/servers/jdk1.8.0_65
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar" >> /etc/profile
   source /etc/profile
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m加载环境变量失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m加载环境变量成功!\n \033[0m"

   java -version
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31mJDK安装失败!\n\033[0m"
exit 2
    fi
echo -e "\033[0;32mJDK安装完成!\n \033[0m"
       scp -r /export/servers/jdk1.8.0_65 ${ip2}:/export/servers/
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31m复制JDK到${ip2}失败!\n\033[0m"
exit 2
       fi
       echo -e "\033[0;32m复制JDK到${ip2}完成!\n \033[0m"
       scp -r /export/servers/jdk1.8.0_65 ${ip3}:/export/servers/
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31m复制JDK到${ip3}失败!\n\033[0m"
exit 2
       fi
       echo -e "\033[0;32m复制JDK到${ip3}完成!\n \033[0m"
}
#"4. 关闭机器的防火墙和selinux"
setFilewalld()
{
   systemctl stop firewalld.service
   systemctl disable firewalld.service
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m关闭防火墙失败!\n\033[0m"
exit 2
    fi
echo -e "\033[0;32m关闭防火墙成功!\n \033[0m"
   sed -i '7c SELINUX=disabled' /etc/selinux/config
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m更改selinux失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m更改selinux成功!\n \033[0m"
}
#setIP()
#{
#   echo -e "${ip1} ${hostname1} \n ${ip2} ${hostname2} \n ${ip3} ${hostname3} " >> /etc/hosts
#   if [ $? -ne 0 ] ; then
# echo -e "\033[0;31m更改host失败!\n\033[0m"
# exit 2
# fi
# echo -e "\033[0;32m更改host成功!\n \033[0m"
#   #echo -e "${hostname1} \n ${hostname2} \n ${hostname3} " >> /etc/hostname
#   #if [ $? -ne 0 ] ; then
# # echo -e "\033[0;31m更改主机名失败!\n\033[0m"
# # exit 2
# #fi
# #echo -e "\033[0;32m更改主机名成功!\n \033[0m"
#}
#配置ssh免秘钥
setSSH()
{
   ssh-keygen -t rsa
   ssh-copy-id ${ip1}
   ssh-copy-id ${ip2}
   ssh-copy-id ${ip3}
}
#安装hadoop及配置信息
installHadoop()
{
   cd ~
ls -l | grep hadoop-2.6.0-cdh5.14.0.tar.gz
if [ $? -ne 0 ] ; then
echo -e "\033[0;34m没有这个文件\033[0m"
sleep 1.5
fi
tar -zxvf hadoop-2.6.0-cdh5.14.0.tar.gz -C /export/servers/
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31m解压失败!\n\033[0m"
exit 2
       fi
echo -e "\033[0;32m解压Hadoop包成功!\n \033[0m"
cd /export/servers/hadoop-2.6.0-cdh5.14.0/
#配置环境变量
echo '#HADOOP_HOME
export HADOOP_HOME=/export/servers/hadoop-2.6.0-cdh5.14.0
export PATH=:$PATH:$HADOOP_HOME/bin' >> /etc/profile
       source /etc/profile
       if [ $? -ne 0 ] ; then
                   echo -e "\033[0;31m环境变量加载失败!\n\033[0m"
                   exit 2
       fi
           echo -e "\033[0;32m环境变量加载成功!\n \033[0m"
       sleep 1
           #这个目录用于存放Hadoop的临时数据
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/tempDatas
           #这两个目录是用于存储元数据文件的
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas1
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas2
           #这两个目录是存放hdfs数据的目录
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas1
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas2
           #secondary同步元数据用到的目录
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/nn/edits
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/snn/name
       mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/dfs/snn/edits
       if [ $? -ne 0 ] ; then
                   echo -e "\033[0;31m创建hadoop数据文件夹失败!\n\033[0m"
                   exit 2
       fi
echo -e "\033[0;32m创建hadoop数据文件夹成功!\n \033[0m"
#配置hadoop
       echo -e "\033[0;34m正在配置Hadoop...\033[0m"
sleep 1.5
       cd /export/servers/hadoop-2.6.0-cdh5.14.0/etc/hadoop
       #修改hadoop-env.sh
       sed -i '25c export JAVA_HOME=/export/servers/jdk1.8.0_65' hadoop-env.sh
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31mhadoop-env.sh配置失败!\n\033[0m"
exit 2
       fi
echo -e "\033[0;32mhadoop-env.sh配置成功!\n \033[0m"
       #修改mapred-env.sh
       sed -i '17c export JAVA_HOME=/export/servers/jdk1.8.0_65' mapred-env.sh
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31mmapred-env.sh配置失败!\n\033[0m"
exit 2
       fi
echo -e "\033[0;32mmapred-env.sh配置成功!\n \033[0m"
       #修改hadoop-env.sh
       sed -i '26c JAVA_HOME=/export/servers/jdk1.8.0_65' yarn-env.sh
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31myarn-env.sh配置失败!\n\033[0m"
exit 2
       fi
echo -e "\033[0;32myarn-env.sh配置成功!\n \033[0m"

mv mapred-site.xml.template mapred-site.xml
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>




<configuration>


<property>
   <name>fs.defaultFS</name>
   <value>hdfs://${ip1}:8020</value>
</property>

<property>
   <name>hadoop.tmp.dir</name>
   <value>/export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/tempDatas</value>
</property>
<property>
   <name>io.file.buffer.size</name>
   <value>4096</value>
</property>
<property>
   <name>fs.trash.interval</name>
   <value>10080</value>
</property>
</configuration>" > core-site.xml
           if [ $? -ne 0 ] ; then
echo -e "\033[0;31mcore-site.xml配置失败!\n\033[0m"
exit 2
           fi
           echo -e "\033[0;32mcore-site.xml配置成功!\n \033[0m"

   echo "<?xml version=\"1.0\"?>
<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>




<configuration>

<property>
   <name>mapreduce.job.ubertask.enable</name>
   <value>true</value>
</property>

<property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
</property>

</configuration>" > mapred-site.xml

           if [ $? -ne 0 ] ; then
echo -e "\033[0;31mcore-site.xml配置失败!\n\033[0m"
exit 2
           fi
           echo -e "\033[0;32mmapred-site.xml配置成功!\n \033[0m"

讯享网

   echo "<?xml version=\"1.0\"?>

<configuration>




<property>
   <name>yarn.resourcemanager.hostname</name>
   <value>${ip3}</value>
</property>

<property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce_shuffle</value>
</property>
<property>
   <name>yarn.nodemanager.resource.memory-mb</name>
   <value>8192</value>
</property>
<property>
   <name>yarn.scheduler.minimum-allocation-mb</name>
   <value>1024</value>
</property>
<property>
   <name>yarn.nodemanager.vmem-pmem-ratio</name>
   <value>2.1</value>
</property>

</configuration>" > yarn-site.xml

               if [ $? -ne 0 ] ; then
                   echo -e "\033[0;31myarn-site.xml配置失败!\n\033[0m"
                   exit 2
               fi
               echo -e "\033[0;32myarn-site.xml配置成功!\n \033[0m"

   echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>




<configuration>


<property>
   <name>dfs.namenode.secondary.http-address</name>
   <value>${ip1}:50090</value>
</property>

<property>
   <name>dfs.namenode.http-address</name>
   <value>${ip1}:50070</value>
</property>

<property>
   <name>dfs.namenode.name.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas1,file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas2</value>
</property>

<property>
   <name>dfs.datanode.data.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas1,file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas2</value>
</property>

<property>
   <name>dfs.namenode.edits.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/nn/edits</value>
</property>
<property>
   <name>dfs.namenode.checkpoint.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/snn/name</value>
</property>
<property>
   <name>dfs.namenode.checkpoint.edits.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/dfs/snn/edits</value>
</property>

<property>
   <name>dfs.replication</name>
   <value>3</value>
</property>

<property>
   <name>dfs.permissions.enabled</name>
   <value>false</value>
</property>

<property>
   <name>dfs.blocksize</name>
   <value></value>
</property>

</configuration>" > hdfs-site.xml
                   if [ $? -ne 0 ] ; then
                           echo -e "\033[0;31mhdfs-site.xml配置失败!\n\033[0m"
                           exit 2
                   fi
                   echo -e "\033[0;32mhdfs-site.xml配置成功!\n \033[0m"

   echo -e "${ip1}\n${ip2}\n${ip3}" >slaves
                   if [ $? -ne 0 ] ; then
                           echo -e "\033[0;31mslaves配置失败!\n\033[0m"
                           exit 2
                   fi
                   echo -e "\033[0;32mslaves配置成功!\n \033[0m"

       scp -r /export/servers/hadoop-2.6.0-cdh5.14.0 ${ip2}:/export/servers/
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31m复制Hadoop到${ip2}失败!\n\033[0m"
exit 2
       fi
       echo -e "\033[0;32m复制Hadoop到${ip2}完成!\n \033[0m"
       scp -r /export/servers/hadoop-2.6.0-cdh5.14.0 ${ip3}:/export/servers/
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31m复制Hadoop到${ip3}失败!\n\033[0m"
exit 2
       fi
       echo -e "\033[0;32m复制Hadoop到${ip3}完成!\n \033[0m"


       scp /etc/profile ${ip2}:/export/servers/
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31m复制环境变量到${ip2}失败!\n\033[0m"
exit 2
       fi
       echo -e "\033[0;32m复制环境变量到${ip2}完成!\n \033[0m"
       scp /etc/profile ${ip3}:/export/servers/
       if [ $? -ne 0 ] ; then
echo -e "\033[0;31m复制环境变量到${ip3}失败!\n\033[0m"
exit 2
       fi
       echo -e "\033[0;32m复制环境变量到${ip3}完成!\n \033[0m"

# cd /export/servers/hadoop-2.6.0-cdh5.14.0
# echo -e "\033[0;34m正在执行 NameNode 的格式化...\033[0m"
#   hdfs namenode -format
# echo -e "\033[0;34m正在开启 NaneNode 和 DataNode 守护进程...\033[0m"
# sbin/start-dfs.sh #启动dfs
# echo -e "\033[0;34m正在启动YARN...\033[0m"
# sbin/start-yarn.sh #启动YARN
if [ $? -ne 0 ] ; then
echo -e "\033[0;32mHadoop配置失败 \033[0m"
exit 2;
fi
       echo -e "\033[0;32mHadoop配置成功"
#echo -e "\033[0;32mHadoop配置并启动成功请至http://${ip1}:50070和http://${ip3}:8088/cluster查看\033[0m"
sleep 1.5
}

main()
{ echo "请选择功能"
echo "1. 使用统一的用户"
echo "1. 统一目录"
echo "2. 安装统一的JDK"
echo "3. 关闭机器的防火墙和selinux"
echo "4. 配置好对应的ip以及主机名映射"
       echo "5. 配置机器的ssh免秘钥登录"
       echo "6. 配置机器的时钟同步"
       echo "7. Hadoop的分布式的部署"
echo "8. 退出本脚本"
read a
case $a in
1) setUnifiedDirectory;;
2) setJDK;;
3) setFilewalld;;
4) setIP;;
5) setSSH;;
               7) installHadoop;;
q) exit 0;;
*) echo -e "\033[0;31m没有此选项\033[0m";;

esac
}
main

3. 单台机器部署

#!/bin/sh
#1. 使用统一的用户
#2. 统一目录
setUnifiedDirectory()
{
   mkdir -p /export/servers

if [ $? -ne 0 ] ; then
echo -e "\033[0;31m创建/export/servers失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m创建/export/servers成功!\n \033[0m"
}
#"3. 安装统一的JDK"
setJDK()
{
cd ~
   tar -zxvf jdk-8u65-linux-x64.tar.gz -C /export/servers
   cd /export/servers
   echo "export JAVA_HOME=/export/servers/jdk1.8.0_65
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar" >> /etc/profile
   source /etc/profile
   java -version
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31mJDK安装失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32mJDK安装完成!\n \033[0m"
}
#"4. 关闭机器的防火墙和selinux"
setFilewalld()
{
   systemctl stop firewalld.service
   systemctl disable firewalld.service
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m关闭防火墙失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m关闭防火墙成功!\n \033[0m"
   sed -i '7c SELINUX=disabled' /etc/selinux/config
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m更改selinux失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m更改selinux成功!\n \033[0m"
}
setIP()
{
   ifconfig ens192|grep "inet"|awk '{print $2}'|awk 'NR==1' >> /etc/hosts
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m更改host失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m更改host成功!\n \033[0m"
   echo 'hadoop' >> /etc/hostname
   if [ $? -ne 0 ] ; then
echo -e "\033[0;31m更改主机名失败!\n\033[0m"
exit 2
fi
echo -e "\033[0;32m更改主机名成功!\n \033[0m"
}
#配置ssh免秘钥
setSSH()
{
   ssh-keygen -t rsa
   ssh-copy-id ifconfig ens192|grep "inet"|awk '{print $2}'|awk 'NR==1'
}
#安装hadoop及配置信息
installHadoop()
{
   cd ~
ls -l | grep hadoop-2.6.0-cdh5.14.0.tar.gz
if [ $? -ne 0 ] ; then
echo -e "\033[0;34m没有这个文件\033[0m"
sleep 1.5
fi
tar -zxvf hadoop-2.6.0-cdh5.14.0.tar.gz -C /export/servers/
cd /export/servers/
#配置环境变量
echo '#HADOOP_HOME
export HADOOP_HOME=/export/servers/hadoop-2.6.0-cdh5.14.0
export PATH=:$PATH:$HADOOP_HOME/bin' >> /etc/profile
   source /etc/profile
   sleep 2
#这个目录用于存放Hadoop的临时数据
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/tempDatas
       #这两个目录是用于存储元数据文件的
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas1
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas2
       #这两个目录是存放hdfs数据的目录
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas1
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas2
       #secondary同步元数据用到的目录
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/nn/edits
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/snn/name
   mkdir -p /export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/dfs/snn/edits
sleep 1.5
#配置hadoop
       echo -e "\033[0;34m正在配置Hadoop...\033[0m"
sleep 1.5
       cd /export/servers/hadoop-2.6.0-cdh5.14.0/etc/hadoop
       #修改hadoop-env.sh
       sed -i '25c export JAVA_HOME=/export/servers/jdk1.8.0_65' hadoop-env.sh
       #修改mapred-env.sh
       echo 'export JAVA_HOME=/export/servers/jdk1.8.0_65' >> mapred-env.sh
       #修改hadoop-env.sh
       sed -i '26c export JAVA_HOME=/export/servers/jdk1.8.0_65' hadoop-env.sh

mv mapred-site.xml.template mapred-site.xml
echo '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>




<configuration>


<property>
   <name>fs.defaultFS</name>
   <value>hdfs://hadoop:8020</value>
</property>

<property>
   <name>hadoop.tmp.dir</name>
   <value>/export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/tempDatas</value>
</property>
<property>
   <name>io.file.buffer.size</name>
   <value>4096</value>
</property>
<property>
   <name>fs.trash.interval</name>
   <value>10080</value>
</property>
</configuration>' > core-site.xml

   echo '<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>




<configuration>

<property>
   <name>mapreduce.job.ubertask.enable</name>
   <value>true</value>
</property>

<property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
</property>

</configuration>' > mapred-site.xml

   echo '<?xml version="1.0"?>

<configuration>




<property>
   <name>yarn.resourcemanager.hostname</name>
   <value>hadoop3</value>
</property>

<property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce_shuffle</value>
</property>
<property>
   <name>yarn.nodemanager.resource.memory-mb</name>
   <value>8192</value>
</property>
<property>
   <name>yarn.scheduler.minimum-allocation-mb</name>
   <value>1024</value>
</property>
<property>
   <name>yarn.nodemanager.vmem-pmem-ratio</name>
   <value>2.1</value>
</property>

</configuration>' > yarn-site.xml

   echo '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>




<configuration>


<property>
   <name>dfs.namenode.secondary.http-address</name>
   <value>hadoop:50090</value>
</property>

<property>
   <name>dfs.namenode.http-address</name>
   <value>hadoop:50070</value>
</property>

<property>
   <name>dfs.namenode.name.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas1,file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/namenodeDatas2</value>
</property>

<property>
   <name>dfs.datanode.data.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas1,file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/datanodeDatas2</value>
</property>

<property>
   <name>dfs.namenode.edits.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/nn/edits</value>
</property>
<property>
   <name>dfs.namenode.checkpoint.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/snn/name</value>
</property>
<property>
   <name>dfs.namenode.checkpoint.edits.dir</name>
   <value>file:///export/servers/hadoop-2.6.0-cdh5.14.0/hadoopDatas/dfs/snn/edits</value>
</property>

<property>
   <name>dfs.replication</name>
   <value>3</value>
</property>

<property>
   <name>dfs.permissions.enabled</name>
   <value>false</value>
</property>

<property>
   <name>dfs.blocksize</name>
   <value></value>
</property>

</configuration>' > hdfs-site.xml

   echo 'hadoop' >slaves
# cd /export/servers/hadoop-2.6.0-cdh5.14.0
# echo -e "\033[0;34m正在执行 NameNode 的格式化...\033[0m"
#   hdfs namenode -format
# echo -e "\033[0;34m正在开启 NaneNode 和 DataNode 守护进程...\033[0m"
# sbin/start-dfs.sh #启动dfs
# echo -e "\033[0;34m正在启动YARN...\033[0m"
# sbin/start-yarn.sh #启动YARN
if [ $? -ne 0 ] ; then
echo -e "\033[0;32mHadoop配置失败 \033[0m"
exit 2;
fi
echo -e "\033[0;32mHadoop配置并启动成功请至http://hadoop1:50070和http://localhost:8088/cluster查看\033[0m"
sleep 1.5
}

main()
{ echo "请选择功能"
echo "1. 使用统一的用户"
echo "2. 统一目录"
echo "3. 安装统一的JDK"
echo "4. 关闭机器的防火墙和selinux"
echo "5. 配置好对应的ip以及主机名映射"
       echo "6. 配置机器的ssh免秘钥登录"
       echo "7. 配置机器的时钟同步"
       echo "8. Hadoop的分布式的部署"
echo "q. 退出本脚本"
read a
case $a in
2) setUnifiedDirectory;;
3) setJDK
exit;;
4) setFilewalld;;
5) setIP;;
6) setSSH;;
       8) installHadoop;;
q) exit 0;;
*) echo -e "\033[0;31m没有此选项\033[0m";;

esac
}
main

小讯

2025年电容读数

上一篇 2025-04-03 12:49

2025年数字逻辑应用设计--运算器实验

下一篇 2025-03-04 10:56

2025年电容读数 1735977440
2025年Java API操作ES 高级查询 1735977439
flex与folat 1735977438
2025年1900-2100公历年以查表方式取农历二十四节气(VBA) 1735977438
计算机键盘上的符号英语怎么说,电脑键盘上特殊符号和标点符号名称的输入方法汇总... 1735977437
LDF文件步骤及内容详解 1735977437
Day12_08_Redis教程之Redis集群原理及集群搭建 1735977435
虚拟机安装 - 网络配置 1735977435
MP2帧格式介绍 1735977433
2025年数字逻辑应用设计--运算器实验 1735977444
设计模式结构型——组合模式 1735977446
Makefile基础1 1735977447
2025年Aruba 胖AP配置-Aruba胖AP默认密码 1735977447
2025年NewPhy.-揭秘优势种dominant species 1735977448
0028-多个事务互相调用 1735977449
好评率超高的9个公众号，值得收藏 1735977450
2025年项目01——图书进、销、存(jxc)系统(单机版) 1735977450
分享86个时间日期JS特效，总有一款适合您 1735977453

版权声明：本文内容由互联网用户自发贡献，该文观点仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容，请联系我们，一经查实，本站将立刻删除。
如需转载请保留出处：https://51itzy.com/kjqy/11463.html