2025年Shell编程基础-04-骚气的hadoop生态圈一键安装配置环境脚本

Shell编程基础-04-骚气的hadoop生态圈一键安装配置环境脚本bin bash 移除环境变量配置和在 software 里已安装过的资源 BEGIN cat n etc profile grep done awk print 1 找到环境变量的前一行行号 END cat

大家好,我是讯享网,很高兴认识大家。
#!/bin/bash #---------------------移除环境变量配置和在software里已安装过的资源-------------- BEGIN=`cat -n /etc/profile|grep done|awk '{print $1}'` #找到环境变量的前一行行号 END=`cat -n /etc/profile|grep "unset i"|awk '{print $1}'` #找到环境变量的后一行行号 RST=$(((--END)-(++BEGIN))) #用来判断这两行之间有没有内容,大于0就是有 if [ $RST -gt 0 ] #有内容的话,就清空掉 then RST="sed -i '$BEGIN,$END d' /etc/profile" RST=`eval $RST` echo "/etc/profile has been cleared" fi if [ -e /opt/software ] #判断software文件夹是否存在,如果存在就删除掉 then rm -rf /opt/software echo "/opt/software has been deleted" fi #---------------------------------待安装文件清单-------------------------------- INSTALL_LIST="jdk-8u111-linux-x64.tar.gz scala-2.12.10.tgz hadoop-2.6.0-cdh5.14.2.tar.gz hive-1.1.0-cdh5.14.2.tar.gz zeppelin-0.8.2-bin-all.tgz zookeeper-3.4.5-cdh5.14.2.tar.gz hbase-1.2.0-cdh5.14.2.tar.gz sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz" #---------------------------检查安装包是否缺少---------------------------------- LIST=`ls /opt/download/hadoop` #查看文件夹内的所有安装包 for item in $INSTALL_LIST do if [[ $LIST =~ $item ]] #如果每一个待安装文件都在这个文件夹内,就继续 then continue #只要在,就继续循环检查下一个 else echo "install exit abnormally , for the lack of $item" exit 1 #只要缺少一个,就退出脚本 fi done #---------------------------开始安装并重命名------------------------------------ echo "start to installing" echo "first,we'll unpack all the items and rename it ..." #--------确保解压路径存在,如果不存在则创建一个 DEST_DIR="/opt/software/" #这个目标路径后续会调用,最后面的/不能缺少 if [ ! -e $DEST_DIR ] then RST=`mkdir $DEST_DIR` fi cd /opt/software #--------解压和重命名 RENAME_LIST="java180 scala21210 hadoop260 hive110 zeppelin082 zookeeper345 hbase120 sqoop146" RENAME_LIST=($RENAME_LIST) IX=0 echo "change dir to "`pwd` #切换目录到/opt/software for item in $INSTALL_LIST do if [[ $item =~ tar|gz$ ]] then echo "start unpack $item..." OPT=${ 
   item*.tar} OPT="-"$([ $OPT ] && echo "z" || echo "")"xf" RST=`eval "tar $OPT /opt/download/hadoop/$item"` RST=`eval "mv ${item%%-*}* ${RENAME_LIST[$IX]}"` echo "unpack $item and rename to ${RENAME_LIST[$IX]} ok" fi ((IX++)) done #--------------------------配置/etc/profile环境和路径 RENAME_ITEMS="java180_bin scala21210_bin hadoop260_bin_sbin hive110_bin zeppelin082_bin zookeeper345_sbin hbase120_bin sqoop146_bin" ROAD="" ROAD="export PATH=" for item in $RENAME_ITEMS do HOME=${ 
   item%%[0-9]*} HOME=${ 
   HOME^^}"_HOME" echo "NB $HOME" T=${ 
   item//_/ } FIRST=1 for var in $T do if [ $FIRST -eq 1 ];then FIRST=0 else ROAD+="$"$HOME"/"$var":" fi done echo "---- $HOME finished" HOME="export $HOME=$DEST_DIR${item%%_*}" RST=`eval "sed -i '/unset i/i\$HOME' /etc/profile"` done ROAD+="$"PATH RST=`eval "sed -i '/unset i/i\$ROAD' /etc/profile"` #----------JAVA CLASSPATH ROAD="export CLASSPATH=.:$""JAVA_HOME/lib/dt.jat:$""JAVA_HOME/lib/tools.jar" RST=`eval "sed -i '/unset i/i\$ROAD' /etc/profile"` #----------ZOOKEEPER LOGS ROAD="export LOGDIR=$""SQOOP_HOME/logs/" RST=`eval "sed -i '/unset i/i\$ROAD' /etc/profile"` #-----------------------------激活环境配置----------------------------- source /etc/profile #--------------------------找出本机IP--------------------------------- HOST_IP=`ip addr|awk -F '/' '/noprefixroute ens33/{print $1}'|awk -F ' ' '{print $2}'` #-------------------------配置HADOOP环境----------------------------- echo " entering HADOOP configuration" DEST_DIR=$HADOOP_HOME"/etc/hadoop/" #配置文件都在/etc/hadoop/下 `sed -i 's/${JAVA_HOME}/\/opt\/software\/java180/g' ${ 
   DEST_DIR}hadoop-env.sh` #配置hadoop环境的java路径 CONFS="core-site.xml hdfs-site.xml mapred-site.xml.template yarn-site.xml" readonly CONFI1=" <property> <name>fs.defaultFS</name> <value>hdfs://${HOST_IP}:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>$HADOOP_HOME</value> </property> " RST=`mkdir $HADOOP_HOME"/tmp"` RST=`mkdir $HADOOP_HOME"/tmp/name" $HADOOP_HOME"/tmp/data"` readonly CONFI2=" <property> <name>dfs.replication</name> <value>1</value> </property> <property> <name>dfs.namenode.name.dir</name> <value>$HADOOP_HOME/tmp/name</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>$HADOOP_HOME/tmp/data</value> </property> " readonly CONFI3=" <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> " readonly CONFI4=" <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> " IX=1 for file in $CONFS do if [[ $file =~ .template ]] #把里面的template结尾的文件后缀删掉 then CMD="mv $DEST_DIR$file $DEST_DIR${file%.*}" RST=`eval $CMD` file=${ 
   file%.*} #改完名后,配置清单也更新下 fi CMD="sed -i '/<\/configuration>/i\REPL\' $DEST_DIR$file" #把line一行一行插入到config上面 V="CONFI"$IX LINES=`eval echo "$"$V` echo "---------configing $file..." for line in $LINES do RST=${ 
   CMD/REPL/$line} RST=`eval $RST` #执行上面的CMD命令,第一遍是替换REPL为line,第二遍是执行CMD done ((IX++)) done #--------hadoop namenode format echo -n "-----start to format namenode ..." cd $HADOOP_HOME/tmp/name #进入到name文件夹下 RST=`eval "hdfs namenode -format>/root/mnfmt.log 2>&1"` #把格式化日志输出到一个新的文件里,不展示 RST=`eval "cat /root/mnfmt.log|grep util.ExitUtil"` #筛选日志文件信息判断是否格式化成功 RST=${ 
   RST* } #把空格前的都删掉 echo $([ $RST -eq 0 ] && echo "ok" || echo "error") if [ $RST -ne 0 ];then exit 1;fi #------------------hive---------------------- cd $HIVE_HOME #进入到hive目录下 echo -n " start to config hive lib ..." echo "----- copy mysql driver to hive lib ..." cp /opt/download/hadoop/mysql*.jar lib/ echo "ok" echo -n "---- create hive warehouse" mkdir warehouse/ echo "ok" echo -n "---- start to create hive-site.xml ..." touch ./conf/hive-site.xml echo "ok" echo -n "---- start to config hive-site.xml ..." cat > ./conf/hive-site.xml <<EOF #cat和eof结合使用具有追加功能 <?xml version="1.0" encoding="UTF-8" standalone="no"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> <configuration> <property> <name>hive.metastore.warehouse.dir</name> <value>$HIVE_HOME/warehouse</value> </property> <property> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://${ 
   HOST_IP}:3306/hive110?createDatabaseIfNotExist=true</value> </property> <property> <name>javax.jdo.option.ConnectionDriverName</name> <value>com.mysql.jdbc.Driver</value> </property> <property> <name>javax.jdo.option.ConnectionUserName</name> <value>root</value> </property> <property> <name>javax.jdo.option.ConnectionPassword</name> <value>root</value> </property> </configuration> EOF echo "ok" #------------------zookepper--------------------- RST=`mkdir $ZOOKEEPER_HOME/mydata` RST=`echo "1">$ZOOKEEPER_HOME/mydata/myid` RST=`cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg` RST=`sed -i 's/\/tmp\/zookeeper/\/opt\/software\/zookeeper345\/mydata/' $ZOOKEEPER_HOME/conf/zoo.cfg` RST=`eval "sed -i '/dataDir/a\server.1=$HOST_IP:2888:3888'" $ZOOKEEPER_HOME/conf/zoo.cfg` #-------hbase---------- RST=`sed -i 's/\# export JAVA_HOME=\/usr\/java\/jdk1.6.0\//export JAVA_HOME=\/opt\/software\/java180/' $HBASE_HOME/conf/hbase-env.sh`RST=`sed -i 's/\# export HBASE_MANAGES_ZK=true/export HBASE_MANAGES_ZK=false/' $HBASE_HOME/conf/hbase-env.sh` CONFI=" <property> <name>hbase.rootdir</name> <value>hdfs://${HOST_IP}:9000/hbase</value> </property> <property> <name>hbase.cluster.distributed</name> <value>true</value> </property> <property> <name>hbase.zookeeper.property.clientPort</name> <value>2181</value> </property> <property> <name>hbase.zookeeper.property.dataDir</name> <value>$ZOOKEEPER_HOME/mydata</value> </property> " for line in $CONFI do RST=`eval "sed -i '/<\/configuration>/i\$line' $HBASE_HOME/conf/hbase-site.xml"` done 

讯享网
小讯
上一篇 2025-03-19 22:55
下一篇 2025-04-01 20:49

相关推荐

版权声明:本文内容由互联网用户自发贡献,该文观点仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容,请联系我们,一经查实,本站将立刻删除。
如需转载请保留出处:https://51itzy.com/kjqy/80544.html