2019独角兽企业重金招聘Python工程师标准>>> 
我们都知道在Hadoop 2.7.0中系统刚开始执行时需要执行一个命令就是
hadoop namenode -format
下面我们就来逐行分析下这个命令对应的脚本的真正内涵
---
binwhich $0bindirname ${bin}bincd "$bin"; pwd
打印出来的bin值为
/root/hadoop-2.7.0-bin/bin
---
DEFAULT_LIBEXEC_DIR"$bin"/../libexec
则导致DEFAULT_LIBEXEC_DIR的值为
/root/hadoop-2.7.0-bin/bin/../libexec
---
cygwinfalsecase "$(uname)" inCYGWIN*) cygwintrue;;esac
这个一看就知道跟cywin相关也就是如果你在windows下执行这个脚本才有意义我的是linux,无视这段代码
---
HADOOP_LIBEXEC_DIR${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
打印其值为
/root/hadoop-2.7.0-bin/bin/../libexec
---
接下来执行一个脚本
. $HADOOP_LIBEXEC_DIR/hadoop-config.sh
打印的值是/root/hadoop-2.7.0-bin/bin/../libexec/hadoop-config.sh
实际上也就是
/root/hadoop-2.7.0-bin/libexec/hadoop-config.sh
所以先转向这个脚本看看
|
|
|
---解析/root/hadoop-2.7.0-bin/libexec/hadoop-config.sh
刚开始是
this"${BASH_SOURCE-$0}"common_bin$(cd -P -- "$(dirname -- "$this")" "$(basename -- "$this")"this"$common_bin/$script"
打印出所有的值为
this/root/hadoop-2.7.0-bin/libexec/hadoop-config.shcommon_bin/root/hadoop-2.7.0-bin/libexecscripthadoop-config.sh
---
然后执行一段脚本
[ -f "$common_bin/hadoop-layout.sh" ] 这里并没有这个脚本所以不需要执行
---
接下来是一大波赋值语句
HADOOP_COMMON_DIR${HADOOP_COMMON_DIR:-"share/hadoop/common"}HADOOP_COMMON_LIB_JARS_DIR${HADOOP_COMMON_LIB_JARS_DIR:-"share/hadoop/common/lib"}HADOOP_COMMON_LIB_NATIVE_DIR${HADOOP_COMMON_LIB_NATIVE_DIR:-"lib/native"}HDFS_DIR${HDFS_DIR:-"share/hadoop/hdfs"}HDFS_LIB_JARS_DIR${HDFS_LIB_JARS_DIR:-"share/hadoop/hdfs/lib"}YARN_DIR${YARN_DIR:-"share/hadoop/yarn"}YARN_LIB_JARS_DIR${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}MAPRED_DIR${MAPRED_DIR:-"share/hadoop/mapreduce"}MAPRED_LIB_JARS_DIR${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
打印值如下
HADOOP_COMMON_DIRshare/hadoop/commonHADOOP_COMMON_LIB_JARS_DIRshare/hadoop/common/libHADOOP_COMMON_LIB_NATIVE_DIRlib/nativeHDFS_DIRshare/hadoop/hdfsHDFS_LIB_JARS_DIRshare/hadoop/hdfs/libYARN_DIRshare/hadoop/yarnYARN_LIB_JARS_DIRshare/hadoop/yarn/libMAPRED_DIRshare/hadoop/mapreduceMAPRED_LIB_JARS_DIRshare/hadoop/mapreduce/lib
---
接下来是一小波赋值语句
HADOOP_DEFAULT_PREFIX$(cd -P -- "$common_bin"/.. ${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX}export HADOOP_PREFIX
打印值如下
HADOOP_DEFAULT_PREFIX/root/hadoop-2.7.0-binHADOOP_PREFIX/root/hadoop-2.7.0-bin
---
接下来是一段判断语句
#check to see if the conf dir is given as an optional argumentif [ $# -gt 1 ]thenif [ "--config" "$1" ]thenshiftconfdir$1if [ ! -d "$confdir" ]; thenecho "Error: Cannot find configuration directory: $confdir"exit 1fishiftHADOOP_CONF_DIR$confdirfifi
执行结果如下
本次命令下没有执行
---
if [ $# -gt 1 ]thenif [ "--loglevel" "$1" ]thenshiftHADOOP_LOGLEVEL$1shiftfifiHADOOP_LOGLEVEL"${HADOOP_LOGLEVEL:-INFO}"
执行结果为
HADOOP_LOGLEVELINFO
---
# Allow alternate conf dir location.if [ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]; thenDEFAULT_CONF_DIR"conf"elseDEFAULT_CONF_DIR"etc/hadoop"fiexport HADOOP_CONF_DIR"${HADOOP_CONF_DIR:-$HADOOP_PREFIX/$DEFAULT_CONF_DIR}"
打印为
HADOOP_CONF_DIR/root/hadoop-2.7.0-bin/etc/hadoop
---
接下来是判断脚本
# User can specify hostnames or a file where the hostnames are (not both)if [[ ( "$HADOOP_SLAVES" ! ) ) ]] ; thenecho \"Error: Please specify one variable HADOOP_SLAVES or " \"HADOOP_SLAVE_NAME and not both."exit 1fi
结果为
通过
---
# Process command line options that specify hosts or file with host# listif [ $# -gt 1 ]thenif [ "--hosts" "$1" ]thenshiftexport HADOOP_SLAVES"${HADOOP_CONF_DIR}/$1"shiftelif [ "--hostnames" "$1" ]thenshiftexport HADOOP_SLAVE_NAMES$1shiftfifi
执行结果为
很抱歉任何一个分支都没有被执行
---
# User can specify hostnames or a file where the hostnames are (not both)# (same check as above but now we know its command line options that cause# the problem)if [[ ( "$HADOOP_SLAVES" ! ) ) ]] ; thenecho \"Error: Please specify one of --hosts or --hostnames options and not both."exit 1fi
显然执行结果为不执行
---
接下来又执行另一个脚本
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then. "${HADOOP_CONF_DIR}/hadoop-env.sh"fi
路径为
/root/hadoop-2.7.0-bin/etc/hadoop/hadoop-env.sh
下面来执行这个脚本
|
|
|
---/root/hadoop-2.7.0-bin/etc/hadoop/hadoop-env.sh
这个脚本全部是export,就不细细分析了。
让我们再回到hadoop-config.sh
---
接下来是
# check if net.ipv6.bindv6only is set to 1bindv6only$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)if [ -n "$bindv6only" ] "yes" ]thenecho "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken"echo "For more info: http://wiki.apache.org/hadoop/HadoopIPv6"exit 1fi
显然不执行
---
接下来是
# Newer versions of glibc use an arena memory allocator that causes virtual# memory usage to explode. This interacts badly with the many threads that# we use in Hadoop. Tune the variable down to prevent vmem explosion.export MALLOC_ARENA_MAX${MALLOC_ARENA_MAX:-4}
打印
MALLOC_ARENA_MAX4
---
接下来是JAVA_HOME
# Attempt to set JAVA_HOME if it is not setif [[ -z $JAVA_HOME ]]; then# On OSX use java_home (or /Library for older versions)if [ "Darwin" "$(uname -s)" ]; thenif [ -x /usr/libexec/java_home ]; thenexport JAVA_HOME($(/usr/libexec/java_home))elseexport JAVA_HOME(/Library/Java/Home)fifi# Bail if we did not detect itif [[ -z $JAVA_HOME ]]; thenecho "Error: JAVA_HOME is not set and could not be found." 1>/usr/java/jdk1.8.0_45
---
JAVA$JAVA_HOME/bin/java# some Java parametersJAVA_HEAP_MAX-Xmx1000m # check envvars which might override default argsif [ "$HADOOP_HEAPSIZE" ! "" ]; then#echo "run with heapsize $HADOOP_HEAPSIZE"JAVA_HEAP_MAX"-Xmx""$HADOOP_HEAPSIZE""m"#echo $JAVA_HEAP_MAXfi
打印值为
JAVA_HEAP_MAX-Xmx1000m
---
# CLASSPATH initially contains $HADOOP_CONF_DIRCLASSPATH"${HADOOP_CONF_DIR}"
打印
CLASSPATH/root/hadoop-2.7.0-bin/etc/hadoop
---
# so that filenames w/ spaces are handled correctly in loops belowIFSif [ "$HADOOP_COMMON_HOME" "" ]; thenif [ -d "${HADOOP_PREFIX}/$HADOOP_COMMON_DIR" ]; thenexport HADOOP_COMMON_HOME$HADOOP_PREFIXfifi
打印
HADOOP_COMMON_HOME/root/hadoop-2.7.0-bin
---
# for releases, add core hadoop jar thenCLASSPATH${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIRfiif [ -d "$HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR" ]; thenCLASSPATH${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR/*fiCLASSPATH${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR/*
打印值
CLASSPATH/root/hadoop-2.7.0-bin/etc/hadoop:/root/hadoop-2.7.0-bin/share/hadoop/common/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/common/*
---
# default log directory "" ]; thenHADOOP_LOG_DIR"$HADOOP_PREFIX/logs"fiif [ "$HADOOP_LOGFILE" "" ]; thenHADOOP_LOGFILEhadoop.logfi# default policy file for service-level authorizationif [ "$HADOOP_POLICYFILE" "" ]; thenHADOOP_POLICYFILE"hadoop-policy.xml"fi# restore ordinary behaviourunset IFS
打印
HADOOP_LOG_DIR/root/hadoop-2.7.0-bin/logsHADOOP_LOGFILEhadoop.logHADOOP_POLICYFILEhadoop-policy.xml
---
# setup java.library.path for native-hadoop code if necessaryif [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; thenif [ -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; thenif [ "x$JAVA_LIBRARY_PATH" ! "x" ]; thenJAVA_LIBRARY_PATH${JAVA_LIBRARY_PATH}:${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIRelseJAVA_LIBRARY_PATH${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIRfififi
打印
JAVA_LIBRARY_PATH/root/hadoop-2.7.0-bin/lib/native
---
# setup a default TOOL_PATHTOOL_PATH"${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}"HADOOP_OPTS"$HADOOP_OPTS -Dhadoop.log.dir$HADOOP_LOG_DIR"HADOOP_OPTS"$HADOOP_OPTS -Dhadoop.log.file$HADOOP_LOGFILE"HADOOP_HOME$HADOOP_PREFIXif $cygwin; thenHADOOP_HOME$(cygpath -w "$HADOOP_HOME" 2>/dev/null)fiexport HADOOP_HOMEHADOOP_OPTS"$HADOOP_OPTS -Dhadoop.home.dir$HADOOP_HOME"HADOOP_OPTS"$HADOOP_OPTS -Dhadoop.id.str$HADOOP_IDENT_STRING"HADOOP_OPTS"$HADOOP_OPTS -Dhadoop.root.logger${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"if [ "x$JAVA_LIBRARY_PATH" ! "x" ]; thenif $cygwin; thenJAVA_LIBRARY_PATH$(cygpath -w "$JAVA_LIBRARY_PATH" 2>/dev/null)fiHADOOP_OPTS"$HADOOP_OPTS -Djava.library.path$JAVA_LIBRARY_PATH"export LD_LIBRARY_PATH$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATHfi HADOOP_OPTS"$HADOOP_OPTS -Dhadoop.policy.file$HADOOP_POLICYFILE"# Disable ipv6 as it can cause issuesHADOOP_OPTS"$HADOOP_OPTS -Djava.net.preferIPv4Stacktrue"
一大波赋值语句就不多说了。
---
# put hdfs in classpath if presentif [ "$HADOOP_HDFS_HOME" "" ]; thenif [ -d "${HADOOP_PREFIX}/$HDFS_DIR" ]; thenexport HADOOP_HDFS_HOME$HADOOP_PREFIXfifi
打印
HADOOP_HDFS_HOME/root/hadoop-2.7.0-bin
---
if [ -d "$HADOOP_HDFS_HOME/$HDFS_DIR/webapps" ]; thenCLASSPATH${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_DIRfiif [ -d "$HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR" ]; thenCLASSPATH${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR/*fiCLASSPATH${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_DIR/*
打印
CLASSPATH/root/hadoop-2.7.0-bin/etc/hadoop:/root/hadoop-2.7.0-bin/share/hadoop/common/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/common/*:/root/hadoop-2.7.0-bin/share/hadoop/hdfs:/root/hadoop-2.7.0-bin/share/hadoop/hdfs/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/hdfs/*
---
# put yarn in classpath if presentif [ "$HADOOP_YARN_HOME" "" ]; thenif [ -d "${HADOOP_PREFIX}/$YARN_DIR" ]; thenexport HADOOP_YARN_HOME$HADOOP_PREFIXfifi
打印
HADOOP_YARN_HOME/root/hadoop-2.7.0-bin
---
if [ -d "$HADOOP_YARN_HOME/$YARN_DIR/webapps" ]; thenCLASSPATH${CLASSPATH}:$HADOOP_YARN_HOME/$YARN_DIRfiif [ -d "$HADOOP_YARN_HOME/$YARN_LIB_JARS_DIR" ]; thenCLASSPATH${CLASSPATH}:$HADOOP_YARN_HOME/$YARN_LIB_JARS_DIR/*fiCLASSPATH${CLASSPATH}:$HADOOP_YARN_HOME/$YARN_DIR/*# put mapred in classpath if present AND different from YARNif [ "$HADOOP_MAPRED_HOME" "" ]; thenif [ -d "${HADOOP_PREFIX}/$MAPRED_DIR" ]; thenexport HADOOP_MAPRED_HOME$HADOOP_PREFIXfifiif [ "$HADOOP_MAPRED_HOME/$MAPRED_DIR" ! "$HADOOP_YARN_HOME/$YARN_DIR" ] ; thenif [ -d "$HADOOP_MAPRED_HOME/$MAPRED_DIR/webapps" ]; thenCLASSPATH${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIRfiif [ -d "$HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR" ]; thenCLASSPATH${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR/*fiCLASSPATH${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR/*fi
打印
CLASSPATH/root/hadoop-2.7.0-bin/etc/hadoop:/root/hadoop-2.7.0-bin/share/hadoop/common/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/common/*:/root/hadoop-2.7.0-bin/share/hadoop/hdfs:/root/hadoop-2.7.0-bin/share/hadoop/hdfs/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/hdfs/*:/root/hadoop-2.7.0-bin/share/hadoop/yarn/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/yarn/*:/root/hadoop-2.7.0-bin/share/hadoop/mapreduce/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/mapreduce/*
---
# Add the user-specified CLASSPATH via HADOOP_CLASSPATH# Add it first or last depending on if user has# set env-var HADOOP_USER_CLASSPATH_FIRST# if the user set HADOOP_USE_CLIENT_CLASSLOADER, HADOOP_CLASSPATH is not added# to the classpathif [[ ( "$HADOOP_CLASSPATH" ! "" ) "" ) ]]; then# Prefix it if its to be precededif [ "$HADOOP_USER_CLASSPATH_FIRST" ! "" ]; thenCLASSPATH${HADOOP_CLASSPATH}:${CLASSPATH}elseCLASSPATH${CLASSPATH}:${HADOOP_CLASSPATH}fifi
打印
CLASSPATH/root/hadoop-2.7.0-bin/etc/hadoop:/root/hadoop-2.7.0-bin/share/hadoop/common/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/common/*:/root/hadoop-2.7.0-bin/share/hadoop/hdfs:/root/hadoop-2.7.0-bin/share/hadoop/hdfs/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/hdfs/*:/root/hadoop-2.7.0-bin/share/hadoop/yarn/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/yarn/*:/root/hadoop-2.7.0-bin/share/hadoop/mapreduce/lib/*:/root/hadoop-2.7.0-bin/share/hadoop/mapreduce/*:/contrib/capacity-scheduler/*.jar
---然后再回到最开始的hadoop脚本
|
|
|
---继续执行
function print_usage(){echo "Usage: hadoop [--config confdir] [COMMAND | CLASSNAME]"echo " CLASSNAME run the class named CLASSNAME"echo " or"echo " where COMMAND is one of:"echo " fs run a generic filesystem user client"echo " version print the version"echo " jar run a jar file"echo " note: please use \"yarn jar\" to launch"echo " YARN applications, not this command."echo " checknative [-a|-h] check native hadoop and compression libraries availability"echo " distcp copy file or directories recursively"echo " archive -archiveName NAME -p * create a hadoop archive"echo " classpath prints the class path needed to get the"echo " credential interact with credential providers"echo " Hadoop jar and the required libraries"echo " daemonlog get/set the log level for each daemon"echo " trace view and modify Hadoop tracing settings"echo ""echo "Most commands print help when invoked w/o parameters."}
这个就不多说了
不多细说
---
if [ $# 0 ]; thenprint_usageexitfi
这个就不用我多说了。
---接下来开始执行命令
走的分支为
#hdfs commandsnamenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|fetchdt|oiv|dfsgroups|portmap|nfs3)exitecho "DEPRECATED: Use of this script to execute hdfs command is deprecated." 1> thenexec "${HADOOP_HDFS_HOME}"/bin/hdfs ${COMMAND/dfsgroups/groups} "$"elif [ -f "${HADOOP_PREFIX}"/bin/hdfs ]; thenexec "${HADOOP_PREFIX}"/bin/hdfs ${COMMAND/dfsgroups/groups} "$"elseecho "HADOOP_HDFS_HOME not found!"exit 1fi;;
最终执行的是
/root/hadoop-2.7.0-bin/bin/hdfs namenode -format
下一篇来讲解这个命令。
转:https://my.oschina.net/qiangzigege/blog/423648