安装 请点击下载hadoop-2.6.4
解压到 /home/hdfs/hadoop
1 useradd hdfs // 创建hdfs用户 2 passwd hdfs // 设置hdfs用户的登陆密码 3 su hdfs // 切换到hdfs用户登陆 4 cd ~ // 进入hdfs的家目录 5 rz // 上传windows本地的文件到此家目录 6 tar -zxvf hadoop-2.6.4.tar.gz // 解压安装包 7 mv hadoop-2.6.4 hadoop // 重命名解压后的文件名
hdfs机器免密验证设置 1 cd ~/.ssh 2 ssh-keygen -t rsa // 一路摁回车 3 id_rsa // 私钥 4 id_rsa.pub // 公钥 5 cat id_rsa.pub >> authorized_keys // 在master上将公钥放到authorized_keys(已认证的keys)里 6 sudo scp authorized_keys hdfs@192.168.153.129:~/.ssh 7 // 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下 8 sudo scp authorized_keys hdfs@192.168.153.128:~/.ssh 9 // 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下 10 chmod 644 authorized_keys // 修改钥的权限
设置Java环境变量 1 vi ~/bashrc 2 export JAVA_HOME=/java_install_dir/jdk1.8.0_91 3 export JRE_HOME=/java_install_dir/jdk1.8.0_91/jre 4 export CLASSPATH=.: C L A S S P A T H : CLASSPATH: CLASSPATH:JAVA_HOME/lib: J R E H O M E / l i b 5 e x p o r t P A T H = JRE_HOME/lib 5 export PATH= JREHOME/lib5exportPATH=PATH: J A V A H O M E / b i n : JAVA_HOME/bin: JAVAHOME/bin:JRE_HOME/bin 6 source ~/.bashrc // 让配置生效 7 echo $JAVA_HOME 8 /java_install_dir/jdk1.8.0_91
配置 *.xml 配置hadoop-env.sh 1 export JAVA_HOME=/java_install_dir/jdk1.8.0_91
配置 yarn-env.sh 1 export JAVA_HOME=/java_install_dir/jdk1.8.0_91
配置 core-site.xml 1 2 3 fs.defaultFS 4 hdfs://wtf-ubuntu:9000 5 6 7 io.file.buffer.size 8 131072 9 10 11 hadoop.tmp.dir 12 file:/home/hdfs/hadoop/tmp 13 Abasefor other temporary directories. 14 15 16 hadoop.proxyuser.spark.hosts 17 18 19 20 hadoop.proxyuser.spark.groups 21 22
配置hdfs-site.xml dfs.namenode.secondary.http-address wtf-ubuntu:9001
<property> <name>dfs.namenode.name.dir</name> <value>file:/home/hdfs/hadoop/name</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>file:/home/hdfs/hadoop/data</value> </property> <property> <name>dfs.replication</name> <value>3</value> </property> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> </property>配置 mapred-site.xml 文件 cp mapred-site.xml.template mapred-site.xml // 复制一份模板重命名为 mapred-site.xml mapreduce.framework.name yarn mapreduce.jobhistory.address wtf-ubuntu:10020 mapreduce.jobhistory.webapp.address wtf-ubuntu:19888
配置 yarn-site.xml yarn.nodemanager.aux-services mapreduce_shuffle yarn.nodemanager.aux-services.mapreduce.shuffle.class org.apache.hadoop.mapred.ShuffleHandler yarn.resourcemanager.address wtf-ubuntu:8032 yarn.resourcemanager.scheduler.address wtf-ubuntu:8030 yarn.resourcemanager.resource-tracker.address wtf-ubuntu:8035 yarn.resourcemanager.admin.address wtf-ubuntu:8033 yarn.resourcemanager.webapp.address wtf-ubuntu:8088
启动 格式化NameNode ./bin/hdfs namenode -format 1 操作HDFS ./sbin/start-dfs.sh ./sbin/stop-dfs.sh ./sbin/start-yarn.sh ./sbin/stop-yarn.sh ./bin/hdfs dfsadmin -report
./bin/hadoop fs -mkdir /tmp ./bin/hadoop fs -mkdir /tmp/input ./bin/hadoop fs -ls /tmp/ ./bin/hadoop fs -put logs/ /tmp ./bin/hadoop fs -ls /tmp/ ./bin/hadoop fs -cat /tmp/logs/ ./bin/hadoop fs -cat /tmp/logs/hadoop-hdfs-namenode-wtf-ubuntu.log
转载自：https://blog.csdn.net/wtf0215/article/details/52039744
