centos7安装hadoop

时间:2017-03-16 10:36:30 阅读:867次
centos7安装hadoop

注: nodeManager所在服务器内存最好在2G以上 否则nodeManager可能启动失败


1.三台服务器 

    192.168.0.116 master

    192.168.0.131  node

    192.168.0.115  node2

    

    三台服务器分别执行 vi /etc/hosts 添加

    192.168.0.116 master

    192.168.0.131 node

    192.168.0.115 node2


2.ssh免密码登录

    三台服务器分别执行:

        vi /etc/ssh/sshd_config

            blob.png

        1.创建一个用户和组,并修改密码   

            groupadd hadoop

            useradd hadoop -g hadoop

            passwd hadoop

        2.  su hadoop

            ssh-keygen -t dsa

            cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys            

                chmod 0600 ~/.ssh/authorized_keys

            3.    scp hadoop@master:~/.ssh/id_dsa.pub master_dsa

                    cat ~/.ssh/master_dsa >> ~/.ssh/authorized_keys

            4.     三台服务器分别测试 ssh localhost 第一次链接需要密码,第二次不需要密码直接登录即为设置成功


3.安装hadoop

        1.安装java   yum install java java-devel

     2.安装hadoop 去官网下载hadoop,我使用的是2.7.3版本 binary

     3.hadoop解压后路径 ~/hadoop/hadoop-2.7.3 全路径 /home/hadoop/hadoop/hadoop-2.7.3

     4.配置环境 编辑 /etc/profile 和 ~/.bashrc 添加:

            export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.121-0.b13.el7_3.x86_64/jre

            export HADOOP_INSTALL=/home/hadoop/hadoop/hadoop-2.7.3

            export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_INSTALL/bin

     5.配置hadoop,在hadoop-2.7.3下创建文件夹  data name hdfs tmp 并赋予写权限

            core-site.xml

            <configuration>

                <property>

                    <name>fs.defaultFS</name>

                    <value>hdfs://master:9000</value>

                </property>

                <property>

                    <name>hadoop.tmp.dir</name>

                    <value>file:/home/hadoop/hadoop/hadoop-2.7.3/tmp</value>

                </property>

                <property>

                    <name>io.file.buffer.size</name>

                    <value>131702</value>

                </property>

            </configuration>

            

            hdfs-site.xml

            <configuration>

                <property>

                    <name>dfs.namenode.name.dir</name>

                    <value>file:///home/hadoop/hadoop/hadoop-2.7.3/name</value>

                </property>

                <property>

                    <name>dfs.datanode.data.dir</name>

                    <value>file:///home/hadoop/hadoop/hadoop-2.7.3/data</value>

                </property>

            

                <property>

                    <name>dfs.namenode.secondary.http-address</name>

                    <value>master:9001</value>

                </property>

            

                <property>

                    <name>dfs.replication</name>

                    <value>2</value>

                </property>

            </configuration>

            

            

            mapred-site.xml

            <configuration>

                <property>

                    <name>mapreduce.framework.name</name>

                    <value>yarn</value>

                </property>

                <property>

                    <name>mapreduce.jobhistory.address</name>

                    <value>master:10020</value>

                </property>

                <property>

                    <name>mapreduce.jobhistory.webapp.address</name>

                    <value>master:19888</value>

                </property>

                <property>

                    <name>mapred.job.tracker</name>

                    <value>hdfs://master:8021</value>

                </property>

            </configuration>

            

            

            yarn-site.xml

            <configuration>

                <property>

                    <name>yarn.nodemanager.aux-services</name>

                    <value>mapreduce_shuffle</value>

                </property>

                <property>

                    <name>yarn.resourcemanager.address</name>

                    <value>master:8032</value>

                </property>

                <property>

                    <name>yarn.resourcemanager.scheduler.address</name>

                    <value>master:8030</value>

                </property>

                <property>

                    <name>yarn.resourcemanager.resource-tracker.address</name>

                    <value>master:8031</value>

                </property>

                <property>

                    <name>yarn.resourcemanager.admin.address</name>

                    <value>master:8033</value>

                </property>

                <property>

                    <name>yarn.resourcemanager.webapp.address</name>

                    <value>master:8088</value>

                </property>

                <property>

                    <name>yarn.nodemanager.resource.memory-mb</name>

                    <value>1024</value>

                </property>

                <property>

                    <name>yarn.nodemanager.resource.cpu-vcores</name>

                    <value>1</value>

                </property>

            </configuration>

 

            slaves

            192.168.0.115

            192.168.0.131

        6.运行 

            ./bin/hdfs namenode -format

            ./sbin/start-all.sh

            jps

            master截图

            blob.png

            node 截图

            blob.png


hadoop fs -lsr   /                       查看HDFS的文件结构
hadoop fs -mkdir /tmp/test              在HDFS上新建文件夹
hadoop fs -put /home/cjy/VMwareTools-9.6.1-1378637.tar.gz /tmp/test  把本地文件传到HDFS的/tmp/test目录下
hadoop fs -get   /tmp/test/VMwareTools-9.6.1-1378637.tar.gz /home/  把HDFS文件拉到本地
hadoop fs -ls    /tmp                 列出HDFS的某目录
hadoop fs -cat   /tmp/test/VMwareTools-9.6.1-1378637.tar.gz             查看HDFS上的文件
hadoop fs -rm    /tmp/test/VMwareTools-9.6.1-1378637.tar.gz             删除HDFS上的文件
hadoop fs -rmr   /tmp/test                删除HDFS上的目录
hadoop dfsadmin -report 查看HDFS状态,比如有哪些datanode,每个datanode的情况
hadoop dfsadmin -safemode leave  离开安全模式
hadoop dfsadmin -safemode enter  进入安全模式


扫描二维码关注程序员爱笔记,接收更多资讯

评论

快速评论