CentOS下安装Hadoop

CentOS下安装Hadoop

安装前系统和用户配置

  1. Centos 7系统更新

    1
    2
    yum install epel-release
    yum update
  2. 创建用户并切换用户

    1
    2
    adduser hadoop
    passwd hadoop

    vi /etc/sudoers

    1
    2
    3
    ## Allow root to run any commands anywhere
    root ALL=(ALL) ALL
    hadoop ALL=(ALL) ALL
    1
    2
    3
    4
    5
    6
    7
    8
    9
        su - hadoop
    mkdir /home/hadoop/Downloads
    ```
    ### 安装Hadoop
    1. 设置无密码登录
    ```shell
    sudo yum install openssh-server
    ssh-keygen -t rsa # 一直回车
    cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  3. 配置java环境

    1
    2
    3
    export JAVA_HOME=/usr/java/jdk1.8.0_201-amd64
    export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar
    export PATH=$PATH:$JAVA_HOME/bin
  4. 安装Hadoop

    1
    2
    3
    wget https://archive.apache.org/dist/hadoop/core/hadoop-2.7.7/hadoop-2.7.7.tar.gz
    tar -zxvf hadoop-2.7.7.tar.gz
    mv hadoop-2.7.7.tar.gz hadoop
  5. 修改权限

    1
    sudo chown -R hadoop hadoop
  6. 修改配置文件

    • vim /etc/profile

      1
      2
      3
      export HADOOP_HOME=/usr/local/hadoop
      export HADOOP_SSH_OPTS="-p 36000"
      export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
    • vi ./etc/hadoop/core-site.xml

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      <configuration>
      <property>
      <name>hadoop.tmp.dir</name>
      <value>/home/spark/Downloads/hadoop/tmp</value>
      <description>Abase for other temporary directories.</description>
      </property>
      <property>
      <name>fs.defaultFS</name>
      <value>hdfs://localhost:9000</value>
      </property>
      </configuration>
    • vi ./etc/hadoop/hdfs-site.xml

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20
      21
      22
      <configuration>
      <property>
      <name>dfs.replication</name>
      <value>1</value>
      </property>
      <property>
      <name>dfs.datanode.data.dir</name>
      <value>file:/home/spark/Downloads/hadoop-2.7.2/data</value>
      </property>
      <property>
      <name>yarn.nodemanager.aux-services</name>
      <value>mapreduce_shuffle</value>
      </property>
      <property>
      <name>dfs.http.address</name>
      <value>0.0.0.0:50070</value>
      </property>
      <property>
      <name>dfs.permissions</name>
      <value>false</value>
      </property>
      </configuration>

      cp ./etc/hadoop/mapred-site.xml.template ./etc/hadoop/mapred-site.xml
      vi ./etc/hadoop/mapred-site.xml

      1
      2
      3
      4
      5
      6
      <configuration>
      <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
      </property>
      </configuration>
    • vi /etc/profile

      1
      2
      3
      4
      5
      6
      7
      # JAVA_HOME
      export JAVA_HOME=/usr/lib/jvm/java
      export PATH=$JAVA_HOME/bin:$PATH
      export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/tools.jar

      source /etc/profile
      echo $JAVA_HOME

修改当前java系统:alternatives –config java

  1. 启动Hadoop
    1
    2
    3
    bin/hdfs namenode -format
    ./sbin/start-dfs.sh
    jps

安全模式
hadoop dfsadmin -safemode leave