diff --git a/hadoop/3.3.1/Dockerfile b/hadoop/3.3.1/Dockerfile new file mode 100644 index 0000000..9fc63a5 --- /dev/null +++ b/hadoop/3.3.1/Dockerfile @@ -0,0 +1,61 @@ +FROM ccr.ccs.tencentyun.com/yy-dk/centos-sshd:7.0 + +MAINTAINER Yang + +ENV HADOOP_VERSION hadoop-3.3.1 + +RUN yum install -y wget which + +RUN cd / && wget http://yy-bak.oss-cn-qingdao.aliyuncs.com/jdk/jdk-8u172-linux-x64.tar.gz \ + && tar -zvxf jdk-8u172-linux-x64.tar.gz -C /usr/local \ + && rm -rf jdk-8u172-linux-x64.tar.gz + +ENV JDK_VERSION "jdk1.8.0_172" +ENV JAVA_HOME "/usr/local/$JDK_VERSION" + +RUN cd / && wget https://downloads.apache.org/hadoop/common/$HADOOP_VERSION/$HADOOP_VERSION.tar.gz \ + && tar -zxvf $HADOOP_VERSION.tar.gz -C /usr/local \ + && rm -rf $HADOOP_VERSION.tar.gz + +ENV HADOOP_HOME "/usr/local/$HADOOP_VERSION" +ENV PATH "$PATH:$JAVA_HOME/bin" +ENV PATH "$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin" + +RUN echo "export JAVA_HOME=/usr/local/$JDK_VERSION" >> /etc/profile && \ + echo "export HADOOP_HOME=$HADOOP_HOME" >> /etc/profile && \ + echo 'export PATH=$PATH:$JAVA_HOME/bin' >> /etc/profile && \ + echo 'export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin' >> /etc/profile && \ + source /etc/profile + +RUN mkdir /conf /logs /data + +RUN echo "ZONE=Asia/Shanghai" > /etc/sysconfig/clock && \ + rm -rf /etc/localtime && \ + ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + +RUN ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa && \ + cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys + + +ENV YARN_LOG_DIR "/logs" + +RUN sed -i 's/# export JAVA_HOME=/export JAVA_HOME=\/usr\/local\/jdk1.8.0_172/g' $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + echo "export HADOOP_LOG_DIR=/logs" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + echo "export HADOOP_PREFIX=$HADOOP_HOME" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + echo "export HDFS_NAMENODE_USER=root" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + echo "export HDFS_DATANODE_USER=root" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + echo "export HDFS_SECONDARYNAMENODE_USER=root" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + echo "export YARN_RESOURCEMANAGER_USER=root" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + echo "export YARN_NODEMANAGER_USER=root" >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh && \ + sed -e "1 i\export YARN_LOG_DIR=\/logs" $HADOOP_HOME/etc/hadoop/yarn-env.sh > $HADOOP_HOME/etc/hadoop/yarn-env.sh.bak && \ + \mv $HADOOP_HOME/etc/hadoop/yarn-env.sh.bak $HADOOP_HOME/etc/hadoop/yarn-env.sh + +COPY conf/* $HADOOP_HOME/etc/hadoop/ + +COPY start.sh / + +RUN chmod 777 /start.sh + +EXPOSE 22 8020 8088 8042 9870 + +CMD ["/start.sh"] diff --git a/hadoop/3.3.1/conf/core-site.xml b/hadoop/3.3.1/conf/core-site.xml new file mode 100644 index 0000000..b79f6b8 --- /dev/null +++ b/hadoop/3.3.1/conf/core-site.xml @@ -0,0 +1,29 @@ + + + + + + + + + fs.defaultFS + hdfs://localhost:8020 + + + + hadoop.tmp.dir + /data + + diff --git a/hadoop/3.3.1/conf/hdfs-site.xml b/hadoop/3.3.1/conf/hdfs-site.xml new file mode 100644 index 0000000..778e635 --- /dev/null +++ b/hadoop/3.3.1/conf/hdfs-site.xml @@ -0,0 +1,34 @@ + + + + + + + + + dfs.namenode.secondary.http-address + localhost:50090 + + + + dfs.namenode.secondary.https-address + localhost:50091 + + + + dfs.replication + 1 + + \ No newline at end of file diff --git a/hadoop/3.3.1/conf/mapred-site.xml b/hadoop/3.3.1/conf/mapred-site.xml new file mode 100644 index 0000000..9aec834 --- /dev/null +++ b/hadoop/3.3.1/conf/mapred-site.xml @@ -0,0 +1,36 @@ + + + + + + + + + mapreduce.framework.name + yarn + + + yarn.app.mapreduce.am.env + HADOOP_MAPRED_HOME=${HADOOP_COMMON_HOME} + + + mapreduce.map.env + HADOOP_MAPRED_HOME=${HADOOP_COMMON_HOME} + + + mapreduce.reduce.env + HADOOP_MAPRED_HOME=${HADOOP_COMMON_HOME} + + diff --git a/hadoop/3.3.1/conf/masters b/hadoop/3.3.1/conf/masters new file mode 100644 index 0000000..d18580b --- /dev/null +++ b/hadoop/3.3.1/conf/masters @@ -0,0 +1 @@ +localhost \ No newline at end of file diff --git a/hadoop/3.3.1/conf/slaves b/hadoop/3.3.1/conf/slaves new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/hadoop/3.3.1/conf/slaves @@ -0,0 +1 @@ +localhost diff --git a/hadoop/3.3.1/conf/yarn-site.xml b/hadoop/3.3.1/conf/yarn-site.xml new file mode 100644 index 0000000..bc8733a --- /dev/null +++ b/hadoop/3.3.1/conf/yarn-site.xml @@ -0,0 +1,20 @@ + + + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + diff --git a/hadoop/3.3.1/start.sh b/hadoop/3.3.1/start.sh new file mode 100644 index 0000000..04b493a --- /dev/null +++ b/hadoop/3.3.1/start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + + +\cp /conf/hadoop/* $HADOOP_HOME/etc/hadoop + +echo "127.0.0.1 localhost" > /etc/hosts + +/usr/sbin/sshd -D