docker hadoop 3.0.3
This commit is contained in:
yangyu
parent
d2817fd59a
commit
0b463c7a44
54
hadoop/3.0.3/Dockerfile
Normal file
54
hadoop/3.0.3/Dockerfile
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
FROM ccr.ccs.tencentyun.com/yy-dk/centos-sshd:7.0
|
||||||
|
|
||||||
|
MAINTAINER Yang <yangyufans@qq.com>
|
||||||
|
|
||||||
|
ENV HADOOP_VERSION hadoop-3.0.3
|
||||||
|
|
||||||
|
RUN yum install -y wget which
|
||||||
|
|
||||||
|
RUN cd / && wget http://yy-bak.oss-cn-qingdao.aliyuncs.com/jdk/jdk-8u172-linux-x64.tar.gz \
|
||||||
|
&& tar -zvxf jdk-8u172-linux-x64.tar.gz -C /usr/local/ \
|
||||||
|
&& rm -rf jdk-8u172-linux-x64.tar.gz
|
||||||
|
|
||||||
|
ENV JAVA_HOME "/usr/local/jdk1.8.0_172"
|
||||||
|
ENV PATH "$PATH:$JAVA_HOME/bin"
|
||||||
|
|
||||||
|
RUN cd / && wget http://mirrors.hust.edu.cn/apache/hadoop/common/$HADOOP_VERSION/$HADOOP_VERSION.tar.gz \
|
||||||
|
&& tar -zxvf $HADOOP_VERSION.tar.gz -C /usr/local\
|
||||||
|
&& rm -rf $HADOOP_VERSION.tar.gz
|
||||||
|
|
||||||
|
RUN mkdir /conf /logs /data
|
||||||
|
|
||||||
|
RUN echo "ZONE=Asia/Shanghai" > /etc/sysconfig/clock && \
|
||||||
|
rm -rf /etc/localtime && \
|
||||||
|
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
|
||||||
|
|
||||||
|
RUN ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa && \
|
||||||
|
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
|
||||||
|
|
||||||
|
ENV HADOOP_HOME "/usr/local/$HADOOP_VERSION"
|
||||||
|
|
||||||
|
ENV YARN_LOG_DIR /logs
|
||||||
|
|
||||||
|
RUN sed -i 's/# export JAVA_HOME=/export JAVA_HOME=\/usr\/local\/jdk1.8.0_172/g' /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
echo "export HADOOP_LOG_DIR=/logs" >> /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
echo "export HADOOP_PREFIX=$HADOOP_HOME" >> /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
echo "export HDFS_NAMENODE_USER=root" >> /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
echo "export HDFS_DATANODE_USER=root" >> /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
echo "export HDFS_SECONDARYNAMENODE_USER=root" >> /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
echo "export YARN_RESOURCEMANAGER_USER=root" >> /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
echo "export YARN_NODEMANAGER_USER=root" >> /usr/local/$HADOOP_VERSION/etc/hadoop/hadoop-env.sh && \
|
||||||
|
sed -e "1 i\export YARN_LOG_DIR=\/logs" $HADOOP_HOME/etc/hadoop/yarn-env.sh > $HADOOP_HOME/etc/hadoop/yarn-env.sh.bak && \
|
||||||
|
\mv $HADOOP_HOME/etc/hadoop/yarn-env.sh.bak $HADOOP_HOME/etc/hadoop/yarn-env.sh
|
||||||
|
|
||||||
|
COPY conf/* /usr/local/$HADOOP_VERSION/etc/hadoop/
|
||||||
|
|
||||||
|
ENV PATH "$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin"
|
||||||
|
|
||||||
|
COPY start.sh /
|
||||||
|
|
||||||
|
RUN chmod 777 /start.sh
|
||||||
|
|
||||||
|
EXPOSE 22 8020 8088 8042 9870
|
||||||
|
|
||||||
|
CMD ["/start.sh"]
|
||||||
29
hadoop/3.0.3/conf/core-site.xml
Normal file
29
hadoop/3.0.3/conf/core-site.xml
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. See accompanying LICENSE file.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Put site-specific property overrides in this file. -->
|
||||||
|
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>fs.defaultFS</name>
|
||||||
|
<value>hdfs://localhost:8020</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>hadoop.tmp.dir</name>
|
||||||
|
<value>/data</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
34
hadoop/3.0.3/conf/hdfs-site.xml
Normal file
34
hadoop/3.0.3/conf/hdfs-site.xml
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. See accompanying LICENSE file.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Put site-specific property overrides in this file. -->
|
||||||
|
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>dfs.namenode.secondary.http-address</name>
|
||||||
|
<value>localhost:50090</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>dfs.namenode.secondary.https-address</name>
|
||||||
|
<value>localhost:50091</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>dfs.replication</name>
|
||||||
|
<value>1</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
24
hadoop/3.0.3/conf/mapred-site.xml
Normal file
24
hadoop/3.0.3/conf/mapred-site.xml
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
<?xml version="1.0"?>
|
||||||
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. See accompanying LICENSE file.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Put site-specific property overrides in this file. -->
|
||||||
|
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>mapreduce.framework.name</name>
|
||||||
|
<value>yarn</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
1
hadoop/3.0.3/conf/masters
Normal file
1
hadoop/3.0.3/conf/masters
Normal file
@ -0,0 +1 @@
|
|||||||
|
localhost
|
||||||
1
hadoop/3.0.3/conf/slaves
Normal file
1
hadoop/3.0.3/conf/slaves
Normal file
@ -0,0 +1 @@
|
|||||||
|
localhost
|
||||||
20
hadoop/3.0.3/conf/yarn-site.xml
Normal file
20
hadoop/3.0.3/conf/yarn-site.xml
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
<?xml version="1.0"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. See accompanying LICENSE file.
|
||||||
|
-->
|
||||||
|
<configuration>
|
||||||
|
<property>
|
||||||
|
<name>yarn.nodemanager.aux-services</name>
|
||||||
|
<value>mapreduce_shuffle</value>
|
||||||
|
</property>
|
||||||
|
</configuration>
|
||||||
8
hadoop/3.0.3/start.sh
Normal file
8
hadoop/3.0.3/start.sh
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
|
||||||
|
\cp /conf/* /usr/local/$HADOOP_VERSION/etc/hadoop
|
||||||
|
|
||||||
|
echo "127.0.0.1 localhost" > /etc/hosts
|
||||||
|
|
||||||
|
/usr/sbin/sshd -D
|
||||||
Loading…
Reference in New Issue
Block a user