|
|
@@ -0,0 +1,83 @@
|
|
|
+FROM maven:3.8.6-openjdk-8 as builder
|
|
|
+
|
|
|
+WORKDIR /workspace
|
|
|
+RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list
|
|
|
+RUN apt update && apt install -y python3 python3-pip python-is-python3 \
|
|
|
+ && pip config set global.index-url https://mirror.baidu.com/pypi/simple \
|
|
|
+ && pip install -U setuptools
|
|
|
+ADD . .
|
|
|
+RUN mkdir -p ~/.m2 && cp -r docker/settings.xml ~/.m2
|
|
|
+# RUN wget http://mirror.nju.edu.cn/apache/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz
|
|
|
+# ADD spark-3.0.3-bin-hadoop3.2.tgz .
|
|
|
+RUN mvn clean package \
|
|
|
+ -B -V -e \
|
|
|
+ -Pspark-3.0 \
|
|
|
+ -Pthriftserver \
|
|
|
+ -DskipTests \
|
|
|
+ -DskipITs \
|
|
|
+ -Dmaven.javadoc.skip=true
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+FROM openjdk:8-jre-slim as builder1
|
|
|
+RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list
|
|
|
+# COPY docker/sources.list /etc/apt
|
|
|
+RUN apt update && apt install -y --no-install-recommends \
|
|
|
+ krb5-user zip unzip procps tar curl supervisor
|
|
|
+# 目录准备
|
|
|
+WORKDIR /opt/cluster
|
|
|
+RUN mkdir -p hadoop spark livy/logs
|
|
|
+
|
|
|
+# spark
|
|
|
+COPY --from=builder /workspace/spark-3.0.3-bin-hadoop3.2.tgz spark
|
|
|
+RUN cd spark && tar zxfv spark-3.0.3-bin-hadoop3.2.tgz && mv spark*/* . && rm -rf spark-3.0.3-bin-hadoop3.2.tgz
|
|
|
+
|
|
|
+# hadoop
|
|
|
+RUN curl -o ./hadoop/hadoop.tgz -O https://archive.apache.org/dist/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz
|
|
|
+# RUN curl -o ./hadoop/hadoop.tgz -O http://mirror.nju.edu.cn/apache/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz
|
|
|
+RUN cd hadoop && tar zxfv hadoop.tgz && mv hadoop*/* . && rm -rf hadoop.tgz
|
|
|
+
|
|
|
+
|
|
|
+# livy
|
|
|
+COPY --from=builder /workspace/assembly/target/apache-livy*.zip livy
|
|
|
+RUN cd /opt/cluster/livy/ && unzip apache-livy*.zip && mv apache-livy*bin/* .
|
|
|
+
|
|
|
+# jar包冲突
|
|
|
+RUN cp hadoop/share/hadoop/common/lib/jersey-core-1.19.jar livy/jars/
|
|
|
+RUN cp hadoop/share/hadoop/common/lib/jersey-core-1.19.jar spark/jars/
|
|
|
+RUN cp hadoop/share/hadoop/yarn/lib/jersey-client-1.19.jar spark/jars/
|
|
|
+RUN rm spark/jars/jersey-client-2.30.jar
|
|
|
+
|
|
|
+# 环境变量
|
|
|
+ENV HADOOP_HOME=/opt/cluster/hadoop
|
|
|
+ENV HADOOP_CONF_DIR=/opt/cluster/conf
|
|
|
+ENV HIVE_HOME=/opt/cluster/hive
|
|
|
+ENV HIVE_CONF_DIR=/opt/cluster/conf
|
|
|
+ENV SPARK_HOME=/opt/cluster/spark
|
|
|
+ENV SPARK_CONF_DIR=/opt/cluster/conf
|
|
|
+ENV LIVY_HOME=/opt/cluster/livy
|
|
|
+ENV LIVY_CONF_DIR=/opt/cluster/conf
|
|
|
+
|
|
|
+# 启动命令
|
|
|
+RUN echo "\
|
|
|
+[program:livy]\n\
|
|
|
+directory=/opt/cluster\n\
|
|
|
+command=livy/bin/livy-server \n\
|
|
|
+autorestart=true\n\
|
|
|
+startretries=0\n\
|
|
|
+redirect_stderr=true\n\
|
|
|
+stdout_logfile=/var/log/livy.log\n\
|
|
|
+stdout_logfile_maxbytes=50MB\n\
|
|
|
+" > /etc/supervisor/conf.d/livy.conf
|
|
|
+
|
|
|
+CMD ["supervisord", "-n"]
|
|
|
+
|
|
|
+
|
|
|
+FROM builder1 as image-test
|
|
|
+COPY docker/dev .
|
|
|
+ADD docker/dev/conf/krb5.conf /etc
|
|
|
+
|
|
|
+
|
|
|
+FROM builder1 as image-prod
|
|
|
+COPY docker/prod .
|
|
|
+ADD docker/prod/conf/krb5.conf /etc
|
Zhang Li