FROM maven:3.8.6-openjdk-8 as builder WORKDIR /workspace RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list RUN apt update && apt install -y python3 python3-pip python-is-python3 \ && pip config set global.index-url https://mirror.baidu.com/pypi/simple \ && pip install -U setuptools ADD . . RUN mkdir -p ~/.m2 && cp -r docker/settings.xml ~/.m2 # RUN wget http://mirror.nju.edu.cn/apache/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz # ADD spark-3.0.3-bin-hadoop3.2.tgz . RUN mvn clean package \ -B -V -e \ -Pspark-3.0 \ -Pthriftserver \ -DskipTests \ -DskipITs \ -Dmaven.javadoc.skip=true FROM openjdk:8-jre-slim as builder1 RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list # COPY docker/sources.list /etc/apt RUN apt update && apt install -y --no-install-recommends \ krb5-user zip unzip procps tar curl supervisor # 目录准备 WORKDIR /opt/cluster RUN mkdir -p hadoop spark livy/logs # spark COPY --from=builder /workspace/spark-3.0.3-bin-hadoop3.2.tgz spark RUN cd spark && tar zxfv spark-3.0.3-bin-hadoop3.2.tgz && mv spark*/* . && rm -rf spark-3.0.3-bin-hadoop3.2.tgz # hadoop RUN curl -o ./hadoop/hadoop.tgz -O https://archive.apache.org/dist/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz # RUN curl -o ./hadoop/hadoop.tgz -O http://mirror.nju.edu.cn/apache/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz RUN cd hadoop && tar zxfv hadoop.tgz && mv hadoop*/* . && rm -rf hadoop.tgz # livy COPY --from=builder /workspace/assembly/target/apache-livy*.zip livy RUN cd /opt/cluster/livy/ && unzip apache-livy*.zip && mv apache-livy*bin/* . # jar包冲突 RUN cp hadoop/share/hadoop/common/lib/jersey-core-1.19.jar livy/jars/ RUN cp hadoop/share/hadoop/common/lib/jersey-core-1.19.jar spark/jars/ RUN cp hadoop/share/hadoop/yarn/lib/jersey-client-1.19.jar spark/jars/ RUN rm spark/jars/jersey-client-2.30.jar # 环境变量 ENV HADOOP_HOME=/opt/cluster/hadoop ENV HADOOP_CONF_DIR=/opt/cluster/conf ENV HIVE_HOME=/opt/cluster/hive ENV HIVE_CONF_DIR=/opt/cluster/conf ENV SPARK_HOME=/opt/cluster/spark ENV SPARK_CONF_DIR=/opt/cluster/conf ENV LIVY_HOME=/opt/cluster/livy ENV LIVY_CONF_DIR=/opt/cluster/conf # 启动命令 RUN echo "\ [program:livy]\n\ directory=/opt/cluster\n\ command=livy/bin/livy-server \n\ autorestart=true\n\ startretries=0\n\ redirect_stderr=true\n\ stdout_logfile=/var/log/livy.log\n\ stdout_logfile_maxbytes=50MB\n\ " > /etc/supervisor/conf.d/livy.conf CMD ["supervisord", "-n"] FROM builder1 as image-test COPY docker/dev . ADD docker/dev/conf/krb5.conf /etc FROM builder1 as image-prod COPY docker/prod . ADD docker/prod/conf/krb5.conf /etc