1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283 |
- FROM maven:3.8.6-openjdk-8 as builder
- WORKDIR /workspace
- RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list
- RUN apt update && apt install -y python3 python3-pip python-is-python3 \
- && pip config set global.index-url https://mirror.baidu.com/pypi/simple \
- && pip install -U setuptools
- ADD . .
- RUN mkdir -p ~/.m2 && cp -r docker/settings.xml ~/.m2
- # RUN wget http://mirror.nju.edu.cn/apache/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz
- # ADD spark-3.0.3-bin-hadoop3.2.tgz .
- RUN mvn clean package \
- -B -V -e \
- -Pspark-3.0 \
- -Pthriftserver \
- -DskipTests \
- -DskipITs \
- -Dmaven.javadoc.skip=true
- FROM openjdk:8-jre-slim as builder1
- RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list
- # COPY docker/sources.list /etc/apt
- RUN apt update && apt install -y --no-install-recommends \
- krb5-user zip unzip procps tar curl supervisor
- # 目录准备
- WORKDIR /opt/cluster
- RUN mkdir -p hadoop spark livy/logs
- # spark
- COPY --from=builder /workspace/spark-3.0.3-bin-hadoop3.2.tgz spark
- RUN cd spark && tar zxfv spark-3.0.3-bin-hadoop3.2.tgz && mv spark*/* . && rm -rf spark-3.0.3-bin-hadoop3.2.tgz
- # hadoop
- RUN curl -o ./hadoop/hadoop.tgz -O https://archive.apache.org/dist/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz
- # RUN curl -o ./hadoop/hadoop.tgz -O http://mirror.nju.edu.cn/apache/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz
- RUN cd hadoop && tar zxfv hadoop.tgz && mv hadoop*/* . && rm -rf hadoop.tgz
- # livy
- COPY --from=builder /workspace/assembly/target/apache-livy*.zip livy
- RUN cd /opt/cluster/livy/ && unzip apache-livy*.zip && mv apache-livy*bin/* .
- # jar包冲突
- RUN cp hadoop/share/hadoop/common/lib/jersey-core-1.19.jar livy/jars/
- RUN cp hadoop/share/hadoop/common/lib/jersey-core-1.19.jar spark/jars/
- RUN cp hadoop/share/hadoop/yarn/lib/jersey-client-1.19.jar spark/jars/
- RUN rm spark/jars/jersey-client-2.30.jar
- # 环境变量
- ENV HADOOP_HOME=/opt/cluster/hadoop
- ENV HADOOP_CONF_DIR=/opt/cluster/conf
- ENV HIVE_HOME=/opt/cluster/hive
- ENV HIVE_CONF_DIR=/opt/cluster/conf
- ENV SPARK_HOME=/opt/cluster/spark
- ENV SPARK_CONF_DIR=/opt/cluster/conf
- ENV LIVY_HOME=/opt/cluster/livy
- ENV LIVY_CONF_DIR=/opt/cluster/conf
- # 启动命令
- RUN echo "\
- [program:livy]\n\
- directory=/opt/cluster\n\
- command=livy/bin/livy-server \n\
- autorestart=true\n\
- startretries=0\n\
- redirect_stderr=true\n\
- stdout_logfile=/var/log/livy.log\n\
- stdout_logfile_maxbytes=50MB\n\
- " > /etc/supervisor/conf.d/livy.conf
- CMD ["supervisord", "-n"]
- FROM builder1 as image-test
- COPY docker/dev .
- ADD docker/dev/conf/krb5.conf /etc
- FROM builder1 as image-prod
- COPY docker/prod .
- ADD docker/prod/conf/krb5.conf /etc
|