livy/docker/Dockerfile

FROM maven:3.8.6-openjdk-8 as builder

WORKDIR /workspace
RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list
RUN apt update && apt install -y python3 python3-pip python-is-python3 \
    && pip config set global.index-url https://mirror.nju.edu.cn/pypi/web/simple
ADD . .
RUN mkdir -p ~/.m2 && cp -r docker/settings.xml ~/.m2
# RUN wget http://mirror.nju.edu.cn/apache/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz
ADD spark-3.0.3-bin-hadoop3.2.tgz  .
RUN mvn clean package \
    -B -V -e \
    -Pspark-3.0 \
    -Pthriftserver \
    -DskipTests \
    -DskipITs \
    -Dmaven.javadoc.skip=true



FROM openjdk:8-jre-slim
RUN sed -i "s@http://\(deb\|security\).debian.org@https://mirrors.aliyun.com@g" /etc/apt/sources.list
# COPY docker/sources.list /etc/apt
RUN apt update && apt install -y --no-install-recommends \
    krb5-user zip unzip procps tar curl supervisor
WORKDIR /opt/cluster
COPY docker/cluster .
ENV HADOOP_HOME=/opt/cluster/hadoop
ENV HADOOP_CONF_DIR=/opt/cluster/conf
ENV HIVE_HOME=/opt/cluster/hive
ENV HIVE_CONF_DIR=/opt/cluster/conf
ENV SPARK_HOME=/opt/cluster/spark
ENV SPARK_CONF_DIR=/opt/cluster/conf
ENV LIVY_HOME=/opt/cluster/livy
ENV LIVY_CONF_DIR=/opt/cluster/conf
RUN rm -rf /opt/cluster/livy/*
# COPY docker/livy /opt/cluster/livy/

RUN mkdir -p hadoop spark livy/logs

RUN curl -o ./hadoop/hadoop.tgz -O  http://mirror.nju.edu.cn/apache/hadoop/common/hadoop-3.2.3/hadoop-3.2.3.tar.gz
#&& \
#    curl -o ./spark/spark.tgz -O  http://mirror.nju.edu.cn/apache/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz

COPY  --from=builder /workspace/spark-3.0.3-bin-hadoop3.2.tgz  spark
# RUN mv spark-3.0.3-bin-hadoop3.2.tgz

RUN cd spark && tar zxfv spark-3.0.3-bin-hadoop3.2.tgz && mv spark*/* . && rm -rf spark-3.0.3-bin-hadoop3.2.tgz
RUN cd hadoop && tar zxfv hadoop.tgz && mv hadoop*/* . && rm -rf hadoop.tgz

ADD docker/krb5.conf /etc
COPY --from=builder /workspace/assembly/target/apache-livy*.zip livy
# ADD assembly/target/apache-livy*.zip /opt/cluster/livy/
RUN cd /opt/cluster/livy/ && unzip apache-livy*.zip && mv apache-livy*bin/* .
# CMD ["livy/bin/livy-server"]


RUN echo "\
[program:livy]\n\
directory=/opt/cluster\n\
command=livy/bin/livy-server \n\
autorestart=true\n\
startretries=0\n\
redirect_stderr=true\n\
stdout_logfile=/var/log/livy.log\n\
stdout_logfile_maxbytes=50MB\n\
" > /etc/supervisor/conf.d/livy.conf

CMD ["supervisord", "-n"]