|
@@ -0,0 +1,202 @@
|
|
|
+FROM node:16.15.1-bullseye-slim AS builder-fe
|
|
|
+COPY requirements.txt .
|
|
|
+RUN sed -i "s@http://\(deb\|security\).debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list
|
|
|
+RUN apt update
|
|
|
+RUN apt update && apt install -y libpython3-dev python3 python3-pip python-is-python3 make \
|
|
|
+ && pip config set global.index-url https://mirror.baidu.com/pypi/simple \
|
|
|
+ && pip install -U pip setuptools && pip install -r requirements.txt -i https://mirror.baidu.com/pypi/simple
|
|
|
+ # \
|
|
|
+ # && yarn config set registry https://registry.npm.taobao.org \
|
|
|
+ # && npm config set registry https://registry.npm.taobao.org \
|
|
|
+ # && npm config set disturl https://npm.taobao.org/dist
|
|
|
+#https://registry.npmmirror.com
|
|
|
+COPY . /src/
|
|
|
+ENV PYTHONPATH=/src
|
|
|
+RUN cd /src && chmod a+x ./bdist_wheel.sh && ./bdist_wheel.sh
|
|
|
+
|
|
|
+
|
|
|
+FROM gettyimages/spark:2.4.0-hadoop-3.0 as builder
|
|
|
+RUN sed -i "s@http://\(deb\|security\).debian.org@http://mirrors.163.com@g" /etc/apt/sources.list
|
|
|
+RUN apt update && apt install -y --no-install-recommends \
|
|
|
+ krb5-user zip unzip procps tar curl supervisor net-tools telnet vim iputils-ping netcat jq wget zsh
|
|
|
+# 目录准备
|
|
|
+ENV WORKSPACE /hadoop
|
|
|
+WORKDIR ${WORKSPACE}
|
|
|
+
|
|
|
+
|
|
|
+# 环境变量
|
|
|
+ENV HADOOP_HOME=/usr/hadoop-3.0.0/
|
|
|
+ENV HADOOP_CONF_DIR=${WORKSPACE}/conf
|
|
|
+# ENV HIVE_HOME=${WORKSPACE}/hive
|
|
|
+# ENV HIVE_CONF_DIR=${WORKSPACE}/conf
|
|
|
+ENV PATH=$PATH:$HADOOP_HOME/bin:$SPARK_HOME/bin
|
|
|
+ENV LD_LIBRARY_PATH=$HADOOP_HOME/lib/native
|
|
|
+
|
|
|
+ENV LANG=zh_CN.UTF-8 LANGUAGE=zh_CN:zh LC_ALL=zh_CN.UTF-8 DEBIAN_FRONTEND=noninteractive
|
|
|
+
|
|
|
+RUN rm -rf /etc/apt/sources.list.d/ && apt update
|
|
|
+
|
|
|
+RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
|
+ supervisor \
|
|
|
+ iputils-ping \
|
|
|
+ wget \
|
|
|
+ zsh \
|
|
|
+ build-essential \
|
|
|
+ cmake \
|
|
|
+ git \
|
|
|
+ curl \
|
|
|
+ vim \
|
|
|
+ ca-certificates \
|
|
|
+ libjpeg-dev \
|
|
|
+ zip \
|
|
|
+ unzip \
|
|
|
+ libpng-dev \
|
|
|
+ openssh-server \
|
|
|
+ autojump \
|
|
|
+ ttf-wqy-zenhei \
|
|
|
+ libgl1-mesa-glx \
|
|
|
+ libglib2.0-0 \
|
|
|
+ locales \
|
|
|
+ krb5-user net-tools telnet && \
|
|
|
+ rm -rf /var/lib/apt/lists/*
|
|
|
+
|
|
|
+
|
|
|
+RUN locale-gen zh_CN.UTF-8
|
|
|
+RUN dpkg-reconfigure locales
|
|
|
+
|
|
|
+
|
|
|
+CMD ["supervisord", "-n"]
|
|
|
+
|
|
|
+
|
|
|
+FROM builder as builder1
|
|
|
+
|
|
|
+ENV PYTHON_VERSION 3
|
|
|
+RUN chsh -s `which zsh`
|
|
|
+RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh && \
|
|
|
+ chmod +x ~/miniconda.sh && \
|
|
|
+ ~/miniconda.sh -b -p /opt/conda && \
|
|
|
+ rm ~/miniconda.sh
|
|
|
+
|
|
|
+RUN ln /opt/conda/bin/conda /usr/local/bin/conda
|
|
|
+RUN conda init zsh
|
|
|
+RUN conda install mamba -n base -c conda-forge
|
|
|
+RUN ln /opt/conda/bin/mamba /usr/local/bin/mamba && mamba init zsh
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+FROM builder1 as builder2
|
|
|
+WORKDIR /workspace
|
|
|
+RUN /opt/conda/bin/pip install jupyterlab jupyterlab-language-pack-zh-CN kazoo==2.9.0 -i https://pypi.douban.com/simple
|
|
|
+RUN mamba install nodejs sparkmagic pyhive pandas nbconvert>=7 mistune>=2 -y
|
|
|
+# pyhive thrift sasl thrift_sasl scikit-learn
|
|
|
+# password 'sx'
|
|
|
+RUN /opt/conda/bin/jupyter lab --generate-config && \
|
|
|
+ echo "c.NotebookApp.password='argon2:\$argon2id\$v=19\$m=10240,t=10,p=8\$+zIUCF9Uk2FiCHlV8njX5A\$I5Mm/64DORArcXYTXWRVng'">>/root/.jupyter/jupyter_lab_config.py
|
|
|
+ENV PATH /opt/conda/bin/:$PATH
|
|
|
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
|
|
|
+
|
|
|
+
|
|
|
+FROM builder2 as builder3
|
|
|
+
|
|
|
+RUN cd /opt/conda/lib/python3.7/site-packages/ \
|
|
|
+ && jupyter-kernelspec install sparkmagic/kernels/sparkkernel \
|
|
|
+ && jupyter-kernelspec install sparkmagic/kernels/pysparkkernel \
|
|
|
+ && /opt/conda/bin/jupyter serverextension enable --py sparkmagic
|
|
|
+
|
|
|
+EXPOSE 8888
|
|
|
+
|
|
|
+# 电信云测试
|
|
|
+FROM builder3 as image-idctest
|
|
|
+COPY --from=builder-fe /src/dist dist
|
|
|
+RUN /opt/conda/bin/pip install dist/*.whl -i https://mirror.baidu.com/pypi/simple
|
|
|
+ADD confs/idctest/config.json .
|
|
|
+RUN mkdir -p $HOME/.sparkmagic && cp config.json $HOME/.sparkmagic
|
|
|
+COPY confs/idctest/ydtk /ydtk
|
|
|
+COPY confs/idctest/conf ${WORKSPACE}/conf
|
|
|
+RUN echo "\
|
|
|
+[program:jupyter]\n\
|
|
|
+directory=/workspace\n\
|
|
|
+command=/bin/bash -c '/opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser --NotebookApp.allow_origin=* --NotebookApp.allow_remote_access=1' \n\
|
|
|
+autorestart=true\n\
|
|
|
+startretries=0\n\
|
|
|
+redirect_stderr=true\n\
|
|
|
+stdout_logfile=/var/log/jupyter.log\n\
|
|
|
+stdout_logfile_maxbytes=50MB\n\
|
|
|
+environment=PYTHONPATH=/\n\
|
|
|
+" > /etc/supervisor/conf.d/jupyter.conf
|
|
|
+
|
|
|
+
|
|
|
+FROM builder2 as jupyter0-idctest
|
|
|
+COPY --from=builder-fe /src/dist dist
|
|
|
+RUN /opt/conda/bin/pip install dist/*.whl -i https://mirror.baidu.com/pypi/simple
|
|
|
+COPY confs/idctest/ydtk /ydtk
|
|
|
+COPY confs/idctest/conf ${WORKSPACE}/conf
|
|
|
+RUN echo "\
|
|
|
+[program:jupyter]\n\
|
|
|
+directory=/workspace\n\
|
|
|
+command=/bin/bash -c '/opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser --NotebookApp.allow_origin=* --NotebookApp.allow_remote_access=1' \n\
|
|
|
+autorestart=true\n\
|
|
|
+startretries=0\n\
|
|
|
+redirect_stderr=true\n\
|
|
|
+stdout_logfile=/var/log/jupyter.log\n\
|
|
|
+stdout_logfile_maxbytes=50MB\n\
|
|
|
+environment=PYTHONPATH=/\n\
|
|
|
+" > /etc/supervisor/conf.d/jupyter.conf
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+# 电信云生产
|
|
|
+FROM builder3 as image-idcprod
|
|
|
+COPY --from=builder-fe /src/dist dist
|
|
|
+RUN /opt/conda/bin/pip install dist/*.whl -i https://mirror.baidu.com/pypi/simple
|
|
|
+ADD confs/idcprod/config.json .
|
|
|
+RUN mkdir -p $HOME/.sparkmagic && cp config.json $HOME/.sparkmagic
|
|
|
+COPY confs/idcprod/ydtk /ydtk
|
|
|
+COPY confs/idcprod/conf ${WORKSPACE}/conf
|
|
|
+RUN echo "\
|
|
|
+[program:jupyter]\n\
|
|
|
+directory=/workspace\n\
|
|
|
+command=/bin/bash -c '/opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser --NotebookApp.allow_origin=* --NotebookApp.allow_remote_access=1' \n\
|
|
|
+autorestart=true\n\
|
|
|
+startretries=0\n\
|
|
|
+redirect_stderr=true\n\
|
|
|
+stdout_logfile=/var/log/jupyter.log\n\
|
|
|
+stdout_logfile_maxbytes=50MB\n\
|
|
|
+environment=PYTHONPATH=/\n\
|
|
|
+" > /etc/supervisor/conf.d/jupyter.conf
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+FROM builder2 as jupyter0-idcprod
|
|
|
+COPY --from=builder-fe /src/dist dist
|
|
|
+RUN /opt/conda/bin/pip install dist/*.whl -i https://mirror.baidu.com/pypi/simple
|
|
|
+COPY confs/idcprod/ydtk /ydtk
|
|
|
+COPY confs/idcprod/conf ${WORKSPACE}/conf
|
|
|
+RUN echo "\
|
|
|
+[program:jupyter]\n\
|
|
|
+directory=/workspace\n\
|
|
|
+command=/bin/bash -c '/opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser --NotebookApp.allow_origin=* --NotebookApp.allow_remote_access=1' \n\
|
|
|
+autorestart=true\n\
|
|
|
+startretries=0\n\
|
|
|
+redirect_stderr=true\n\
|
|
|
+stdout_logfile=/var/log/jupyter.log\n\
|
|
|
+stdout_logfile_maxbytes=50MB\n\
|
|
|
+environment=PYTHONPATH=/\n\
|
|
|
+" > /etc/supervisor/conf.d/jupyter.conf
|
|
|
+
|
|
|
+# 普通版
|
|
|
+
|
|
|
+FROM builder2 as jupyter0
|
|
|
+
|
|
|
+COPY --from=builder-fe /src/dist dist
|
|
|
+RUN /opt/conda/bin/pip install dist/*.whl -i https://mirror.baidu.com/pypi/simple
|
|
|
+RUN echo "\
|
|
|
+[program:jupyter]\n\
|
|
|
+directory=/workspace\n\
|
|
|
+command=/bin/bash -c '/opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser --NotebookApp.allow_origin=* --NotebookApp.allow_remote_access=1' \n\
|
|
|
+autorestart=true\n\
|
|
|
+startretries=0\n\
|
|
|
+redirect_stderr=true\n\
|
|
|
+stdout_logfile=/var/log/jupyter.log\n\
|
|
|
+stdout_logfile_maxbytes=50MB\n\
|
|
|
+" > /etc/supervisor/conf.d/jupyter.conf
|