Merge branch 'yili' of http://gogsb.soaringnova.com/ylproj/jupyterlab into yili

Dockerfile.dev

@@ -76,7 +76,7 @@ FROM builder1 as builder2
 WORKDIR /workspace
 
 RUN /opt/conda/bin/pip install jupyterlab jupyterlab-language-pack-zh-CN -i https://pypi.douban.com/simple
-RUN mamba install nodejs  sparkmagic   -y
+RUN mamba install nodejs  sparkmagic  pyhive pandas -y
 # pyhive thrift sasl thrift_sasl scikit-learn
 # password 'sx'
 RUN /opt/conda/bin/jupyter lab --generate-config && \
@@ -115,10 +115,10 @@ stdout_logfile_maxbytes=50MB\n\
 FROM builder3 as image-txtest
 COPY --from=builder-fe /src/dist  dist
 RUN /opt/conda/bin/pip install dist/*.whl  -i https://mirror.baidu.com/pypi/simple
-ADD confs/test/krb5.conf /etc/
-ADD confs/test/config.json .
+ADD confs/txtest/krb5.conf /etc/
+ADD confs/txtest/config.json .
 RUN mkdir -p $HOME/.sparkmagic && cp config.json $HOME/.sparkmagic
-ADD confs/test/user.keytab /
+ADD confs/txtest/user.keytab /
 RUN echo "\
 [program:jupyter]\n\
 directory=/workspace\n\
@@ -157,6 +157,7 @@ COPY --from=builder-fe /src/dist  dist
 RUN /opt/conda/bin/pip install dist/*.whl  -i https://mirror.baidu.com/pypi/simple
 ADD confs/idctest/config.json .
 RUN mkdir -p $HOME/.sparkmagic && cp config.json $HOME/.sparkmagic
+COPY confs/idctest/ydtk /ydtk
 RUN echo "\
 [program:jupyter]\n\
 directory=/workspace\n\
@@ -169,12 +170,31 @@ stdout_logfile_maxbytes=50MB\n\
 " > /etc/supervisor/conf.d/jupyter.conf
 
 
+FROM builder2 as jupyter0-idctest
+COPY --from=builder-fe /src/dist  dist
+RUN /opt/conda/bin/pip install dist/*.whl  -i https://mirror.baidu.com/pypi/simple
+COPY confs/idctest/ydtk /ydtk
+RUN echo "\
+[program:jupyter]\n\
+directory=/workspace\n\
+command=/bin/bash -c '/opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser --NotebookApp.allow_origin=* --NotebookApp.allow_remote_access=1' \n\
+autorestart=true\n\
+startretries=0\n\
+redirect_stderr=true\n\
+stdout_logfile=/var/log/jupyter.log\n\
+stdout_logfile_maxbytes=50MB\n\
+environment=PYTHONPATH=/\n\
+" > /etc/supervisor/conf.d/jupyter.conf
+
+
+
 # 电信云生产
 FROM builder3 as image-idcprod
 COPY --from=builder-fe /src/dist  dist
 RUN /opt/conda/bin/pip install dist/*.whl  -i https://mirror.baidu.com/pypi/simple
 ADD confs/idcprod/config.json .
 RUN mkdir -p $HOME/.sparkmagic && cp config.json $HOME/.sparkmagic
+COPY confs/idcprod/ydtk /ydtk
 RUN echo "\
 [program:jupyter]\n\
 directory=/workspace\n\
@@ -184,9 +204,29 @@ startretries=0\n\
 redirect_stderr=true\n\
 stdout_logfile=/var/log/jupyter.log\n\
 stdout_logfile_maxbytes=50MB\n\
+environment=PYTHONPATH=/\n\
+" > /etc/supervisor/conf.d/jupyter.conf
+
+
+
+FROM builder2 as jupyter0-idcprod
+COPY --from=builder-fe /src/dist  dist
+RUN /opt/conda/bin/pip install dist/*.whl  -i https://mirror.baidu.com/pypi/simple
+COPY confs/idcprod/ydtk /ydtk
+RUN echo "\
+[program:jupyter]\n\
+directory=/workspace\n\
+command=/bin/bash -c '/opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser --NotebookApp.allow_origin=* --NotebookApp.allow_remote_access=1' \n\
+autorestart=true\n\
+startretries=0\n\
+redirect_stderr=true\n\
+stdout_logfile=/var/log/jupyter.log\n\
+stdout_logfile_maxbytes=50MB\n\
+environment=PYTHONPATH=/\n\
 " > /etc/supervisor/conf.d/jupyter.conf
 
 # 普通版
+
 FROM builder2 as jupyter0
 
 COPY --from=builder-fe /src/dist  dist

Makefile

@@ -34,7 +34,7 @@ idcprod:
 	@sed 's#http.*\.com#http://aihub-dag-idc\.digitalyili\.com#' -i  packages/jldbq-extenison/src/api/config.ts
 	@sed 's#http.*\.com#http://aihub-dag-idc\.digitalyili\.com#' -i  packages/filebrowser/src/api/config.ts
 	@docker build -f Dockerfile.dev  --target image-idcprod -t registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab:idcprod .
-	@docker build -f Dockerfile.dev  --target jupyter0 -t registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab0:idcprod .
+	@docker build -f Dockerfile.dev  --target jupyter0-idcprod -t registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab0:idcprod .
 	@docker push registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab:idcprod
 	@docker push registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab0:idcprod
 
@@ -44,7 +44,7 @@ txtest:
 	@sed 's#http.*\.com#http://aihub-dag-test\.digitalyili\.com#' -i  packages/jldbq-extenison/src/api/config.ts
 	@sed 's#http.*\.com#http://aihub-dag-test\.digitalyili\.com#' -i  packages/filebrowser/src/api/config.ts
 	@docker build -f Dockerfile.dev  --target image-txtest -t registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab:txtest .
-	@docker build -f Dockerfile.dev  --target jupyter0 -t registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab0:txtest .
+	@docker build -f Dockerfile.dev  --target jupyter0-idctest -t registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab0:txtest .
 	@docker push registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab:txtest
 	@docker push registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab0:txtest
 

confs/idcprod/ydtk/utils.py

@@ -0,0 +1,9 @@
+from pyhive import hive
+import pandas as pd
+
+connection = hive.Connection(host='10.150.5.20', database='dataming', port=10000,  username='aiuser', password='qaz2wsx#', auth='CUSTOM')
+
+
+def get_df(table):
+    df = pd.read_sql(f'select * from dataming.`{table}`', connection)
+    return df

confs/idctest/config.json

@@ -2,19 +2,19 @@
     "kernel_python_credentials": {
         "username": "",
         "password": "",
-        "url": "http://10.138.143.16:8998",
+        "url": "http://10.138.143.25:8998",
         "auth": "None"
     },
     "kernel_scala_credentials": {
         "username": "",
         "password": "",
-        "url": "http://10.138.143.16:8998",
+        "url": "http://10.138.143.25:8998",
         "auth": "None"
     },
     "kernel_r_credentials": {
         "username": "",
         "password": "",
-        "url": "http://10.138.143.16:8998"
+        "url": "http://10.138.143.25:8998"
     },
     "logging_config": {
         "version": 1,

confs/idctest/ydtk/utils.py

@@ -0,0 +1,9 @@
+from pyhive import hive
+import pandas as pd
+
+connection = hive.Connection(host='10.116.1.75', database='dataming', port=10000,  username='aiuser', password='qaz2wsx#', auth='CUSTOM')
+
+
+def get_df(table):
+    df = pd.read_sql(f'select * from dataming.`{table}`', connection)
+    return df