add conf & sparkmagic

.dockerignore

@@ -1,7 +1,9 @@
 .git
 .dockerignore
 Dockerfile
-
+Dockerfile.dev
+docker-compose.yml
+sparkmagic/
 # project files
 coverage/
 

Dockerfile.dev

@@ -5,9 +5,10 @@ RUN echo "deb http://mirror.nju.edu.cn/debian/ bullseye main contrib non-free" >
     echo "deb http://mirror.nju.edu.cn/debian/ bullseye-backports main contrib non-free" >> /etc/apt/sources.list && \
     echo "deb http://mirror.nju.edu.cn/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \
     apt update && apt install -y python3 python3-pip python-is-python3 make \
-    && pip config set global.index-url https://mirror.nju.edu.cn/pypi/web/simple \
+    && pip config set global.index-url https://pypi.douban.com/simple \
     && pip install -U pip setuptools && pip install -r requirements.txt \
-    && yarn config set registry https://registry.npmmirror.com
+    && yarn config set registry https://registry.npm.taobao.org
+#https://registry.npmmirror.com
 COPY . /src/
 ENV PYTHONPATH=/src
 RUN cd /src && chmod a+x  ./bdist_wheel.sh && ./bdist_wheel.sh
@@ -86,7 +87,7 @@ RUN /opt/conda/bin/jupyter lab --generate-config && \
 FROM builder2 as builder3
 
 RUN apt-get update &&  apt-get install -y --no-install-recommends openjdk-8-jdk  krb5-user
-RUN mamba install nodejs  sparkmagic -y
+RUN mamba install nodejs  sparkmagic  pyhive thrift sasl thrift_sasl -y
 # RUN /opt/conda/bin/pip install sparkmagipc -i https://pypi.douban.com/simple
 ENV PATH /opt/conda/bin/:$PATH
 RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
@@ -94,12 +95,15 @@ RUN cd /opt/conda/lib/python3.9/site-packages/ \
     && jupyter-kernelspec install sparkmagic/kernels/sparkkernel --user \
     && jupyter-kernelspec install sparkmagic/kernels/pysparkkernel --user\
     && /opt/conda/bin/jupyter serverextension enable --py sparkmagic
+ADD confs/krb5.conf /etc/
+ADD confs/config.json ~/.sparkmagic
 ADD emr-5xjsy31f_ailab.keytab .
+# RUN kinit -k -t emr-5xjsy31f_ailab.keytab ailab
 
-RUN mkdir -p /workspace && echo "\
+RUN echo "\
 [program:jupyter]\n\
 directory=/workspace\n\
-command=kinit -k -t emr-5xjsy31f_ailab.keytab ailab && /opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser \n\
+command=/bin/bash -c 'kinit -k -t emr-5xjsy31f_ailab.keytab ailab && /opt/conda/bin/jupyter lab --ip 0.0.0.0 --port 8888 --allow-root --no-browser' \n\
 autorestart=true\n\
 startretries=0\n\
 redirect_stderr=true\n\

bdist_wheel.sh

@@ -4,7 +4,7 @@ set -e
 
 if ! command -v jlpm &> /dev/null
 then
-    pip install --no-build-isolation --no-deps -e $(dirname "$0")
+    pip install --no-build-isolation --no-deps -e $(dirname "$0") -i https://pypi.douban.com/simple
 fi
 
 jlpm install --ignore-scripts

confs/config.json

@@ -0,0 +1,77 @@
+{
+  "kernel_python_credentials" : {
+    "username": "",
+    "password": "",
+    "url": "http://192.168.199.27:8998",
+    "auth": "None"
+  },
+
+  "kernel_scala_credentials" : {
+    "username": "",
+    "password": "",
+    "url": "http://192.168.199.27:8998",
+    "auth": "None"
+  },
+  "kernel_r_credentials": {
+    "username": "",
+    "password": "",
+    "url": "http://192.168.199.27:8998"
+  },
+
+  "logging_config": {
+    "version": 1,
+    "formatters": {
+      "magicsFormatter": {
+        "format": "%(asctime)s\t%(levelname)s\t%(message)s",
+        "datefmt": ""
+      }
+    },
+    "handlers": {
+      "magicsHandler": {
+        "class": "hdijupyterutils.filehandler.MagicsFileHandler",
+        "formatter": "magicsFormatter",
+        "home_path": "~/.sparkmagic"
+      }
+    },
+    "loggers": {
+      "magicsLogger": {
+        "handlers": ["magicsHandler"],
+        "level": "DEBUG",
+        "propagate": 0
+      }
+    }
+  },
+  "authenticators": {
+    "Kerberos": "sparkmagic.auth.kerberos.Kerberos",
+    "None": "sparkmagic.auth.customauth.Authenticator",
+    "Basic_Access": "sparkmagic.auth.basic.Basic"
+  },
+
+  "wait_for_idle_timeout_seconds": 15,
+  "livy_session_startup_timeout_seconds": 60,
+
+  "fatal_error_suggestion": "The code failed because of a fatal error:\n\t{}.\n\nSome things to try:\na) Make sure Spark has enough available resources for Jupyter to create a Spark context.\nb) Contact your Jupyter administrator to make sure the Spark magics library is configured correctly.\nc) Restart the kernel.",
+
+  "ignore_ssl_errors": false,
+
+  "session_configs": {
+    "driverMemory": "1000M",
+    "executorCores": 2
+  },
+
+  "use_auto_viz": true,
+  "coerce_dataframe": true,
+  "max_results_sql": 2500,
+  "pyspark_dataframe_encoding": "utf-8",
+
+  "heartbeat_refresh_seconds": 30,
+  "livy_server_heartbeat_timeout_seconds": 0,
+  "heartbeat_retry_seconds": 10,
+
+  "server_extension_default_kernel_name": "pysparkkernel",
+  "custom_headers": {},
+
+  "retry_policy": "configurable",
+  "retry_seconds_to_sleep_list": [0.2, 0.5, 1, 3, 5],
+  "configurable_retry_policy_max_retries": 8
+}

confs/core-site.xml

@@ -0,0 +1,256 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+    <property>
+        <name>emr.cfs.group.id.map</name>
+        <value>root:0;hadoop:500</value>
+    </property>
+
+    <property>
+        <name>emr.cfs.io.blocksize</name>
+        <value>1048576</value>
+    </property>
+
+    <property>
+        <name>emr.cfs.user.id.map</name>
+        <value>root:0;hadoop:500</value>
+    </property>
+
+    <property>
+        <name>emr.cfs.write.level</name>
+        <value>2</value>
+    </property>
+
+    <property>
+        <name>emr.temrfs.download.md5</name>
+        <value>3d695d5306f3df7cbc1eff5676acbc11</value>
+    </property>
+
+    <property>
+        <name>emr.temrfs.download.region</name>
+        <value>bj</value>
+    </property>
+
+    <property>
+        <name>emr.temrfs.download.version</name>
+        <value>3.1.0-8.1.4-1.0.6</value>
+    </property>
+
+    <property>
+        <name>emr.temrfs.tmp.cache.dir</name>
+        <value>/data/emr/hdfs/tmp/temrfs</value>
+    </property>
+
+    <property>
+        <name>fs.AbstractFileSystem.ofs.impl</name>
+        <value>com.qcloud.chdfs.fs.CHDFSDelegateFSAdapter</value>
+    </property>
+
+    <property>
+        <name>fs.cfs.impl</name>
+        <value>com.tencent.cloud.emr.CFSFileSystem</value>
+    </property>
+
+    <property>
+        <name>fs.cos.buffer.dir</name>
+        <value>/data/emr/hdfs/tmp</value>
+    </property>
+
+    <property>
+        <name>fs.cos.local_block_size</name>
+        <value>2097152</value>
+    </property>
+
+    <property>
+        <name>fs.cos.userinfo.appid</name>
+        <value>1302259445</value>
+    </property>
+
+    <property>
+        <name>fs.cos.userinfo.region</name>
+        <value>bj</value>
+    </property>
+
+    <property>
+        <name>fs.cos.userinfo.useCDN</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.block.size</name>
+        <value>67108864</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.credentials.provider</name>
+        <value>org.apache.hadoop.fs.auth.EMRInstanceCredentialsProvider</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.emrv2.instance.provider.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.impl</name>
+        <value>com.qcloud.emr.fs.TemrfsHadoopFileSystemAdapter</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.local_block_size</name>
+        <value>2097152</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.posix_bucket.fs.impl</name>
+        <value>com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.tmp.dir</name>
+        <value>/data/emr/hdfs/tmp/hadoop_cos</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.trsf.fs.AbstractFileSystem.ofs.impl</name>
+        <value>com.qcloud.chdfs.fs.CHDFSDelegateFSAdapter</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.trsf.fs.ofs.bucket.region</name>
+        <value>ap-beijing</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.trsf.fs.ofs.impl</name>
+        <value>com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.trsf.fs.ofs.tmp.cache.dir</name>
+        <value>/data/emr/hdfs/tmp/posix</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.trsf.fs.ofs.user.appid</name>
+        <value>1302259445</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.upload.buffer</name>
+        <value>mapped_disk</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.upload.buffer.size</name>
+        <value>-1</value>
+    </property>
+
+    <property>
+        <name>fs.cosn.userinfo.region</name>
+        <value>ap-beijing</value>
+    </property>
+
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://HDFS8000912</value>
+    </property>
+
+    <property>
+        <name>fs.emr.version</name>
+        <value>9c06b7b</value>
+    </property>
+
+    <property>
+        <name>fs.ofs.bucket.region</name>
+        <value>ap-beijing</value>
+    </property>
+
+    <property>
+        <name>fs.ofs.impl</name>
+        <value>com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter</value>
+    </property>
+
+    <property>
+        <name>fs.ofs.tmp.cache.dir</name>
+        <value>/data/emr/hdfs/tmp/chdfs</value>
+    </property>
+
+    <property>
+        <name>fs.ofs.user.appid</name>
+        <value>1302259445</value>
+    </property>
+
+    <property>
+        <name>fs.trash.interval</name>
+        <value>1440</value>
+    </property>
+
+    <property>
+        <name>ha.health-monitor.rpc-timeout.ms</name>
+        <value>180000</value>
+    </property>
+
+    <property>
+        <name>ha.zookeeper.session-timeout.ms</name>
+        <value>180000</value>
+    </property>
+
+    <property>
+        <name>hadoop.http.staticuser.user</name>
+        <value>hadoop</value>
+    </property>
+
+    <property>
+        <name>hadoop.logfile.count</name>
+        <value>20</value>
+    </property>
+
+    <property>
+        <name>hadoop.logfile.size</name>
+        <value>1000000000</value>
+    </property>
+
+    <property>
+        <name>hadoop.proxyuser.hadoop.groups</name>
+        <value>*</value>
+    </property>
+
+    <property>
+        <name>hadoop.proxyuser.hadoop.hosts</name>
+        <value>*</value>
+    </property>
+
+    <property>
+        <name>hadoop.security.authentication</name>
+        <value>kerberos</value>
+    </property>
+
+    <property>
+        <name>hadoop.security.authorization</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/data/emr/hdfs/tmp</value>
+    </property>
+
+    <property>
+        <name>hadoop.zk.timeout-ms</name>
+        <value>60000</value>
+    </property>
+
+    <property>
+        <name>io.compression.codec.lzo.class</name>
+        <value>com.hadoop.compression.lzo.LzoCodec</value>
+    </property>
+
+    <property>
+        <name>io.compression.codecs</name>
+        <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+    </property>
+
+</configuration>

confs/hdfs-site.xml

@@ -0,0 +1,325 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+	
+	<property>
+		<name>dfs.balance.bandwidthPerSec</name>
+		<value>10485760</value>
+	</property>
+	
+	<property>
+		<name>dfs.block.access.token.enable</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.blockdfs.webhdfs.enabled.access.token.enable</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.blocksize</name>
+		<value>134217728</value>
+	</property>
+	
+	<property>
+		<name>dfs.client.block.write.replace-datanode-on-failure.enable</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.client.failover.proxy.provider.HDFS8000912</name>
+		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+	</property>
+	
+	<property>
+		<name>dfs.client.slow.io.warning.threshold.ms</name>
+		<value>900000</value>
+	</property>
+	
+	<property>
+		<name>dfs.client.socket-timeout</name>
+		<value>60000</value>
+	</property>
+	
+	<property>
+		<name>dfs.client.use.datanode.hostname</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.handler.count</name>
+		<value>128</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.kerberos.principal</name>
+		<value>hadoop/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.keytab.file</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.max.transfer.threads</name>
+		<value>16480</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.max.xcievers</name>
+		<value>8000</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.slow.io.warning.threshold.ms</name>
+		<value>90000</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.socket.write.timeout</name>
+		<value>480000</value>
+	</property>
+	
+	<property>
+		<name>dfs.datanode.use.datanode.hostname</name>
+		<value>false</value>
+	</property>
+	
+	<property>
+		<name>dfs.encryption.key.provider.uri</name>
+		<value></value>
+	</property>
+	
+	<property>
+		<name>dfs.ha.automatic-failover.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.ha.fencing.methods</name>
+		<value>shell(/bin/true)</value>
+	</property>
+	
+	<property>
+		<name>dfs.ha.namenodes.HDFS8000912</name>
+		<value>nn1,nn2</value>
+	</property>
+	
+	<property>
+		<name>dfs.heartbeat.interval</name>
+		<value>2</value>
+	</property>
+	
+	<property>
+		<name>dfs.hosts</name>
+		<value>/usr/local/service/hadoop/etc/hadoop/hdfshosts</value>
+	</property>
+	
+	<property>
+		<name>dfs.hosts.exclude</name>
+		<value>/usr/local/service/hadoop/etc/hadoop/hdfsexcludedhosts</value>
+	</property>
+	
+	<property>
+		<name>dfs.journalnode.edits.dir</name>
+		<value>/data/emr/hdfs/journalnode</value>
+	</property>
+	
+	<property>
+		<name>dfs.journalnode.kerberos.internal.spnego.principal</name>
+		<value>HTTP/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.journalnode.kerberos.principal</name>
+		<value>hadoop/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.journalnode.keytab.file</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.avoid.read.stale.datanode</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.avoid.write.stale.datanode</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.datanode.registration.ip-hostname-check</name>
+		<value>false</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.handler.count</name>
+		<value>64</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.heartbeat.recheck-interval</name>
+		<value>90000</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.http-address.HDFS8000912.nn1</name>
+		<value>10.254.20.18:4008</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.http-address.HDFS8000912.nn2</name>
+		<value>10.254.20.22:4008</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.https-address.HDFS8000912.nn1</name>
+		<value>10.254.20.18:4009</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.https-address.HDFS8000912.nn2</name>
+		<value>10.254.20.22:4009</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.inode.attributes.provider.class</name>
+		<value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.kerberos.internal.spnego.principal</name>
+		<value>HTTP/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.kerberos.principal</name>
+		<value>hadoop/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.keytab.file</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.name.dir</name>
+		<value>/data/emr/hdfs/namenode</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.rpc-address.HDFS8000912.nn1</name>
+		<value>10.254.20.18:4007</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.rpc-address.HDFS8000912.nn2</name>
+		<value>10.254.20.22:4007</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.service.handler.count</name>
+		<value>64</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.servicerpc-address.HDFS8000912.nn1</name>
+		<value>10.254.20.18:4010</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.servicerpc-address.HDFS8000912.nn2</name>
+		<value>10.254.20.22:4010</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.shared.edits.dir</name>
+		<value>qjournal://10.254.20.23:4005;10.254.20.29:4005;10.254.20.26:4005/hadoop</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.stale.datanode.interval</name>
+		<value>30000</value>
+	</property>
+	
+	<property>
+		<name>dfs.namenode.write.stale.datanode.ratio</name>
+		<value>0.5f</value>
+	</property>
+	
+	<property>
+		<name>dfs.nameservices</name>
+		<value>HDFS8000912</value>
+	</property>
+	
+	<property>
+		<name>dfs.permissions</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>dfs.permissions.umask-mode</name>
+		<value>077</value>
+	</property>
+	
+	<property>
+		<name>dfs.replication</name>
+		<value>3</value>
+	</property>
+	
+	<property>
+		<name>dfs.secondary.namenode.kerberos.internal.spnego.principal</name>
+		<value>HTTP/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.secondary.namenode.kerberos.principal</name>
+		<value>hadoop/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.secondary.namenode.keytab.file</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>dfs.socket.timeout</name>
+		<value>900000</value>
+	</property>
+	
+	<property>
+		<name>dfs.web.authentication.kerberos.keytab</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>dfs.web.authentication.kerberos.principal</name>
+		<value>HTTP/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>dfs.webhdfs.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>ha.zookeeper.quorum</name>
+		<value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
+	</property>
+	
+	<property>
+		<name>ignore.secure.ports.for.testing</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>output.replace-datanode-on-failure</name>
+		<value>false</value>
+	</property>
+	
+</configuration>

confs/hive-site.xml

@@ -0,0 +1,405 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+
+    <property>
+        <name>datanucleus.schema.autoCreateTables</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hbase.zookeeper.quorum</name>
+        <value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
+    </property>
+
+    <property>
+        <name>hive.downloaded.resources.dir</name>
+        <value>/data/emr/hive/tmp/${hive.session.id}_resources</value>
+    </property>
+
+    <property>
+        <name>hive.exec.dynamic.partition.mode</name>
+        <value>nonstrict</value>
+    </property>
+
+    <property>
+        <name>hive.exec.failure.hooks</name>
+        <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+    </property>
+
+    <property>
+        <name>hive.exec.local.scratchdir</name>
+        <value>/data/emr/hive/tmp</value>
+    </property>
+
+    <property>
+        <name>hive.exec.parallel = true</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.exec.parallel.thread.number</name>
+        <value>16</value>
+    </property>
+
+    <property>
+        <name>hive.exec.post.hooks</name>
+        <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+    </property>
+
+    <property>
+        <name>hive.exec.pre.hooks</name>
+        <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+    </property>
+
+    <property>
+        <name>hive.execution.engine</name>
+        <value>tez</value>
+    </property>
+
+    <property>
+        <name>hive.fetch.task.aggr</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.hwi.listen.host</name>
+        <value>0.0.0.0</value>
+    </property>
+
+    <property>
+        <name>hive.hwi.listen.port</name>
+        <value>7002</value>
+    </property>
+
+    <property>
+        <name>hive.llap.daemon.output.service.port</name>
+        <value>7009</value>
+    </property>
+
+    <property>
+        <name>hive.llap.daemon.rpc.port</name>
+        <value>7007</value>
+    </property>
+
+    <property>
+        <name>hive.llap.daemon.web.port</name>
+        <value>7008</value>
+    </property>
+
+    <property>
+        <name>hive.llap.daemon.yarn.shuffle.port</name>
+        <value>7006</value>
+    </property>
+
+    <property>
+        <name>hive.llap.management.rpc.port</name>
+        <value>7005</value>
+    </property>
+
+    <property>
+        <name>hive.merge.mapfiles</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.merge.mapredfiles</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.merge.size.per.task</name>
+        <value>134217728</value>
+    </property>
+
+    <property>
+        <name>hive.merge.smallfiles.avgsize</name>
+        <value>134217728</value>
+    </property>
+
+    <property>
+        <name>hive.merge.sparkfiles</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.merge.tezfiles</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.db.encoding</name>
+        <value>UTF-8</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.event.db.notification.api.auth</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.kerberos.keytab.file</name>
+        <value>/var/krb5kdc/emr.keytab</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.kerberos.principal</name>
+        <value>hadoop/_HOST@EMR-5XJSY31F</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.metrics.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.port</name>
+        <value>7004</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.sasl.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.schema.verification</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.schema.verification.record.version</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.uris</name>
+        <value>thrift://10.254.20.18:7004,thrift://10.254.20.22:7004</value>
+    </property>
+
+    <property>
+        <name>hive.metastore.warehouse.dir</name>
+        <value>/usr/hive/warehouse</value>
+    </property>
+
+    <property>
+        <name>hive.querylog.location</name>
+        <value>/data/emr/hive/tmp</value>
+    </property>
+
+    <property>
+        <name>hive.server2.authentication</name>
+        <value>kerberos</value>
+    </property>
+
+    <property>
+        <name>hive.server2.authentication.kerberos.keytab</name>
+        <value>/var/krb5kdc/emr.keytab</value>
+    </property>
+
+    <property>
+        <name>hive.server2.authentication.kerberos.principal</name>
+        <value>hadoop/_HOST@EMR-5XJSY31F</value>
+    </property>
+
+    <property>
+        <name>hive.server2.logging.operation.log.location</name>
+        <value>/data/emr/hive/tmp/operation_logs</value>
+    </property>
+
+    <property>
+        <name>hive.server2.metrics.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.server2.support.dynamic.service.discovery</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.server2.thrift.bind.host</name>
+        <value>10.254.20.28</value>
+    </property>
+
+    <property>
+        <name>hive.server2.thrift.http.port</name>
+        <value>7000</value>
+    </property>
+
+    <property>
+        <name>hive.server2.thrift.port</name>
+        <value>7001</value>
+    </property>
+
+    <property>
+        <name>hive.server2.webui.host</name>
+        <value>0.0.0.0</value>
+    </property>
+
+    <property>
+        <name>hive.server2.webui.max.threads</name>
+        <value>50</value>
+    </property>
+
+    <property>
+        <name>hive.server2.webui.port</name>
+        <value>7003</value>
+    </property>
+
+    <property>
+        <name>hive.server2.zookeeper.namespace</name>
+        <value>hiveserver2</value>
+    </property>
+
+    <property>
+        <name>hive.service.metrics.reporter</name>
+        <value>JMX</value>
+    </property>
+
+    <property>
+        <name>hive.stats.autogather</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>hive.tez.am.max.app.attempts</name>
+        <value>5</value>
+    </property>
+
+    <property>
+        <name>hive.tez.am.task.max.failed.attempts</name>
+        <value>10</value>
+    </property>
+
+    <property>
+        <name>hive.tez.auto.reducer.parallelism</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.tez.container.size</name>
+        <value>1024</value>
+    </property>
+
+    <property>
+        <name>hive.vectorized.execution.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.vectorized.execution.mapjoin.minmax.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.vectorized.execution.mapjoin.native.fast.hashtable.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.vectorized.execution.mapjoin.native.multikey.only.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>hive.zookeeper.client.port</name>
+        <value>2181</value>
+    </property>
+
+    <property>
+        <name>hive.zookeeper.quorum</name>
+        <value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
+    </property>
+
+    <property>
+        <name>io.compression.codec.lzo.class</name>
+        <value>com.hadoop.compression.lzo.LzoCodec</value>
+    </property>
+
+    <property>
+        <name>io.compression.codecs</name>
+        <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+    </property>
+
+    <property>
+        <name>javax.jdo.option.ConnectionDriverName</name>
+        <value>com.mysql.jdbc.Driver</value>
+    </property>
+
+    <property>
+        <name>javax.jdo.option.ConnectionPassword</name>
+        <value>UYdfjsdDigf8d3U</value>
+    </property>
+
+    <property>
+        <name>javax.jdo.option.ConnectionURL</name>
+        <value>jdbc:mysql://10.254.20.16:3306/hivemetastore?useSSL=false&amp;createDatabaseIfNotExist=true&amp;characterEncoding=UTF-8</value>
+    </property>
+
+    <property>
+        <name>javax.jdo.option.ConnectionUserName</name>
+        <value>root</value>
+    </property>
+
+    <property>
+        <name>mapreduce.cluster.local.dir</name>
+        <value>/data/emr/hive/tmp/tmp</value>
+    </property>
+
+    <property>
+        <name>spark.driver.extraLibraryPath</name>
+        <value>/usr/local/service/hadoop/lib/native:/usr/local/service/hadoop/lib/native/Linux-amd64-64/lib</value>
+    </property>
+
+    <property>
+        <name>spark.eventLog.dir</name>
+        <value>hdfs://HDFS8000912/spark-history</value>
+    </property>
+
+    <property>
+        <name>spark.eventLog.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>spark.executor.extraLibraryPath</name>
+        <value>/usr/local/service/hadoop/lib/native:/usr/local/service/hadoop/lib/native/Linux-amd64-64/lib</value>
+    </property>
+
+    <property>
+        <name>spark.history.fs.cleaner.enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>spark.home</name>
+        <value>/usr/local/service/spark</value>
+    </property>
+
+    <property>
+        <name>spark.yarn.jars</name>
+        <value>hdfs:///spark/jars/*</value>
+    </property>
+
+</configuration>

confs/hosts

@@ -0,0 +1,16 @@
+10.254.20.18 10.254.20.18
+10.254.20.28 10.254.20.28
+10.254.20.23 10.254.20.23
+10.254.20.26 10.254.20.26
+10.254.20.19 10.254.20.19
+10.254.20.25 10.254.20.25
+10.254.20.22 10.254.20.22
+10.254.20.27 10.254.20.27
+10.254.20.18 10.254.20.18
+10.254.20.29 10.254.20.29
+10.254.20.30 10.254.20.30
+10.254.20.20 10.254.20.20
+10.254.20.21 10.254.20.21
+
+
+

confs/krb5.conf

@@ -0,0 +1,25 @@
+[libdefaults]
+    dns_lookup_realm = false
+    dns_lookup_kdc = false
+    ticket_lifetime = 24h
+    renew_lifetime = 7d
+    forwardable = true
+    rdns = false
+    default_realm = EMR-5XJSY31F
+    default_tgs_enctypes = des3-cbc-sha1
+    default_tkt_enctypes = des3-cbc-sha1
+    permitted_enctypes = des3-cbc-sha1
+    kdc_timeout = 3000
+    max_retries = 3
+[realms]
+    EMR-5XJSY31F = {
+        
+        kdc = 10.254.20.18:88
+        admin_server = 10.254.20.18
+        kdc = 10.254.20.22:88
+		admin_server = 10.254.20.22
+        
+    }
+
+[domain_realm]
+# .example.com = EXAMPLE.COM

confs/yarn-site.xml

@@ -0,0 +1,431 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+	
+	<property>
+		<name>yarn.acl.enable</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>
+		<value>5000</value>
+	</property>
+	
+	<property>
+		<name>yarn.app.mapreduce.am.staging-dir</name>
+		<value>/emr/hadoop-yarn/staging</value>
+	</property>
+	
+	<property>
+		<name>yarn.authorization-provider</name>
+		<value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
+	</property>
+	
+	<property>
+		<name>yarn.log-aggregation-enable</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.log-aggregation.retain-check-interval-seconds</name>
+		<value>604800</value>
+	</property>
+	
+	<property>
+		<name>yarn.log-aggregation.retain-seconds</name>
+		<value>604800</value>
+	</property>
+	
+	<property>
+		<name>yarn.log.server.url</name>
+		<value>http://10.254.20.18:5024/jobhistory/logs</value>
+	</property>
+	
+	<property>
+		<name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
+		<value>100000</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.address</name>
+		<value>10.254.20.28:5006</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.aux-services</name>
+		<value>mapreduce_shuffle</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+		<value>org.apache.hadoop.mapred.ShuffleHandler</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.container-executor.class</name>
+		<value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.container-manager.thread-count</name>
+		<value>100</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.container-metrics.period-ms</name>
+		<value>10000</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.delete.debug-delay-sec</name>
+		<value>180</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.delete.thread-count</name>
+		<value>16</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.hostname</name>
+		<value>10.254.20.28</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.keytab</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+		<value>/hadoop-yarn</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
+		<value>/sys/fs/cgroup</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
+		<value>false</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.linux-container-executor.group</name>
+		<value>hadoop</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user</name>
+		<value>hadoop</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+		<value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.local-dirs</name>
+		<value>/data/emr/yarn/local</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.localizer.address</name>
+		<value>10.254.20.28:5007</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.log-dirs</name>
+		<value>/data/emr/yarn/logs</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.log.retain-seconds</name>
+		<value>604800</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.pmem-check-enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.principal</name>
+		<value>hadoop/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.recovery.dir</name>
+		<value>/data/emr/yarn/nodemanger-recovery</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.recovery.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.remote-app-log-dir</name>
+		<value>/emr/logs</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.resource.cpu-vcores</name>
+		<value>8</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.resource.memory-mb</name>
+		<value>29491</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+		<value>100</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.vmem-check-enabled</name>
+		<value>false</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.vmem-pmem-ratio</name>
+		<value>8</value>
+	</property>
+	
+	<property>
+		<name>yarn.nodemanager.webapp.address</name>
+		<value>10.254.20.28:5008</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.address.rm1</name>
+		<value>10.254.20.18:5000</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.address.rm2</name>
+		<value>10.254.20.22:5000</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.admin.address.rm1</name>
+		<value>10.254.20.18:5003</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.admin.address.rm2</name>
+		<value>10.254.20.22:5003</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.cluster-id</name>
+		<value>emr-5xjsy31f</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.connect.retry-interval.ms</name>
+		<value>2000</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.ha.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.ha.rm-ids</name>
+		<value>rm1,rm2</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.keytab</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.max-completed-applications</name>
+		<value>150</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
+		<value>1000</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.nodes.exclude-path</name>
+		<value>/usr/local/service/hadoop/etc/hadoop/yarnexcludedhosts</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.nodes.include-path</name>
+		<value>/usr/local/service/hadoop/etc/hadoop/yarnhosts</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.principal</name>
+		<value>hadoop/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.recovery.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
+		<value>10.254.20.18:5002</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
+		<value>10.254.20.22:5002</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.scheduler.address.rm1</name>
+		<value>10.254.20.18:5001</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.scheduler.address.rm2</name>
+		<value>10.254.20.22:5001</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.scheduler.class</name>
+		<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.store.class</name>
+		<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.webapp.address.rm1</name>
+		<value>10.254.20.18:5004</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.webapp.address.rm2</name>
+		<value>10.254.20.22:5004</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.webapp.https.address.rm1</name>
+		<value>10.254.20.18:5005</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.webapp.https.address.rm2</name>
+		<value>10.254.20.22:5005</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.zk-address</name>
+		<value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
+	</property>
+	
+	<property>
+		<name>yarn.resourcemanager.zk.state-store.address</name>
+		<value>10.254.20.23:2181,10.254.20.29:2181,10.254.20.26:2181</value>
+	</property>
+	
+	<property>
+		<name>yarn.scheduler.fair.allow-undeclared-pools</name>
+		<value>false</value>
+	</property>
+	
+	<property>
+		<name>yarn.scheduler.fair.user-as-default-queue</name>
+		<value>false</value>
+	</property>
+	
+	<property>
+		<name>yarn.scheduler.maximum-allocation-mb</name>
+		<value>262144</value>
+	</property>
+	
+	<property>
+		<name>yarn.scheduler.maximum-allocation-vcores</name>
+		<value>128</value>
+	</property>
+	
+	<property>
+		<name>yarn.scheduler.minimum-allocation-mb</name>
+		<value>16</value>
+	</property>
+	
+	<property>
+		<name>yarn.scheduler.minimum-allocation-vcores</name>
+		<value>1</value>
+	</property>
+	
+	<property>
+		<name>yarn.timeline-service.bind-host</name>
+		<value>0.0.0.0</value>
+	</property>
+	
+	<property>
+		<name>yarn.timeline-service.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.timeline-service.hostname</name>
+		<value>10.254.20.18</value>
+	</property>
+	
+	<property>
+		<name>yarn.timeline-service.http-cross-origin.enabled</name>
+		<value>true</value>
+	</property>
+	
+	<property>
+		<name>yarn.timeline-service.keytab</name>
+		<value>/var/krb5kdc/emr.keytab</value>
+	</property>
+	
+	<property>
+		<name>yarn.timeline-service.leveldb-timeline-store.path</name>
+		<value>/data/emr/yarn/timeline-db</value>
+	</property>
+	
+	<property>
+		<name>yarn.timeline-service.principal</name>
+		<value>hadoop/_HOST@EMR-5XJSY31F</value>
+	</property>
+	
+</configuration>

confs/信息.txt

@@ -0,0 +1,16 @@
+hive：ailab
+hdfs目录：/user/ailab
+
+
+ailab@EMR-5XJSY31F
+
+
+
+ip+port：10.254.20.18:4007
+username：ailab
+password：EMR-5XJSY31F
+schema：default
+
+
+principal=hadoop/_HOST@EMR-5XJSY31F;auth=kerberos
+

docker-compose.yml

@@ -4,7 +4,7 @@ services:
     # hostname: jupyter
     container_name: jupyter
     restart: always
-    image: ylproj-jupyter:latest
+    image: jupyter:dag
     privileged: true
     ipc: host
     tty: true
@@ -16,5 +16,7 @@ services:
       # - '22522:22'
       # - "10000-10010:10000-10010"
     volumes:
-      - ./config.json:/home/sxkj/.sparkmagic/config.json
+      # - ./config.json:/home/sxkj/.sparkmagic/config.json
       - ./:/workspace
+      - ./confs/krb5.conf:/etc/krb5.conf
+      - ./confs/config.json:/root/.sparkmagic/config.json