1. 修复了kinit -kt /workspace/conf/user.keytab ailab 在本地环境 不需要调用的逻辑

2. 配置了本地新镜像jupyter:0.98

app/core/airflow/af_util.py

@@ -8,7 +8,7 @@ from configs.settings import config
 
 
 def spark_result_tb_name(job_id, task_id, spark_node_id, out_pin, is_tmp=False):
-    return f'job{job_id}_task{task_id}_subnode{spark_node_id}_output{out_pin}{"_tmp" if is_tmp else ""}'
+    return f'{config.get("HIVE","database_name")}.`job{job_id}_task{task_id}_subnode{spark_node_id}_output{out_pin}{"_tmp" if is_tmp else ""}`'
 
 
 def get_sub_task_script_uri(task_id, sub_node_id):

app/core/airflow/task.py

@@ -57,6 +57,7 @@ class PythonTaskCompiler(TaskCompiler):
         self.default_cmd = "echo \"$SCRIPT\" > run.py && python run.py"
 
 
+
 class DataXTaskCompiler(TaskCompiler):
     def __init__(self, item: AirflowTask):
         super(DataXTaskCompiler, self).__init__(item)
@@ -117,7 +118,12 @@ class SparksTaskCompiler(TaskCompiler):
                         }
         param_str = ' '.join([f'--{k} {v}' for k, v in parameters.items()])
         param_str += ''.join([f' --conf {k}={v}' for k, v in spark_config.items()])
-        basic_cmds = "kinit -kt /workspace/conf/user.keytab ailab && cd /workspace && echo \"$SCRIPT\" > run.py && ${SPARK_HOME}/bin/spark-submit"
+
+        basic_cmds = "cd /workspace && echo \"$SCRIPT\" > run.py && ${SPARK_HOME}/bin/spark-submit"
+
+        if config.get('K8S', 'enable_kerberos', fallback=None) in ['true', "True", True]:
+            basic_cmds = f"kinit -kt /workspace/conf/user.keytab ailab && {basic_cmds}"
+
         self.cmd_str = lambda name: f"{basic_cmds} --name {name} {param_str} run.py"
 
     def translate(self, job_id, task_mode=1):

auo_tests/spark_submit/docker-compose.yml

@@ -4,13 +4,15 @@ services:
     hostname: ai2_spark_submit
     container_name: ai2_spark_submit
     restart: always
-    image: SXKJ:32775/jupyter:0.96
+    image: SXKJ:32775/jupyter:latest
     privileged: true
     ipc: host
     tty: true
     working_dir: /workspace
     volumes:
-      - /home/sxkj/luoyulong/ai2/ai2_be/auo_tests/spark_submit:/workspace
+      - /home/sxkj/luoyulong/ai2/ai2_be/auo_tests/spark_submit:/workdir
+
+
 #      - /mnt/nfs/airflow-airflow-dags-pvc-b2638332-6249-4a45-b99e-7a54dc63482f/fc309d7dd0f5c1de9299e5e9a222a098faec1de0:/dags
 #    ports:
 #      - '18082:8080'

auo_tests/spark_submit/spark-submit_1009.sh

@@ -7,15 +7,15 @@ ${SPARK_HOME}/bin/spark-submit \
   --master yarn \
   --name "spark_demo_1009" \
   --deploy-mode cluster \
-  --driver-memory 2g \
-  --driver-cores 4 \
-  --executor-memory 2g \
-  --executor-cores 2 \
+  --driver-memory 1g \
+  --driver-cores 1 \
+  --executor-memory 1g \
+  --executor-cores 1 \
   --num-executors 1 \
   --archives ${sourceDIR}/py37.zip#python3env \
-  --conf spark.default.parallelism=2 \
-  --conf spark.executor.memoryOverhead=4g \
-  --conf spark.driver.memoryOverhead=2g \
+  --conf spark.default.parallelism=1 \
+  --conf spark.executor.memoryOverhead=1g \
+  --conf spark.driver.memoryOverhead=1g \
   --conf spark.yarn.maxAppAttempts=3 \
   --conf spark.yarn.submit.waitAppCompletion=true \
   --conf spark.pyspark.driver.python=${sourceDIR}/py37/bin/python \

auo_tests/spark_submit/spark-submit_lr.sh

@@ -34,15 +34,15 @@ ${SPARK_HOME}/bin/spark-submit \
   --master yarn \
   --name "spark_${ModelType}_${who}" \
   --deploy-mode cluster \
-  --driver-memory 2g \
-  --driver-cores 4 \
-  --executor-memory 2g \
-  --executor-cores 2 \
+  --driver-memory 1g \
+  --driver-cores 1 \
+  --executor-memory 1g \
+  --executor-cores 1 \
   --num-executors 1 \
   --archives ${sourceDIR}/py37.zip#python3env \
-  --conf spark.default.parallelism=2 \
-  --conf spark.executor.memoryOverhead=4g \
-  --conf spark.driver.memoryOverhead=2g \
+  --conf spark.default.parallelism=1 \
+  --conf spark.executor.memoryOverhead=1g \
+  --conf spark.driver.memoryOverhead=1g \
   --conf spark.yarn.maxAppAttempts=3 \
   --conf spark.yarn.submit.waitAppCompletion=true \
   --conf spark.pyspark.driver.python=${sourceDIR}/py37/bin/python \

debug.ipynb

@@ -39,8 +39,12 @@
       "port=18082\n",
       "dag_files_dir=/dags/\n",
       "\n",
+      "[BACKEND]\n",
+      "url=192.168.199.107:18082\n",
+      "\n",
       "[K8S]\n",
-      ";image_pull_key=\n",
+      ";image_pull_key=codingregistrykey\n",
+      "enable_kerberos=true\n",
       "\n",
       "[AIRFLOW]\n",
       "uri=192.168.199.109\n",
@@ -80,10 +84,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "outputs": [],
+   "execution_count": 5,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'str'>\n"
+     ]
+    }
+   ],
    "source": [
-    "config.get('K8S', 'image_pull_key', fallback=None)"
+    "print ( type(config.get('K8S', 'enable_kerberos', fallback=None)))"
    ],
    "metadata": {
     "collapsed": false
@@ -91,7 +103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 4,
    "outputs": [
     {
      "name": "stdout",
@@ -108,6 +120,25 @@
     "collapsed": false
    }
   },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "cd /workspace && echo \"$SCRIPT\" > run.py && ${SPARK_HOME}/bin/spark-submit --name spark_94_aN6SiQdZ8a7OPhnhPKnJU --master yarn --deploy-mode cluster --driver-memory 1g --driver-cores  1 --executor-memory 1g --executor-cores 1 --num-executors 1 --archives /workspace/py37.zip#python3env --conf spark.default.parallelism=1 --conf spark.executor.memoryOverhead=1g --conf spark.driver.memoryOverhead=1g --conf spark.yarn.maxAppAttempts=1 --conf spark.yarn.submit.waitAppCompletion=true --conf spark.pyspark.driver.python=python3env/py37/bin/python --conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=python3env/py37/bin/python --conf spark.pyspark.python=python3env/py37/bin/python run.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('cd /workspace && echo \"$SCRIPT\" > run.py && ${SPARK_HOME}/bin/spark-submit --name spark_94_aN6SiQdZ8a7OPhnhPKnJU --master yarn --deploy-mode cluster --driver-memory 1g --driver-cores X 1 --executor-memory 1g --executor-cores 1 --num-executors 1 --archives /workspace/py37.zip#python3env --conf spark.default.parallelism=1 --conf spark.executor.memoryOverhead=1g --conf spark.driver.memoryOverhead=1g --conf spark.yarn.maxAppAttempts=1 --conf spark.yarn.submit.waitAppCompletion=true --conf spark.pyspark.driver.python=python3env/py37/bin/python --conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=python3env/py37/bin/python --conf spark.pyspark.python=python3env/py37/bin/python run.py')"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
   {
    "cell_type": "code",
    "execution_count": null,

development.ini

@@ -23,6 +23,7 @@ url=192.168.199.107:18082
 
 [K8S]
 ;image_pull_key=codingregistrykey
+enable_kerberos=false
 
 [AIRFLOW]
 uri=192.168.199.109
@@ -45,4 +46,4 @@ uris=thrift://192.168.199.27:9083
 datax=SXKJ:32775/pod_datax:0.9
 python=SXKJ:32775/pod_python:1.1
 java=SXKJ:32775/java:1.0
-sparks=SXKJ:32775/jupyter:0.96
+sparks=SXKJ:32775/jupyter:0.98

production.ini

@@ -19,6 +19,7 @@ dag_files_dir=/dags/
 
 [K8S]
 image_pull_key=codingregistrykey
+enable_kerberos=true
 
 [BACKEND]
 url=aihub-backend-yili-test:8080