Merge remote-tracking branch 'origin/master' into independent

Dockerfile

@@ -113,6 +113,11 @@ RUN apt-get update
 RUN apt-get update && apt-get install -y --no-install-recommends krb5-user
 ADD . ${WORKDIR}
 
+FROM builder2 as image-sxkj
+#RUN apt-get update
+#RUN apt-get update && apt-get install -y --no-install-recommends krb5-user
+ADD . ${WORKDIR}
+
 # FROM builder2 as builder3
 
 # RUN apt-get update && apt-get install -y --no-install-recommends openssh-server && rm -rf /var/lib/apt/lists/*

Makefile

@@ -28,3 +28,7 @@ remote:
 idctest:
 	@docker build --target image-idctest -t registry.cn-hangzhou.aliyuncs.com/sxtest/$(NAME):idctest .
 	@docker push registry.cn-hangzhou.aliyuncs.com/sxtest/$(NAME):idctest
+
+sxkj:
+	@docker build --target image-sxkj -t SXKJ:32775/$(NAME):sxkj .
+	@docker push SXKJ:32775/$(NAME):sxkj

server.py

@@ -57,8 +57,8 @@ app.include_router(router_jm_job_info.router)
 app.include_router(router_jm_job_log.router)
 app.include_router(router_code_check.router)
 
-# AirflowJobSubmitter.auto_submit_data_transfer()
-# AirflowJobSubmitter.auto_submit_requirements_install()
+AirflowJobSubmitter.auto_submit_data_transfer()
+AirflowJobSubmitter.auto_submit_requirements_install()
 print('server init finish:)!!!')
 
 

sxkj.ini

@@ -0,0 +1,71 @@
+[DATABASE]
+user = sxwl
+pwd = sxwldba
+db_name = aihub-dag
+host = mysql.default
+port = 3306
+ssl_disabled = false
+
+[MINIO]
+k8s_url = aihub-dag-minio:9000
+url = aihub-dag-minio:9000
+access_key = minioadmin
+secret_key = minioadmin
+
+
+[AF_BACKEND]
+uri=aihub-dag-backend-af:8080
+host=aihub-dag-backend-af
+port=8080
+dag_files_dir=/dags/
+
+[K8S]
+image_pull_key=
+enable_kerberos=false
+
+[BACKEND]
+url=aihub-dag-backend:8080
+
+[AIRFLOW]
+uri=airflow-webserver:8080
+api_token=YWRtaW46YWRtaW4=
+
+
+[HIVE]
+host = 192.168.199.27
+port = 10000
+username = hive
+password = hive
+database_name = default
+kerberos=0
+keytab = assets/test/user.keytab
+krb5config = assets/test/krb5.conf
+kerberos_service_name = hadoop
+principal = ailab@EMR-5XJSY31F
+
+
+[HIVE_METASTORE]
+uris=thrift://192.168.199.27:9083
+
+[TASK_IMAGES]
+datax=SXKJ:32775/pod_datax:0.9
+python=SXKJ:32775/pod_python:1.1
+java=SXKJ:32775/java:1.0
+sparks=SXKJ:32775/jupyter:0.981
+
+
+[HADOOP_INNER]
+datasource = 1,2,3
+default_fs = hdfs://192.168.199.27:9000
+hadoop_config={
+             }
+kerberos_config = {
+                    "haveKerberos": "false",
+                    "kerberosKeytabFilePath": "/workspace/confs/test/user.keytab",
+                    "kerberosPrincipal": "ailab@EMR-5XJSY31F"
+                  }
+[HOST_ALIAS]
+enable = true
+host_alias = {
+       "192.168.199.31": "SXKJ"
+    }