Merge branch 'integration' of http://gogsb.soaringnova.com/sxwl_DL/datax-admin into integration

.gitignore

@@ -1,3 +1,4 @@
 __pycache__/
 /.idea/
-/.vscode/
+/.vscode/
+.DS_Store

app/core/airflow/task.py

@@ -136,7 +136,8 @@ class SparksTaskCompiler(TaskCompiler):
 
 
         if config.get('K8S', 'enable_kerberos', fallback=None) in ['true', "True", True]:
-            basic_cmds = f"kinit -kt /workspace/conf/user.keytab ailab && {basic_cmds}"
+            principal = config.get('HIVE', 'principal', fallback=None)
+            basic_cmds = f"kinit -kt /workspace/conf/user.keytab {principal} && {basic_cmds}"
 
 
         self.cmd_str = lambda name: f"{basic_cmds} --name {name} {param_str} run.py"

app/core/airflow/templates/dag_template.py.jinja2

@@ -59,6 +59,7 @@ with DAG(start_date=datetime(2022,6,1),catchup=False,schedule_interval=None if '
                                                                     random_name_suffix=True,
                                                                     labels={'app':'backend', 'env':'dev', 'run_ts':{{ "'{{ ts_nodash }}'" }} },
                                                                     reattach_on_restart=True,
+                                                                    startup_timeout_seconds=600,
                                                                     is_delete_operator_pod=False,
                                                                     get_logs=True,
                                                                     {% if image_pull_key != None or image_pull_key != ""%}image_pull_secrets='{{ image_pull_key }}',{% endif %}

txprod.ini

@@ -0,0 +1,104 @@
+[DATABASE]
+user = aihub_prod
+pwd = RttN4RbjQBfv
+db_name = aihub-dag-prod
+host = 172.23.12.194
+port = 3306
+ssl_disabled = true
+
+[MINIO]
+k8s_url = aihub-dag-minio:9000
+url = aihub-dag-minio:9000
+access_key = minioadmin
+secret_key = minioadmin
+
+
+[AF_BACKEND]
+uri=aihub-dag-backend-af:8080
+host=aihub-dag-backend-af
+port=8080
+dag_files_dir=/dags/
+
+[K8S]
+image_pull_key=codingregistrykey
+enable_kerberos=true
+
+[BACKEND]
+url=aihub-dag-backend:8080
+
+[AIRFLOW]
+uri=airflow-webserver:8080
+api_token=YWRtaW46YWRtaW4=
+
+
+
+[HIVE]
+host = 172.23.21.7
+port = 7001
+username = hive
+password = hive
+database_name = ailab
+kerberos = 1
+keytab = assets/txprod/user.keytab
+krb5config = assets/txprod/krb5.conf
+kerberos_service_name = hadoop
+principal = ylaiuser@EMR-56L6ZNTS
+base_path = /user/hive/warehouse/ailab.db/
+
+
+[HIVE_METASTORE]
+uris=thrift://172.23.21.7:7004,thrift://172.23.21.8:7004
+
+[TASK_IMAGES]
+datax=registry.cn-hangzhou.aliyuncs.com/sxtest/txprod:datax
+python=registry.cn-hangzhou.aliyuncs.com/sxtest/txprod:python
+java=registry.cn-hangzhou.aliyuncs.com/sxtest/txprod:java
+sparks=registry.cn-hangzhou.aliyuncs.com/sxtest/txprod:spark
+
+
+
+[HADOOP_INNER]
+default_fs = hdfs://HDFS84854
+hadoop_config={
+            "dfs.nameservices": "HDFS84854",
+            "dfs.ha.namenodes.HDFS84854": "nn1,nn2",
+            "dfs.namenode.rpc-address.HDFS84854.nn1": "172.23.21.7:4007",
+            "dfs.namenode.rpc-address.HDFS84854.nn2": "172.23.21.8:4007",
+            "dfs.client.failover.proxy.provider.HDFS84854": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+             }
+kerberos_config = {
+                    "haveKerberos": "true",
+                    "kerberosKeytabFilePath": "/workspace/confs/user.keytab",
+                    "kerberosPrincipal": "ailab@EMR-5XJSY31F"
+                  }
+
+[LAKE_HADOOP_INNER]
+default_fs = hdfs://HDFS84854
+hadoop_config={
+            "dfs.nameservices": "HDFS84854",
+            "dfs.ha.namenodes.HDFS84854": "nn1,nn2",
+            "dfs.namenode.rpc-address.HDFS84854.nn1": "172.23.21.7:4007",
+            "dfs.namenode.rpc-address.HDFS84854.nn2": "172.23.21.8:4007",
+            "dfs.client.failover.proxy.provider.HDFS84854": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+             }
+kerberos_config = {
+                    "haveKerberos": "true",
+                    "kerberosKeytabFilePath": "/workspace/confs/user.keytab",
+                    "kerberosPrincipal": "ylaiuser@EMR-56L6ZNTS"
+                  }
+
+[HOST_ALIAS]
+enable = false
+host_alias = {}
+
+
+[PROGRAMME]
+url = aihub-dag-helmbe:8080
+namespace = aihub-prod
+super_image = registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab
+ordinary_image = registry.cn-hangzhou.aliyuncs.com/sxtest/jupyterlab
+tag = txprod
+host = aihub-dag.digitalyili.com
+chart = aihub-dag-jupyter.tgz
+path_type = ImplementationSpecific
+ingress_class = prod-pri-ingress

txtest.ini

@@ -93,7 +93,9 @@ host_alias = {}
 
 
 [PERMISSIONS]
+; 超级管理员
 super_admin_role = TENANT_ADMIN
+; 算法组项目id
 special_project_id = bbcs0929
 
 [PROGRAMME]