Merge branch 'master' of http://gogsb.soaringnova.com/sxwl_DL/datax-admin

app/core/airflow/task.py

@@ -1,5 +1,5 @@
 import json
-from typing import Optional
+from typing import Optional, Dict
 from app.core.airflow.af_util import spark_result_tb_name
 from app.schemas import AirflowTask
 from jinja2 import Environment, PackageLoader, select_autoescape
@@ -58,6 +58,10 @@ class PythonTaskCompiler(TaskCompiler):
         self.default_image = config.get('TASK_IMAGES', 'python')  # 'SXKJ:32775/pod_python:1.1'
         self.default_cmd = "python main.py"
         self.task.cmd = "echo \"$SCRIPT\" > main.py && " + (self.task.cmd or self.default_cmd)
+        if config.get('HOST_ALIAS', 'enable', fallback=None) in ['true', "True", True]:
+            host_alias: Dict = json.loads(config.get('HOST_ALIAS', 'host_alias'))
+            for k, v in host_alias.items():
+                self.task.cmd = f"echo '{k} {v}' >> /etc/hosts && {self.task.cmd}"
 
 
 class DataXTaskCompiler(TaskCompiler):
@@ -125,9 +129,16 @@ class SparksTaskCompiler(TaskCompiler):
 
         basic_cmds = "cd /workspace && echo \"$SCRIPT\" > run.py && ${SPARK_HOME}/bin/spark-submit"
 
+        if config.get('HOST_ALIAS', 'enable', fallback=None) in ['true', "True", True]:
+            host_alias: Dict = json.loads(config.get('HOST_ALIAS', 'host_alias'))
+            for k, v in host_alias.items():
+                basic_cmds = f"echo '{k} {v}' >> /etc/hosts && {basic_cmds}"
+
+
         if config.get('K8S', 'enable_kerberos', fallback=None) in ['true', "True", True]:
             basic_cmds = f"kinit -kt /workspace/conf/user.keytab ailab && {basic_cmds}"
 
+
         self.cmd_str = lambda name: f"{basic_cmds} --name {name} {param_str} run.py"
 
     def translate(self, job_id, task_mode=1):

app/core/airflow/templates/dag_template.py.jinja2

@@ -88,6 +88,7 @@ with DAG(start_date=datetime(2022,6,1),catchup=False,schedule_interval=None if '
         get_logs=True,
         image_pull_policy='Always',
         log_events_on_failure=True,
+        pod_template_file='/opt/airflow/pod_templates/pod_template_file.yaml',
         {% if image_pull_key not in ['None',"",None]%}image_pull_secrets='{{ image_pull_key }}',{% endif %}
         cmds={{ node['cmds'] }},
         env_vars= { **{{ node['env'] }},

auo_tests/tasks/development.ini

@@ -13,4 +13,5 @@ port = 10086
 [MINIO]
 url = minio-api.sxkj.com
 access_key = admin
-secret_key = sxkjadmin
+secret_key = sxkjadmin
+

idctest.ini

@@ -34,9 +34,10 @@ api_token=YWRtaW46YWRtaW4=
 [HIVE]
 host = 10.116.1.75
 port = 10000
-username = aiuser
+username = infa
 password = aiuser
-database_name = dataming
+database_name = default
+;database_name = dataming
 kerberos = 0
 keytab = assets/test/user.keytab
 krb5config = assets/test/krb5.conf
@@ -56,17 +57,31 @@ sparks=registry.cn-hangzhou.aliyuncs.com/sxtest/idctest:spark
 
 
 [HADOOP_INNER]
-datasource = -100
-default_fs = hdfs://HDFS8000912
+datasource = 1,2,3
+default_fs = hdfs://nameservice1
 hadoop_config={
-            "dfs.nameservices": "HDFS8000912",
-            "dfs.ha.namenodes.HDFS8000912": "nn1,nn2",
-            "dfs.namenode.rpc-address.HDFS8000912.nn1": "10.254.20.18:4007",
-            "dfs.namenode.rpc-address.HDFS8000912.nn2": "10.254.20.22:4007",
-            "dfs.client.failover.proxy.provider.HDFS8000912": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+            "dfs.nameservices": "nameservice1",
+            "dfs.ha.namenodes.nameservice1": "nn1,nn2",
+            "dfs.namenode.rpc-address.nameservice1.nn1": "10.119.14.67:8020",
+            "dfs.namenode.rpc-address.nameservice1.nn2": "10.119.14.68:8020",
+            "dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
              }
 kerberos_config = {
-                    "haveKerberos": "true",
+                    "haveKerberos": "false",
                     "kerberosKeytabFilePath": "/workspace/confs/test/user.keytab",
                     "kerberosPrincipal": "ailab@EMR-5XJSY31F"
                   }
+[HOST_ALIAS]
+enable = true
+host_alias = {
+       "10.116.1.78": "cdhtestdn08.yili.com",
+        "10.116.1.77": "cdhtestdn07.yili.com",
+        "10.116.1.76": "cdhtestdn06.yili.com",
+        "10.116.1.75": "cdhtestdn05.yili.com",
+        "10.116.1.74": "cdhtestdn04.yili.com",
+        "10.116.1.73": "cdhtestdn03.yili.com",
+        "10.116.1.72": "cdhtestdn02.yili.com",
+        "10.116.1.71": "cdhtestdn01.yili.com",
+        "10.119.14.67": "cdhtestnm01.yili.com",
+        "10.119.14.68": "cdhtestnm02.yili.com"
+    }

production.ini

@@ -71,3 +71,6 @@ kerberos_config = {
                     "kerberosKeytabFilePath": "/workspace/confs/test/user.keytab",
                     "kerberosPrincipal": "ailab@EMR-5XJSY31F"
                   }
+[HOST_ALIAS]
+enable = false
+host_alias = {}