add hostalias

app/core/airflow/task.py

@@ -1,5 +1,5 @@
 import json
-from typing import Optional
+from typing import Optional, Dict
 from app.core.airflow.af_util import spark_result_tb_name
 from app.schemas import AirflowTask
 from jinja2 import Environment, PackageLoader, select_autoescape
@@ -58,6 +58,10 @@ class PythonTaskCompiler(TaskCompiler):
         self.default_image = config.get('TASK_IMAGES', 'python')  # 'SXKJ:32775/pod_python:1.1'
         self.default_cmd = "python main.py"
         self.task.cmd = "echo \"$SCRIPT\" > main.py && " + (self.task.cmd or self.default_cmd)
+        if config.get('HOST_ALIAS', 'enable', fallback=None) in ['true', "True", True]:
+            host_alias: Dict = json.loads(config.get('HOST_ALIAS', 'host_alias'))
+            for k, v in host_alias.items():
+                self.task.cmd = f"echo '{k} {v}' >> /etc/hosts && {self.task.cmd}"
 
 
 class DataXTaskCompiler(TaskCompiler):
@@ -125,9 +129,16 @@ class SparksTaskCompiler(TaskCompiler):
 
         basic_cmds = "cd /workspace && echo \"$SCRIPT\" > run.py && ${SPARK_HOME}/bin/spark-submit"
 
+        if config.get('HOST_ALIAS', 'enable', fallback=None) in ['true', "True", True]:
+            host_alias: Dict = json.loads(config.get('HOST_ALIAS', 'host_alias'))
+            for k, v in host_alias.items():
+                basic_cmds = f"echo '{k} {v}' >> /etc/hosts && {basic_cmds}"
+
+
         if config.get('K8S', 'enable_kerberos', fallback=None) in ['true', "True", True]:
             basic_cmds = f"kinit -kt /workspace/conf/user.keytab ailab && {basic_cmds}"
 
+
         self.cmd_str = lambda name: f"{basic_cmds} --name {name} {param_str} run.py"
 
     def translate(self, job_id, task_mode=1):

auo_tests/tasks/development.ini

@@ -13,4 +13,5 @@ port = 10086
 [MINIO]
 url = minio-api.sxkj.com
 access_key = admin
-secret_key = sxkjadmin
+secret_key = sxkjadmin
+

idctest.ini

@@ -71,3 +71,17 @@ kerberos_config = {
                     "kerberosKeytabFilePath": "/workspace/confs/test/user.keytab",
                     "kerberosPrincipal": "ailab@EMR-5XJSY31F"
                   }
+[HOST_ALIAS]
+enable = true
+host_alias = {
+       "10.116.1.78": "cdhtestdn08.yili.com",
+        "10.116.1.77": "cdhtestdn07.yili.com",
+        "10.116.1.76": "cdhtestdn06.yili.com",
+        "10.116.1.75": "cdhtestdn05.yili.com",
+        "10.116.1.74": "cdhtestdn04.yili.com",
+        "10.116.1.73": "cdhtestdn03.yili.com",
+        "10.116.1.72": "cdhtestdn02.yili.com",
+        "10.116.1.71": "cdhtestdn01.yili.com",
+        "10.119.14.67": "cdhtestnm01.yili.com",
+        "10.119.14.68": "cdhtestnm02.yili.com"
+    }

production.ini

@@ -71,3 +71,6 @@ kerberos_config = {
                     "kerberosKeytabFilePath": "/workspace/confs/test/user.keytab",
                     "kerberosPrincipal": "ailab@EMR-5XJSY31F"
                   }
+[HOST_ALIAS]
+enable = false
+host_alias = {}