|
@@ -115,7 +115,7 @@ class AirflowJobSubmitter:
|
|
|
for k, v in host_alias.items():
|
|
for k, v in host_alias.items():
|
|
|
cmds.append(f"echo '{k} {v}' >> /etc/hosts")
|
|
cmds.append(f"echo '{k} {v}' >> /etc/hosts")
|
|
|
cmds.append("cd /workspace")
|
|
cmds.append("cd /workspace")
|
|
|
- cmds.append("./py37/bin/pip3 install {{ dag_run.conf.get('requirements_str','numpy') }} -i https://pypi.douban.com/simple/")
|
|
|
|
|
|
|
+ cmds.append("./py37/bin/pip3 install {{ dag_run.conf.get('requirements_str','numpy') }} -i https://mirror.baidu.com/pypi/simple")
|
|
|
cmds.append("zip -q -r /tmp/py37.zip py37")
|
|
cmds.append("zip -q -r /tmp/py37.zip py37")
|
|
|
cmds.append("hdfs dfs -put -f /tmp/py37.zip {{dag_run.conf.get('target_path','/tmp/basic.zip')}} ")
|
|
cmds.append("hdfs dfs -put -f /tmp/py37.zip {{dag_run.conf.get('target_path','/tmp/basic.zip')}} ")
|
|
|
py_task_demo = PythonTaskCompiler(item=AirflowTask(id=0, task_type="python", user_id=user_id, cmd_parameters="",
|
|
py_task_demo = PythonTaskCompiler(item=AirflowTask(id=0, task_type="python", user_id=user_id, cmd_parameters="",
|
