pipeline_with_notebooks_and_scripts.pipeline 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338
  1. {
  2. "doc_type": "pipeline",
  3. "version": "3.0",
  4. "json_schema": "http://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json",
  5. "id": "66338755-bce9-4541-b595-b0210c943b38",
  6. "primary_pipeline": "27fd7b90-42de-469a-9bcc-08ba82f48d48",
  7. "pipelines": [
  8. {
  9. "id": "27fd7b90-42de-469a-9bcc-08ba82f48d48",
  10. "nodes": [
  11. {
  12. "id": "6214dcbc-3617-4c37-9c8e-cefeeeab05d5",
  13. "type": "execution_node",
  14. "op": "execute-notebook-node",
  15. "app_data": {
  16. "component_parameters": {
  17. "filename": "notebooks/notebook_1.ipynb",
  18. "outputs": [],
  19. "env_vars": [],
  20. "kubernetes_secrets": [
  21. "USER_NAME=secret-1:myuserid",
  22. "USER_PASSWORD=secret-1:mypassword"
  23. ],
  24. "dependencies": [],
  25. "include_subdirectories": false,
  26. "mounted_volumes": [
  27. "/mnt/vol2=pvc-claim-2"
  28. ]
  29. },
  30. "label": "",
  31. "ui_data": {
  32. "label": "notebook_1.ipynb",
  33. "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20width%3D%2216%22%20viewBox%3D%220%200%2022%2022%22%3E%0A%20%20%3Cg%20class%3D%22jp-icon-warn0%20jp-icon-selectable%22%20fill%3D%22%23EF6C00%22%3E%0A%20%20%20%20%3Cpath%20d%3D%22M18.7%203.3v15.4H3.3V3.3h15.4m1.5-1.5H1.8v18.3h18.3l.1-18.3z%22%2F%3E%0A%20%20%20%20%3Cpath%20d%3D%22M16.5%2016.5l-5.4-4.3-5.6%204.3v-11h11z%22%2F%3E%0A%20%20%3C%2Fg%3E%0A%3C%2Fsvg%3E%0A",
  34. "x_pos": 195,
  35. "y_pos": 67,
  36. "description": "Run notebook file"
  37. }
  38. },
  39. "inputs": [
  40. {
  41. "id": "inPort",
  42. "app_data": {
  43. "ui_data": {
  44. "cardinality": {
  45. "min": 0,
  46. "max": -1
  47. },
  48. "label": "Input Port"
  49. }
  50. }
  51. }
  52. ],
  53. "outputs": [
  54. {
  55. "id": "outPort",
  56. "app_data": {
  57. "ui_data": {
  58. "cardinality": {
  59. "min": 0,
  60. "max": -1
  61. },
  62. "label": "Output Port"
  63. }
  64. }
  65. }
  66. ]
  67. },
  68. {
  69. "id": "ab0417ac-5d2d-4835-a474-189328c6c95d",
  70. "type": "execution_node",
  71. "op": "execute-notebook-node",
  72. "app_data": {
  73. "component_parameters": {
  74. "filename": "notebooks/notebook_2.ipynb",
  75. "outputs": [],
  76. "env_vars": [],
  77. "kubernetes_secrets": [],
  78. "dependencies": [],
  79. "include_subdirectories": false,
  80. "mounted_volumes": [
  81. "/mnt/vol2=pvc-claim-2"
  82. ]
  83. },
  84. "label": "",
  85. "ui_data": {
  86. "label": "notebook_2.ipynb",
  87. "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20width%3D%2216%22%20viewBox%3D%220%200%2022%2022%22%3E%0A%20%20%3Cg%20class%3D%22jp-icon-warn0%20jp-icon-selectable%22%20fill%3D%22%23EF6C00%22%3E%0A%20%20%20%20%3Cpath%20d%3D%22M18.7%203.3v15.4H3.3V3.3h15.4m1.5-1.5H1.8v18.3h18.3l.1-18.3z%22%2F%3E%0A%20%20%20%20%3Cpath%20d%3D%22M16.5%2016.5l-5.4-4.3-5.6%204.3v-11h11z%22%2F%3E%0A%20%20%3C%2Fg%3E%0A%3C%2Fsvg%3E%0A",
  88. "x_pos": 517.0000305175781,
  89. "y_pos": 214,
  90. "description": "Run notebook file"
  91. }
  92. },
  93. "inputs": [
  94. {
  95. "id": "inPort",
  96. "app_data": {
  97. "ui_data": {
  98. "cardinality": {
  99. "min": 0,
  100. "max": -1
  101. },
  102. "label": "Input Port"
  103. }
  104. },
  105. "links": [
  106. {
  107. "id": "2c1b4c8d-fa2a-496b-b825-26915dc54040",
  108. "node_id_ref": "3858d57c-3235-4c9b-839c-100ede4d54c3",
  109. "port_id_ref": "outPort"
  110. }
  111. ]
  112. }
  113. ],
  114. "outputs": [
  115. {
  116. "id": "outPort",
  117. "app_data": {
  118. "ui_data": {
  119. "cardinality": {
  120. "min": 0,
  121. "max": -1
  122. },
  123. "label": "Output Port"
  124. }
  125. }
  126. }
  127. ]
  128. },
  129. {
  130. "id": "69ddd986-43b4-46b3-9223-ccbf70aba01e",
  131. "type": "execution_node",
  132. "op": "execute-python-node",
  133. "app_data": {
  134. "component_parameters": {
  135. "filename": "scripts/script_1.py",
  136. "outputs": [],
  137. "env_vars": [],
  138. "kubernetes_secrets": [
  139. "API_KEY=secret-2:myapikey"
  140. ],
  141. "dependencies": [],
  142. "include_subdirectories": false,
  143. "mounted_volumes": [],
  144. "runtime_image": "tensorflow/tensorflow:2.8.0-gpu"
  145. },
  146. "label": "",
  147. "ui_data": {
  148. "label": "script_1.py",
  149. "image": "data:image/svg+xml;utf8,%3Csvg%20width%3D%2232%22%20height%3D%2232%22%20viewBox%3D%220%200%20238%20237%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20xmlns%3Axlink%3D%22http%3A%2F%2Fwww.w3.org%2F1999%2Fxlink%22%20overflow%3D%22hidden%22%3E%0A%20%20%20%20%3Cg%20transform%3D%22translate(-706%20314)%22%3E%0A%20%20%20%20%20%20%20%20%3Cpath%20class%3D%22jp-icon-selectable%22%20d%3D%22M792.441-295.763C786.496-295.763%20781.697-290.979%20781.697-285.062%20781.697-279.166%20786.496-274.382%20792.441-274.382%20798.365-274.382%20803.184-279.166%20803.184-285.062%20803.184-290.979%20798.365-295.763%20792.441-295.763ZM823.472-312.998C833.277-313.043%20843.484-312.329%20853.336-310.724%20868.899-308.185%20882-296.728%20882-281.516L882-228.072C882-212.398%20869.282-199.557%20853.336-199.557L796.03-199.557C776.58-199.557%20760.189-183.169%20760.189-164.641L760.189-139%20740.485-139C723.817-139%20714.114-150.877%20710.037-167.494%20704.538-189.82%20704.772-203.124%20710.037-224.505%20714.602-243.159%20729.189-253%20745.857-253L767.365-253%20824.693-253%20824.693-260.134%20767.365-260.134%20767.365-281.516C767.365-297.715%20771.76-306.527%20796.03-310.724%20804.268-312.151%20813.668-312.953%20823.472-312.998Z%22%20fill%3D%22%23366A96%22%20fill-rule%3D%22evenodd%22%2F%3E%0A%20%20%20%20%20%20%20%20%3Cpath%20class%3D%22jp-icon-selectable%22%20d%3D%22M857.377-117.071C851.466-117.071%20846.655-112.267%20846.655-106.348%20846.655-100.406%20851.466-95.6026%20857.377-95.6026%20863.31-95.6026%20868.099-100.406%20868.099-106.348%20868.099-112.267%20863.31-117.071%20857.377-117.071ZM889.563-253%20911.007-253C927.662-253%20935.502-240.696%20939.614-224.39%20945.334-201.743%20945.589-184.804%20939.614-167.148%20933.828-150%20927.642-138.539%20911.007-138.539L882.402-138.539%20825.211-138.539%20825.211-131.375%20882.402-131.375%20882.402-109.908C882.402-93.6435%20868.205-85.4055%20853.796-81.2973%20832.12-75.1034%20814.722-76.0513%20796.606-81.2973%20781.476-85.6801%20768-94.6332%20768-109.908L768-163.568C768-179.01%20780.947-192.199%20796.606-192.199L853.796-192.199C872.846-192.199%20889.563-208.568%20889.563-227.971Z%22%20fill%3D%22%23FFC836%22%20fill-rule%3D%22evenodd%22%2F%3E%0A%20%20%20%20%3C%2Fg%3E%0A%3C%2Fsvg%3E%0A",
  150. "x_pos": 519,
  151. "y_pos": 70,
  152. "description": "Run Python script"
  153. }
  154. },
  155. "inputs": [
  156. {
  157. "id": "inPort",
  158. "app_data": {
  159. "ui_data": {
  160. "cardinality": {
  161. "min": 0,
  162. "max": -1
  163. },
  164. "label": "Input Port"
  165. }
  166. },
  167. "links": [
  168. {
  169. "id": "47930592-4222-4274-9349-85914750cb50",
  170. "node_id_ref": "6214dcbc-3617-4c37-9c8e-cefeeeab05d5",
  171. "port_id_ref": "outPort"
  172. }
  173. ]
  174. }
  175. ],
  176. "outputs": [
  177. {
  178. "id": "outPort",
  179. "app_data": {
  180. "ui_data": {
  181. "cardinality": {
  182. "min": 0,
  183. "max": -1
  184. },
  185. "label": "Output Port"
  186. }
  187. }
  188. }
  189. ]
  190. },
  191. {
  192. "id": "3858d57c-3235-4c9b-839c-100ede4d54c3",
  193. "type": "execution_node",
  194. "op": "execute-python-node",
  195. "app_data": {
  196. "component_parameters": {
  197. "filename": "scripts/script_2.py",
  198. "outputs": [],
  199. "env_vars": [],
  200. "kubernetes_secrets": [
  201. "API_KEY=secret-2:myapikey"
  202. ],
  203. "dependencies": [],
  204. "include_subdirectories": false,
  205. "mounted_volumes": [],
  206. "runtime_image": "tensorflow/tensorflow:2.8.0"
  207. },
  208. "label": "",
  209. "ui_data": {
  210. "label": "script_2.py",
  211. "image": "data:image/svg+xml;utf8,%3Csvg%20width%3D%2232%22%20height%3D%2232%22%20viewBox%3D%220%200%20238%20237%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20xmlns%3Axlink%3D%22http%3A%2F%2Fwww.w3.org%2F1999%2Fxlink%22%20overflow%3D%22hidden%22%3E%0A%20%20%20%20%3Cg%20transform%3D%22translate(-706%20314)%22%3E%0A%20%20%20%20%20%20%20%20%3Cpath%20class%3D%22jp-icon-selectable%22%20d%3D%22M792.441-295.763C786.496-295.763%20781.697-290.979%20781.697-285.062%20781.697-279.166%20786.496-274.382%20792.441-274.382%20798.365-274.382%20803.184-279.166%20803.184-285.062%20803.184-290.979%20798.365-295.763%20792.441-295.763ZM823.472-312.998C833.277-313.043%20843.484-312.329%20853.336-310.724%20868.899-308.185%20882-296.728%20882-281.516L882-228.072C882-212.398%20869.282-199.557%20853.336-199.557L796.03-199.557C776.58-199.557%20760.189-183.169%20760.189-164.641L760.189-139%20740.485-139C723.817-139%20714.114-150.877%20710.037-167.494%20704.538-189.82%20704.772-203.124%20710.037-224.505%20714.602-243.159%20729.189-253%20745.857-253L767.365-253%20824.693-253%20824.693-260.134%20767.365-260.134%20767.365-281.516C767.365-297.715%20771.76-306.527%20796.03-310.724%20804.268-312.151%20813.668-312.953%20823.472-312.998Z%22%20fill%3D%22%23366A96%22%20fill-rule%3D%22evenodd%22%2F%3E%0A%20%20%20%20%20%20%20%20%3Cpath%20class%3D%22jp-icon-selectable%22%20d%3D%22M857.377-117.071C851.466-117.071%20846.655-112.267%20846.655-106.348%20846.655-100.406%20851.466-95.6026%20857.377-95.6026%20863.31-95.6026%20868.099-100.406%20868.099-106.348%20868.099-112.267%20863.31-117.071%20857.377-117.071ZM889.563-253%20911.007-253C927.662-253%20935.502-240.696%20939.614-224.39%20945.334-201.743%20945.589-184.804%20939.614-167.148%20933.828-150%20927.642-138.539%20911.007-138.539L882.402-138.539%20825.211-138.539%20825.211-131.375%20882.402-131.375%20882.402-109.908C882.402-93.6435%20868.205-85.4055%20853.796-81.2973%20832.12-75.1034%20814.722-76.0513%20796.606-81.2973%20781.476-85.6801%20768-94.6332%20768-109.908L768-163.568C768-179.01%20780.947-192.199%20796.606-192.199L853.796-192.199C872.846-192.199%20889.563-208.568%20889.563-227.971Z%22%20fill%3D%22%23FFC836%22%20fill-rule%3D%22evenodd%22%2F%3E%0A%20%20%20%20%3C%2Fg%3E%0A%3C%2Fsvg%3E%0A",
  212. "x_pos": 201,
  213. "y_pos": 214,
  214. "description": "Run Python script"
  215. }
  216. },
  217. "inputs": [
  218. {
  219. "id": "inPort",
  220. "app_data": {
  221. "ui_data": {
  222. "cardinality": {
  223. "min": 0,
  224. "max": -1
  225. },
  226. "label": "Input Port"
  227. }
  228. }
  229. }
  230. ],
  231. "outputs": [
  232. {
  233. "id": "outPort",
  234. "app_data": {
  235. "ui_data": {
  236. "cardinality": {
  237. "min": 0,
  238. "max": -1
  239. },
  240. "label": "Output Port"
  241. }
  242. }
  243. }
  244. ]
  245. },
  246. {
  247. "id": "648cc074-65f7-4702-ba99-3ac282cada9d",
  248. "type": "execution_node",
  249. "op": "execute-python-node",
  250. "app_data": {
  251. "component_parameters": {
  252. "filename": "scripts/script_3.py",
  253. "outputs": [],
  254. "env_vars": [],
  255. "kubernetes_secrets": [
  256. "TOKEN=secret-3:mytoken"
  257. ],
  258. "dependencies": [],
  259. "include_subdirectories": false,
  260. "mounted_volumes": [
  261. "/mnt/vol3=pvc-claim-3"
  262. ],
  263. "runtime_image": "amancevice/pandas:1.4.1"
  264. },
  265. "label": "",
  266. "ui_data": {
  267. "label": "script_3.py",
  268. "image": "data:image/svg+xml;utf8,%3Csvg%20width%3D%2232%22%20height%3D%2232%22%20viewBox%3D%220%200%20238%20237%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20xmlns%3Axlink%3D%22http%3A%2F%2Fwww.w3.org%2F1999%2Fxlink%22%20overflow%3D%22hidden%22%3E%0A%20%20%20%20%3Cg%20transform%3D%22translate(-706%20314)%22%3E%0A%20%20%20%20%20%20%20%20%3Cpath%20class%3D%22jp-icon-selectable%22%20d%3D%22M792.441-295.763C786.496-295.763%20781.697-290.979%20781.697-285.062%20781.697-279.166%20786.496-274.382%20792.441-274.382%20798.365-274.382%20803.184-279.166%20803.184-285.062%20803.184-290.979%20798.365-295.763%20792.441-295.763ZM823.472-312.998C833.277-313.043%20843.484-312.329%20853.336-310.724%20868.899-308.185%20882-296.728%20882-281.516L882-228.072C882-212.398%20869.282-199.557%20853.336-199.557L796.03-199.557C776.58-199.557%20760.189-183.169%20760.189-164.641L760.189-139%20740.485-139C723.817-139%20714.114-150.877%20710.037-167.494%20704.538-189.82%20704.772-203.124%20710.037-224.505%20714.602-243.159%20729.189-253%20745.857-253L767.365-253%20824.693-253%20824.693-260.134%20767.365-260.134%20767.365-281.516C767.365-297.715%20771.76-306.527%20796.03-310.724%20804.268-312.151%20813.668-312.953%20823.472-312.998Z%22%20fill%3D%22%23366A96%22%20fill-rule%3D%22evenodd%22%2F%3E%0A%20%20%20%20%20%20%20%20%3Cpath%20class%3D%22jp-icon-selectable%22%20d%3D%22M857.377-117.071C851.466-117.071%20846.655-112.267%20846.655-106.348%20846.655-100.406%20851.466-95.6026%20857.377-95.6026%20863.31-95.6026%20868.099-100.406%20868.099-106.348%20868.099-112.267%20863.31-117.071%20857.377-117.071ZM889.563-253%20911.007-253C927.662-253%20935.502-240.696%20939.614-224.39%20945.334-201.743%20945.589-184.804%20939.614-167.148%20933.828-150%20927.642-138.539%20911.007-138.539L882.402-138.539%20825.211-138.539%20825.211-131.375%20882.402-131.375%20882.402-109.908C882.402-93.6435%20868.205-85.4055%20853.796-81.2973%20832.12-75.1034%20814.722-76.0513%20796.606-81.2973%20781.476-85.6801%20768-94.6332%20768-109.908L768-163.568C768-179.01%20780.947-192.199%20796.606-192.199L853.796-192.199C872.846-192.199%20889.563-208.568%20889.563-227.971Z%22%20fill%3D%22%23FFC836%22%20fill-rule%3D%22evenodd%22%2F%3E%0A%20%20%20%20%3C%2Fg%3E%0A%3C%2Fsvg%3E%0A",
  269. "x_pos": 865,
  270. "y_pos": 151,
  271. "description": "Run Python script"
  272. }
  273. },
  274. "inputs": [
  275. {
  276. "id": "inPort",
  277. "app_data": {
  278. "ui_data": {
  279. "cardinality": {
  280. "min": 0,
  281. "max": -1
  282. },
  283. "label": "Input Port"
  284. }
  285. },
  286. "links": [
  287. {
  288. "id": "aa4c6e6e-ee40-4f76-8235-d7001b1a4cf4",
  289. "node_id_ref": "69ddd986-43b4-46b3-9223-ccbf70aba01e",
  290. "port_id_ref": "outPort"
  291. },
  292. {
  293. "id": "204bba59-6b1a-4ae6-a709-18a0e543c095",
  294. "node_id_ref": "ab0417ac-5d2d-4835-a474-189328c6c95d",
  295. "port_id_ref": "outPort"
  296. }
  297. ]
  298. }
  299. ],
  300. "outputs": [
  301. {
  302. "id": "outPort",
  303. "app_data": {
  304. "ui_data": {
  305. "cardinality": {
  306. "min": 0,
  307. "max": -1
  308. },
  309. "label": "Output Port"
  310. }
  311. }
  312. }
  313. ]
  314. }
  315. ],
  316. "app_data": {
  317. "ui_data": {
  318. "runtime": null,
  319. "comments": []
  320. },
  321. "version": 7,
  322. "properties": {
  323. "pipeline_defaults": {
  324. "runtime_image": "tensorflow/tensorflow:2.8.0-gpu",
  325. "mounted_volumes": [
  326. "/mnt/vol1=pvc-claim-1"
  327. ]
  328. },
  329. "name": "pipeline_with_notebooks_and_scripts",
  330. "runtime": "Generic",
  331. "description": "Generic pipeline with scripts and notebooks"
  332. }
  333. },
  334. "runtime_ref": ""
  335. }
  336. ],
  337. "schemas": []
  338. }