livy.conf 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190
  1. #
  2. # Licensed to the Apache Software Foundation (ASF) under one or more
  3. # contributor license agreements. See the NOTICE file distributed with
  4. # this work for additional information regarding copyright ownership.
  5. # The ASF licenses this file to You under the Apache License, Version 2.0
  6. # (the "License"); you may not use this file except in compliance with
  7. # the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. #
  17. # Use this keystore for the SSL certificate and key.
  18. # livy.keystore =
  19. # Specify the keystore password.
  20. # livy.keystore.password =
  21. #
  22. # Specify the key password.
  23. # livy.key-password =
  24. # Hadoop Credential Provider Path to get "livy.keystore.password" and "livy.key-password".
  25. # Credential Provider can be created using command as follow:
  26. # hadoop credential create "livy.keystore.password" -value "secret" -provider jceks://hdfs/path/to/livy.jceks
  27. # livy.hadoop.security.credential.provider.path =
  28. # What host address to start the server on. By default, Livy will bind to all network interfaces.
  29. # livy.server.host = 0.0.0.0
  30. # What port to start the server on.
  31. livy.server.port = 8998
  32. # What base path ui should work on. By default UI is mounted on "/".
  33. # E.g.: livy.ui.basePath = /my_livy - result in mounting UI on /my_livy/
  34. # livy.ui.basePath = ""
  35. # What spark master Livy sessions should use.
  36. livy.spark.master = yarn
  37. # What spark deploy mode Livy sessions should use.
  38. livy.spark.deploy-mode = cluster
  39. # Configure Livy server http request and response header size.
  40. # livy.server.request-header.size = 131072
  41. # livy.server.response-header.size = 131072
  42. # Enabled to check whether timeout Livy sessions should be stopped.
  43. # livy.server.session.timeout-check = true
  44. #
  45. # Whether or not to skip timeout check for a busy session
  46. # livy.server.session.timeout-check.skip-busy = false
  47. # Time in milliseconds on how long Livy will wait before timing out an inactive session.
  48. # Note that the inactive session could be busy running jobs.
  49. # livy.server.session.timeout = 1h
  50. #
  51. # How long a finished session state should be kept in LivyServer for query.
  52. # livy.server.session.state-retain.sec = 600s
  53. # If livy should impersonate the requesting users when creating a new session.
  54. # livy.impersonation.enabled = false
  55. # Logs size livy can cache for each session/batch. 0 means don't cache the logs.
  56. # livy.cache-log.size = 200
  57. # Comma-separated list of Livy RSC jars. By default Livy will upload jars from its installation
  58. # directory every time a session is started. By caching these files in HDFS, for example, startup
  59. # time of sessions on YARN can be reduced.
  60. livy.rsc.jars = hdfs:/user/AIuser/livy/rsc-jars/*
  61. # Comma-separated list of Livy REPL jars. By default Livy will upload jars from its installation
  62. # directory every time a session is started. By caching these files in HDFS, for example, startup
  63. # time of sessions on YARN can be reduced. Please list all the repl dependencies including
  64. # Scala version-specific livy-repl jars, Livy will automatically pick the right dependencies
  65. # during session creation.
  66. #livy.repl.jars = hdfs:/user/aidevuser/repl_2.12-jars/*
  67. # Location of PySpark archives. By default Livy will upload the file from SPARK_HOME, but
  68. # by caching the file in HDFS, startup time of PySpark sessions on YARN can be reduced.
  69. # livy.pyspark.archives =
  70. # Location of the SparkR package. By default Livy will upload the file from SPARK_HOME, but
  71. # by caching the file in HDFS, startup time of R sessions on YARN can be reduced.
  72. # livy.sparkr.package =
  73. # List of local directories from where files are allowed to be added to user sessions. By
  74. # default it's empty, meaning users can only reference remote URIs when starting their
  75. # sessions.
  76. # livy.file.local-dir-whitelist =
  77. # Whether to enable csrf protection, by default it is false. If it is enabled, client should add
  78. # http-header "X-Requested-By" in request if the http method is POST/DELETE/PUT/PATCH.
  79. # livy.server.csrf-protection.enabled =
  80. # Whether to enable HiveContext in livy interpreter, if it is true hive-site.xml will be detected
  81. # on user request and then livy server classpath automatically.
  82. livy.repl.enable-hive-context = true
  83. # Recovery mode of Livy. Possible values:
  84. # off: Default. Turn off recovery. Every time Livy shuts down, it stops and forgets all sessions.
  85. # recovery: Livy persists session info to the state store. When Livy restarts, it recovers
  86. # previous sessions from the state store.
  87. # Must set livy.server.recovery.state-store and livy.server.recovery.state-store.url to
  88. # configure the state store.
  89. # livy.server.recovery.mode = off
  90. # Zookeeper address used for HA and state store. e.g. host1:port1, host2:port2
  91. # livy.server.zookeeper.url =
  92. # Where Livy should store state to for recovery. Possible values:
  93. # <empty>: Default. State store disabled.
  94. # filesystem: Store state on a file system.
  95. # zookeeper: Store state in a Zookeeper instance.
  96. # livy.server.recovery.state-store =
  97. # For filesystem state store, the path of the state store directory. Please don't use a filesystem
  98. # that doesn't support atomic rename (e.g. S3). e.g. file:///tmp/livy or hdfs:///.
  99. # For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2
  100. # If livy.server.recovery.state-store is zookeeper, this config is for back-compatibility,
  101. # so if both this config and livy.server.zookeeper.url exist,
  102. # livy uses livy.server.zookeeper.url first.
  103. # livy.server.recovery.state-store.url =
  104. # The policy of curator connecting to zookeeper.
  105. # For example, m, n means retry m times and the interval of retry is n milliseconds.
  106. # Please use the new config: livy.server.zk.retry-policy.
  107. # Keep this config for back-compatibility.
  108. # If both this config and livy.server.zk.retry-policy exist,
  109. # livy uses livy.server.zk.retry-policy first.
  110. # livy.server.recovery.zk-state-store.retry-policy = 5,100
  111. # The policy of curator connecting to zookeeper.
  112. # For example, m, n means retry m times and the interval of retry is n milliseconds
  113. # livy.server.zk.retry-policy =
  114. # The dir in zk to store the data about session.
  115. # livy.server.recovery.zk-state-store.key-prefix = livy
  116. # If Livy can't find the yarn app within this time, consider it lost.
  117. livy.server.yarn.app-lookup-timeout = 5m
  118. # When the cluster is busy, we may fail to launch yarn app in app-lookup-timeout, then it would
  119. # cause session leakage, so we need to check session leakage.
  120. # How long to check livy session leakage
  121. # livy.server.yarn.app-leakage.check-timeout = 600s
  122. # how often to check livy session leakage
  123. # livy.server.yarn.app-leakage.check-interval = 60s
  124. # How often Livy polls YARN to refresh YARN app state.
  125. # livy.server.yarn.poll-interval = 5s
  126. #
  127. # Days to keep Livy server request logs.
  128. # livy.server.request-log-retain.days = 5
  129. # If the Livy Web UI should be included in the Livy Server. Enabled by default.
  130. # livy.ui.enabled = true
  131. # Whether to enable Livy server access control, if it is true then all the income requests will
  132. # be checked if the requested user has permission.
  133. # livy.server.access-control.enabled = false
  134. # Allowed users to access Livy, by default any user is allowed to access Livy. If user want to
  135. # limit who could access Livy, user should list all the permitted users with comma separated.
  136. # livy.server.access-control.allowed-users = *
  137. # A list of users with comma separated has the permission to change other user's submitted
  138. # session, like submitting statements, deleting session.
  139. # livy.server.access-control.modify-users =
  140. # A list of users with comma separated has the permission to view other user's infomation, like
  141. # submitted session state, statement results.
  142. # livy.server.access-control.view-users =
  143. #
  144. # Authentication support for Livy server
  145. # Livy has a built-in SPnego authentication support for HTTP requests with below configurations.
  146. # livy.server.auth.type = kerberos
  147. # livy.server.auth.kerberos.principal = <spnego principal>
  148. # livy.server.auth.kerberos.keytab = <spnego keytab>
  149. # livy.server.auth.kerberos.name-rules = DEFAULT
  150. # livy.server.launch.kerberos.principal = ailab
  151. # livy.server.launch.kerberos.keytab = /opt/cluster/conf/user.keytab
  152. #
  153. # If user wants to use custom authentication filter, configurations are:
  154. # livy.server.auth.type = <custom>
  155. # livy.server.auth.<custom>.class = <class of custom auth filter>
  156. # livy.server.auth.<custom>.param.<foo1> = <bar1>
  157. # livy.server.auth.<custom>.param.<foo2> = <bar2>