yili-docker-images/charts/test/airflow/templates/workers/worker-kedaautoscaler.yaml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

################################
## Airflow Worker KEDA Scaler
#################################
{{- if and .Values.workers.keda.enabled (has .Values.executor (list "CeleryExecutor" "CeleryKubernetesExecutor")) }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: {{ .Release.Name }}-worker
  labels:
    tier: airflow
    component: worker-horizontalpodautoscaler
    release: {{ .Release.Name }}
    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
    heritage: {{ .Release.Service }}
    deploymentName: {{ .Release.Name }}-worker
{{- with .Values.labels }}
{{ toYaml . | indent 4 }}
{{- end }}
spec:
  scaleTargetRef:
    kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }}
    name: {{ .Release.Name }}-worker
  pollingInterval:  {{ .Values.workers.keda.pollingInterval }}   # Optional. Default: 30 seconds
  cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }}    # Optional. Default: 300 seconds
  minReplicaCount: {{ .Values.workers.keda.minReplicaCount }}   # Optional. Default: 0
  maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }}   # Optional. Default: 100
  triggers:
    - type: postgresql
      metadata:
        targetQueryValue: "1"
        connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
        query: >-
          SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }})
          FROM task_instance
          WHERE (state='running' OR state='queued')
          {{- if eq .Values.executor "CeleryKubernetesExecutor" }}
          AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}'
          {{- end }}
{{- end }}