Add optional OTel service to the Airflow Helm Chart

chart/templates/_helpers.yaml

@@ -141,6 +141,24 @@ If release name contains chart name it will be used as a full name.
         name: {{ template "opensearch_secret" . }}
         key: connection
   {{- end }}
+  {{- if or .Values.otelCollector.tracesEnabled .Values.otelCollector.metricsEnabled }}
+  - name: OTEL_SERVICE_NAME
+    value: "airflow"
+  - name: OTEL_EXPORTER_OTLP_PROTOCOL
+    value: "http/protobuf"
+  {{- end }}
+  {{- if .Values.otelCollector.tracesEnabled }}
+  - name: OTEL_TRACES_EXPORTER
+    value: "otlp"
+  - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+    value: "http://{{ include "airflow.fullname" . }}-otel-collector:{{ .Values.ports.otelCollectorOtlpHttp }}/v1/traces"
+  {{- end }}
+  {{- if .Values.otelCollector.metricsEnabled }}
+  - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+    value: "http://{{ include "airflow.fullname" . }}-otel-collector:{{ .Values.ports.otelCollectorOtlpHttp }}/v1/metrics"
+  - name: OTEL_METRIC_EXPORT_INTERVAL
+    value: {{ .Values.otelCollector.metricExportIntervalMs | quote }}
+  {{- end }}
 {{- end }}
 
 {{/* User defined Airflow environment variables */}}
@@ -404,6 +422,10 @@ If release name contains chart name it will be used as a full name.
   {{- printf "%s:%s" .Values.images.gitSync.repository .Values.images.gitSync.tag }}
 {{- end }}
 
+{{- define "otel_collector_image" -}}
+  {{- printf "%s:%s" .Values.images.otelCollector.repository .Values.images.otelCollector.tag }}
+{{- end }}
+
 {{- define "fernet_key_secret" -}}
   {{- default (printf "%s-fernet-key" (include "airflow.fullname" .)) .Values.fernetKeySecretName }}
 {{- end }}
@@ -680,6 +702,11 @@ server_tls_key_file = /etc/pgbouncer/server.key
   {{- include "_serviceAccountName" (merge (dict "key" "statsd") .) -}}
 {{- end }}
 
+{{/* Create the name of the OTel Collector service account to use */}}
+{{- define "otelCollector.serviceAccountName" -}}
+  {{- include "_serviceAccountName" (merge (dict "key" "otelCollector" "nameSuffix" "otel-collector") .) -}}
+{{- end }}
+
 {{/* Create the name of the create user job service account to use */}}
 {{- define "createUserJob.serviceAccountName" -}}
   {{- include "_serviceAccountName" (merge (dict "key" "createUserJob" "nameSuffix" "create-user-job") .) -}}

chart/templates/configmaps/otel-collector-configmap.yaml

@@ -0,0 +1,73 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+###########################
+## OTel Collector ConfigMap
+###########################
+{{- if or .Values.otelCollector.tracesEnabled .Values.otelCollector.metricsEnabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "airflow.fullname" . }}-otel-collector
+  labels:
+    tier: airflow
+    component: config
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+data:
+  config.yml: |
+{{- if .Values.otelCollector.config }}
+{{ tpl .Values.otelCollector.config . | indent 4 }}
+{{- else }}
+    extensions:
+      health_check:
+        endpoint: 0.0.0.0:13133
+
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:{{ .Values.ports.otelCollectorOtlpGrpc }}
+          http:
+            endpoint: 0.0.0.0:{{ .Values.ports.otelCollectorOtlpHttp }}
+
+    processors:
+      batch: {}
+
+    exporters:
+      logging:
+        verbosity: basic
+
+    service:
+      extensions: [health_check]
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [logging]
+        metrics:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [logging]
+{{- end }}
+{{- end }}

chart/templates/otel-collector/otel-collector-deployment.yaml

@@ -0,0 +1,115 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+############################
+## OTel Collector Deployment
+############################
+{{- if or .Values.otelCollector.tracesEnabled .Values.otelCollector.metricsEnabled }}
+{{- $nodeSelector := or .Values.otelCollector.nodeSelector .Values.nodeSelector }}
+{{- $affinity := or .Values.otelCollector.affinity .Values.affinity }}
+{{- $tolerations := or .Values.otelCollector.tolerations .Values.tolerations }}
+{{- $topologySpreadConstraints := or .Values.otelCollector.topologySpreadConstraints .Values.topologySpreadConstraints }}
+{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list .Values.otelCollector.revisionHistoryLimit .Values.revisionHistoryLimit) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "airflow.fullname" . }}-otel-collector
+  labels:
+    tier: airflow
+    component: otel-collector
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or .Values.labels .Values.otelCollector.labels }}
+      {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+  {{- with .Values.otelCollector.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: 1
+  {{- if ne $revisionHistoryLimit "" }}
+  revisionHistoryLimit: {{ $revisionHistoryLimit }}
+  {{- end }}
+  selector:
+    matchLabels:
+      tier: airflow
+      component: otel-collector
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        tier: airflow
+        component: otel-collector
+        release: {{ .Release.Name }}
+        {{- if or .Values.labels .Values.otelCollector.labels }}
+          {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml | nindent 8 }}
+        {{- end }}
+      annotations:
+        checksum/otel-collector-config: {{ include (print $.Template.BasePath "/configmaps/otel-collector-configmap.yaml") . | sha256sum }}
+        {{- with .Values.otelCollector.podAnnotations }}
+          {{- tpl (toYaml .) $ | nindent 8 }}
+        {{- end }}
+    spec:
+      nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
+      affinity: {{- toYaml $affinity | nindent 8 }}
+      tolerations: {{- toYaml $tolerations | nindent 8 }}
+      topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
+      terminationGracePeriodSeconds: {{ .Values.otelCollector.terminationGracePeriodSeconds }}
+      serviceAccountName: {{ include "otelCollector.serviceAccountName" . }}
+      {{- if .Values.otelCollector.priorityClassName }}
+      priorityClassName: {{ .Values.otelCollector.priorityClassName }}
+      {{- end }}
+      restartPolicy: Always
+      securityContext: {{- toYaml .Values.otelCollector.securityContexts.pod | nindent 8 }}
+      containers:
+        - name: otel-collector
+          image: {{ template "otel_collector_image" . }}
+          imagePullPolicy: {{ .Values.images.otelCollector.pullPolicy }}
+          securityContext: {{- toYaml .Values.otelCollector.securityContexts.container | nindent 12 }}
+          args: {{- tpl (toYaml .Values.otelCollector.args) . | nindent 12 }}
+          ports:
+            - name: otlp-grpc
+              containerPort: {{ .Values.ports.otelCollectorOtlpGrpc }}
+              protocol: TCP
+            - name: otlp-http
+              containerPort: {{ .Values.ports.otelCollectorOtlpHttp }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 13133
+            initialDelaySeconds: {{ .Values.otelCollector.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.otelCollector.livenessProbe.periodSeconds }}
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 13133
+            initialDelaySeconds: {{ .Values.otelCollector.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.otelCollector.readinessProbe.periodSeconds }}
+          resources: {{- toYaml .Values.otelCollector.resources | nindent 12 }}
+          volumeMounts:
+            - name: config
+              mountPath: /etc/otel-collector
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "airflow.fullname" . }}-otel-collector
+{{- end }}

chart/templates/otel-collector/otel-collector-networkpolicy.yaml

@@ -0,0 +1,59 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+#######################################
+## Airflow OTel Collector NetworkPolicy
+#######################################
+{{- if and .Values.networkPolicies.enabled (or .Values.otelCollector.tracesEnabled .Values.otelCollector.metricsEnabled) }}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: {{ include "airflow.fullname" . }}-otel-collector-policy
+  labels:
+    tier: airflow
+    component: otel-collector-policy
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or .Values.labels .Values.otelCollector.labels }}
+      {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+spec:
+  podSelector:
+    matchLabels:
+      tier: airflow
+      component: otel-collector
+      release: {{ .Release.Name }}
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          tier: airflow
+          release: {{ .Release.Name }}
+    {{- if .Values.otelCollector.extraNetworkPolicies }}
+      {{- toYaml .Values.otelCollector.extraNetworkPolicies | nindent 4 }}
+    {{- end }}
+    ports:
+    - protocol: TCP
+      port: {{ .Values.ports.otelCollectorOtlpHttp }}
+    - protocol: TCP
+      port: {{ .Values.ports.otelCollectorOtlpGrpc }}
+{{- end }}

chart/templates/otel-collector/otel-collector-service.yaml

@@ -0,0 +1,55 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+#########################
+## OTel Collector Service
+#########################
+{{- if or .Values.otelCollector.tracesEnabled .Values.otelCollector.metricsEnabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "airflow.fullname" . }}-otel-collector
+  labels:
+    tier: airflow
+    component: otel-collector
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or .Values.labels .Values.otelCollector.labels }}
+      {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+  {{- with .Values.otelCollector.service.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: ClusterIP
+  selector:
+    tier: airflow
+    component: otel-collector
+    release: {{ .Release.Name }}
+  ports:
+    - name: otlp-grpc
+      protocol: TCP
+      port: {{ .Values.ports.otelCollectorOtlpGrpc }}
+      targetPort: {{ .Values.ports.otelCollectorOtlpGrpc }}
+    - name: otlp-http
+      protocol: TCP
+      port: {{ .Values.ports.otelCollectorOtlpHttp }}
+      targetPort: {{ .Values.ports.otelCollectorOtlpHttp }}
+{{- end }}

chart/templates/otel-collector/otel-collector-serviceaccount.yaml

@@ -0,0 +1,41 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+########################################
+## Airflow OTel Collector ServiceAccount
+########################################
+{{- if and (or .Values.otelCollector.tracesEnabled .Values.otelCollector.metricsEnabled) .Values.otelCollector.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.otelCollector.serviceAccount.automountServiceAccountToken }}
+metadata:
+  name: {{ include "otelCollector.serviceAccountName" . }}
+  labels:
+    tier: airflow
+    component: otel-collector
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or .Values.labels .Values.otelCollector.labels }}
+      {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+  {{- with .Values.otelCollector.serviceAccount.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}