Flink. Upgrade to 1.10

flink/Chart.yaml

@@ -1,8 +1,9 @@
 apiVersion: v1
-appVersion: "1.9.1"
+appVersion: "1.10.0"
 description: Chart for Apache Flink
 name: flink
-version: 0.1.4
+version: 0.1.5
 maintainers:
   - name: charts-maintainers
     email: charts-maintainers@riskfocus.com
+home: https://github.com/riskfocus/helm-charts-public

flink/README.md

@@ -2,6 +2,9 @@
 
 This is an implementation of https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html
 
+This chart will install session cluster https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html#flink-session-cluster-on-kubernetes.
+If you are interested in supporting session/job clusters: https://github.com/GoogleCloudPlatform/flink-on-k8s-operator
+
 ## Pre Requisites:
 
 * Kubernetes 1.3 with alpha APIs enabled and support for storage classes
@@ -54,7 +57,7 @@ following configurable parameters(other parameters can be found in values.yaml):
 | Parameter                                | Description                                                                                                                                                              | Default                |
 |------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
 | `image.repository`                       | Flink Container image name                                                                                                                                               | `flink`                |
-| `image.tag`                              | Flink Container image tag                                                                                                                                                | `1.9.1-scala_2.12`     |
+| `image.tag`                              | Flink Container image tag                                                                                                                                                | `1.10.0-scala_2.12`     |
 | `image.PullPolicy`                       | Flink Containers pull policy                                                                                                                                             | `IfNotPresent`         |
 | `flink.monitoring.enabled`               | Enable flink monitoring                                                                                                                                                  | `true`                 |
 | `jobmanager.highAvailability.enabled`    | Enabled jobmanager HA mode key                                                                                                                                           | `false`                |

flink/templates/_config.tpl

@@ -0,0 +1,52 @@
+{{/*
+ServiceAccount for Jobmanager
+*/}}
+{{- define "jobmanager.serviceAccount" -}}
+{{ default "jobmanager" .Values.jobmanager.serviceAccount.name }}
+{{- end -}}
+
+{{/*
+ServiceAccount for Taskmanager
+*/}}
+{{- define "taskmanager.serviceAccount" -}}
+{{ default "taskmanager" .Values.taskmanager.serviceAccount.name }}
+{{- end -}}
+
+{{/*
+Generate command for Jobmanager
+*/}}
+{{- define "jobmanager.command" -}}
+{{ $cmd := .Values.jobmanager.command }}
+{{- if .Values.jobmanager.highAvailability.enabled }}
+{{ $cmd = (tpl .Values.jobmanager.highAvailability.command .) }}
+{{- end }}
+{{- if .Values.jobmanager.additionalCommand -}}
+{{ printf "%s && %s" .Values.jobmanager.additionalCommand $cmd }}
+{{- else }}
+{{ $cmd }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Generate command for Taskmanager
+*/}}
+{{- define "taskmanager.command" -}}
+{{ $cmd := .Values.taskmanager.command }}
+{{- if .Values.taskmanager.additionalCommand -}}
+{{ printf "%s && %s" .Values.taskmanager.additionalCommand $cmd }}
+{{- else }}
+{{ $cmd }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Check if specific namespace is passed if false
+then .Release.Namespace will be used
+*/}}
+{{- define "serviceMonitor.namespace" -}}
+{{- if .Values.prometheus.serviceMonitor.namespace -}}
+{{ .Values.prometheus.serviceMonitor.namespace }}
+{{- else -}}
+{{ .Release.Namespace }}
+{{- end -}}
+{{- end -}}

flink/templates/_flink_config.tpl

@@ -0,0 +1,50 @@
+{{/*
+Generate Flink Configuration.
+We do it here to support HA mode where we cannot
+provide jobmanager.rpc.address to Taskmanagers
+*/}}
+{{- define "flink.configuration" -}}
+    taskmanager.numberOfTaskSlots: {{ .Values.taskmanager.numberOfTaskSlots }}
+    blob.server.port: {{ .Values.jobmanager.ports.blob }}
+    taskmanager.rpc.port: {{ .Values.taskmanager.ports.rpc }}
+    jobmanager.heap.size: {{ .Values.jobmanager.heapSize }}
+    taskmanager.heap.size: {{ .Values.taskmanager.heapSize }}
+    {{- .Values.flink.params | nindent 4 }}
+    {{- if .Values.flink.monitoring.enabled }}
+    metrics.reporters: prom
+    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+    metrics.reporter.prom.port: {{ .Values.flink.monitoring.port }}
+      {{- if .Values.flink.monitoring.system.enabled }}
+    metrics.system-resource: true
+    metrics.system-resource-probing-interval: {{ .Values.flink.monitoring.system.probingInterval }}
+      {{- end }}
+      {{- if .Values.flink.monitoring.latency.enabled }}
+    metrics.latency.interval: {{ .Values.flink.monitoring.latency.probingInterval }}
+      {{- end }}
+      {{- if .Values.flink.monitoring.rocksdb.enabled }}
+    state.backend.rocksdb.metrics.cur-size-active-mem-table: true
+    state.backend.rocksdb.metrics.cur-size-all-mem-tables: true
+    state.backend.rocksdb.metrics.estimate-live-data-size: true
+    state.backend.rocksdb.metrics.size-all-mem-tables: true
+    state.backend.rocksdb.metrics.estimate-num-keys: true
+      {{- end }}
+    {{- end }}
+    {{- if .Values.flink.state.backend }}
+    state.backend: {{ .Values.flink.state.backend }}
+    {{- .Values.flink.state.params | nindent 4 }}
+      {{- if eq .Values.flink.state.backend "rocksdb" }}
+    {{- .Values.flink.state.rocksdb | nindent 4 }}
+      {{- end }}
+    {{- end }}
+    {{- if .Values.jobmanager.highAvailability.enabled }}
+    high-availability: zookeeper
+    high-availability.zookeeper.quorum: {{ .Values.jobmanager.highAvailability.zookeeperConnect }}
+    high-availability.zookeeper.path.root: /flink
+    high-availability.cluster-id: /flink
+    high-availability.storageDir: {{ .Values.jobmanager.highAvailability.storageDir }}
+    high-availability.jobmanager.port: {{ .Values.jobmanager.highAvailability.syncPort }}
+    {{- else }}
+    jobmanager.rpc.address: {{ include "flink.fullname" . }}-jobmanager
+    jobmanager.rpc.port: {{ .Values.jobmanager.ports.rpc }}
+    {{- end }}
+{{- end -}}

flink/templates/_helpers.tpl

@@ -43,107 +43,3 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
-
-{{/*
-Check if specific namespace is passed if false
-then .Release.Namespace will be used
-*/}}
-{{- define "serviceMonitor.namespace" -}}
-{{- if .Values.prometheus.serviceMonitor.namespace -}}
-{{ .Values.prometheus.serviceMonitor.namespace }}
-{{- else -}}
-{{ .Release.Namespace }}
-{{- end -}}
-{{- end -}}
-
-{{/*
-ServiceAccount for Jobmanager
-*/}}
-{{- define "jobmanager.serviceAccount" -}}
-{{ default "jobmanager" .Values.jobmanager.serviceAccount.name }}
-{{- end -}}
-
-{{/*
-ServiceAccount for Taskmanager
-*/}}
-{{- define "taskmanager.serviceAccount" -}}
-{{ default "taskmanager" .Values.taskmanager.serviceAccount.name }}
-{{- end -}}
-
-{{/*
-Generate command for Jobmanager
-*/}}
-{{- define "jobmanager.command" -}}
-{{ $cmd := .Values.jobmanager.command }}
-{{- if .Values.jobmanager.highAvailability.enabled }}
-{{ $cmd = (tpl .Values.jobmanager.highAvailability.command .) }}
-{{- end }}
-{{- if .Values.jobmanager.additionalCommand -}}
-{{ printf "%s && %s" .Values.jobmanager.additionalCommand $cmd }}
-{{- else }}
-{{ $cmd }}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Generate command for Taskmanager
-*/}}
-{{- define "taskmanager.command" -}}
-{{ $cmd := .Values.taskmanager.command }}
-{{- if .Values.taskmanager.additionalCommand -}}
-{{ printf "%s && %s" .Values.taskmanager.additionalCommand $cmd }}
-{{- else }}
-{{ $cmd }}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Generate Flink Configuration.
-We do it here to support HA mode where we cannot
-provide jobmanager.rpc.address to Taskmanagers
-*/}}
-{{- define "flink.configuration" -}}
-    taskmanager.numberOfTaskSlots: {{ .Values.taskmanager.numberOfTaskSlots }}
-    blob.server.port: {{ .Values.jobmanager.ports.blob }}
-    taskmanager.rpc.port: {{ .Values.taskmanager.ports.rpc }}
-    jobmanager.heap.size: {{ .Values.jobmanager.heapSize }}
-    taskmanager.heap.size: {{ .Values.taskmanager.heapSize }}
-    {{- .Values.flink.params | nindent 4 }}
-    {{- if .Values.flink.monitoring.enabled }}
-    metrics.reporters: prom
-    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
-    metrics.reporter.prom.port: {{ .Values.flink.monitoring.port }}
-      {{- if .Values.flink.monitoring.system.enabled }}
-    metrics.system-resource: true
-    metrics.system-resource-probing-interval: {{ .Values.flink.monitoring.system.probingInterval }}
-      {{- end }}
-      {{- if .Values.flink.monitoring.latency.enabled }}
-    metrics.latency.interval: {{ .Values.flink.monitoring.latency.probingInterval }}
-      {{- end }}
-      {{- if .Values.flink.monitoring.rocksdb.enabled }}
-    state.backend.rocksdb.metrics.cur-size-active-mem-table: true
-    state.backend.rocksdb.metrics.cur-size-all-mem-tables: true
-    state.backend.rocksdb.metrics.estimate-live-data-size: true
-    state.backend.rocksdb.metrics.size-all-mem-tables: true
-    state.backend.rocksdb.metrics.estimate-num-keys: true
-      {{- end }}
-    {{- end }}
-    {{- if .Values.flink.state.backend }}
-    state.backend: {{ .Values.flink.state.backend }}
-    {{- .Values.flink.state.params | nindent 4 }}
-      {{- if eq .Values.flink.state.backend "rocksdb" }}
-    {{- .Values.flink.state.rocksdb | nindent 4 }}
-      {{- end }}
-    {{- end }}
-    {{- if .Values.jobmanager.highAvailability.enabled }}
-    high-availability: zookeeper
-    high-availability.zookeeper.quorum: {{ .Values.jobmanager.highAvailability.zookeeperConnect }}
-    high-availability.zookeeper.path.root: /flink
-    high-availability.cluster-id: /flink
-    high-availability.storageDir: {{ .Values.jobmanager.highAvailability.storageDir }}
-    high-availability.jobmanager.port: {{ .Values.jobmanager.highAvailability.syncPort }}
-    {{- else }}
-    jobmanager.rpc.address: {{ include "flink.fullname" . }}-jobmanager
-    jobmanager.rpc.port: {{ .Values.jobmanager.ports.rpc }}
-    {{- end }}
-{{- end -}}

flink/templates/jobmanager-statefulset.yaml → flink/templates/jobmanager.yaml

@@ -41,6 +41,15 @@ spec:
         {{- toYaml . | nindent 8 }}
     {{- end }}
       serviceAccount: {{ include "jobmanager.serviceAccount" . }}
+      initContainers:
+      {{- range $ct, $params := .Values.jobmanager.initContainers }}
+        - name: {{ $ct }}
+          image: {{ $params.image }}
+          command:
+          {{- range $params.command }}
+            - {{ . }}
+          {{- end }}
+      {{- end }}
       containers:
         - name: jobmanager
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"

flink/templates/taskmanager-statefulset.yaml → flink/templates/taskmanager.yaml

@@ -38,6 +38,15 @@ spec:
         {{- toYaml . | nindent 8 }}
     {{- end }}
       serviceAccount: {{ include "taskmanager.serviceAccount" . }}
+      initContainers:
+      {{- range $ct, $params := .Values.taskmanager.initContainers }}
+        - name: {{ $ct }}
+          image: {{ $params.image }}
+          command:
+          {{- range $params.command }}
+            - {{ . }}
+          {{- end }}
+      {{- end }}
       containers:
         - name: taskmanager
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"

flink/values.yaml

@@ -7,7 +7,7 @@ fullnameOverride: ""
 
 image:
   repository: flink
-  tag: 1.9.1-scala_2.12
+  tag: 1.10.0-scala_2.12
   pullPolicy: IfNotPresent
 imagePullSecrets: []
 
@@ -67,6 +67,15 @@ jobmanager:
   # Statefulset option will create Jobmanager as a StatefulSet
   # A necessary option for HA mode and Persistent
   statefulset: true
+  # Init containers
+  initContainers: {}
+  # Example
+  #  test:
+  #    image: busybox:1.28
+  #    command:
+  #      - /bin/sh
+  #      - -c
+  #      - "echo test"
   # highAvailability configuration based on zookeeper
   highAvailability:
     # enabled also will enable zookeeper Dependency
@@ -112,8 +121,7 @@ jobmanager:
   #      cpu: 3800m
   #      memory: 8000Mi
   additionalCommand: >-
-    cp /opt/flink/opt/flink-metrics-prometheus-1.9.1.jar
-    /opt/flink/opt/flink-s3-fs-hadoop-1.9.1.jar
+    cp /opt/flink/opt/*
     /opt/flink/lib/ &&
     wget https://repo1.maven.org/maven2/com/github/oshi/oshi-core/3.4.0/oshi-core-3.4.0.jar
     -O /opt/flink/lib/oshi-core-3.4.0.jar &&
@@ -187,8 +195,7 @@ taskmanager:
   #      memory: 8000Mi
   additionalCommand: >-
     cp
-    /opt/flink/opt/flink-metrics-prometheus-1.9.1.jar
-    /opt/flink/opt/flink-s3-fs-hadoop-1.9.1.jar
+    /opt/flink/opt/*.jar
     /opt/flink/lib/ &&
     wget https://repo1.maven.org/maven2/com/github/oshi/oshi-core/3.4.0/oshi-core-3.4.0.jar
     -O /opt/flink/lib/oshi-core-3.4.0.jar &&