flink v1

flink/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

flink/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: flink
+version: 0.1.4

flink/templates/NOTES.txt

@@ -0,0 +1,11 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+  {{- end }}
+{{- end }}
+{{- end }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "flink.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:80

flink/templates/_helpers.tpl

@@ -0,0 +1,45 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "flink.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "flink.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "flink.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "flink.labels" -}}
+app.kubernetes.io/name: {{ include "flink.name" . }}
+helm.sh/chart: {{ include "flink.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}

flink/templates/configmap.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "flink.fullname" . }}-config
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+data:
+  flink-conf.yaml: |+
+    jobmanager.rpc.address: flink-jobmanager
+    taskmanager.numberOfTaskSlots: {{ .Values.taskmanager.numberOfTaskSlots }}
+    blob.server.port: 6124
+    jobmanager.rpc.port: 6123
+    taskmanager.rpc.port: 6122
+    jobmanager.heap.size: {{ .Values.jobmanager.heapSize }}
+    taskmanager.heap.size: {{ .Values.taskmanager.heapSize }}
+    {{- if .Values.flink.monitoring.enabled }}
+    metrics.reporters: prom
+    metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+    metrics.reporter.prom.port: 9999
+      {{- if .Values.flink.monitoring.system.enabled }}
+    metrics.system-resource: true
+    metrics.system-resource-probing-interval: {{ .Values.flink.monitoring.system.probingInterval }}
+      {{- end }}
+      {{- if .Values.flink.monitoring.latency.enabled }}
+    metrics.latency.interval: {{ .Values.flink.monitoring.latency.probingInterval }}
+      {{- end }}
+    {{- end }}
+  log4j.properties: |+
+    log4j.rootLogger=INFO, file
+    log4j.logger.akka=INFO
+    log4j.logger.org.apache.kafka=INFO
+    log4j.logger.org.apache.hadoop=INFO
+    log4j.logger.org.apache.zookeeper=INFO
+    log4j.appender.file=org.apache.log4j.FileAppender
+    log4j.appender.file.file=${log.file}
+    log4j.appender.file.layout=org.apache.log4j.PatternLayout
+    log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+    log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file

flink/templates/ingress.yaml

@@ -0,0 +1,36 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "flink.fullname" . -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+{{- if .Values.ingress.tls }}
+  tls:
+  {{- range .Values.ingress.tls }}
+    - hosts:
+      {{- range .hosts }}
+        - {{ . | quote }}
+      {{- end }}
+      secretName: {{ .secretName }}
+  {{- end }}
+{{- end }}
+  rules:
+  {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+        {{- range .paths }}
+          - path: {{ . }}
+            backend:
+              serviceName: flink-jobmanager
+              servicePort: ui
+        {{- end }}
+  {{- end }}
+{{- end }}

flink/templates/prometheus-service-monitors.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: 'flink-jobmanager'
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+    app: prometheus
+    prometheusApp: prometheus-local
+spec:
+  endpoints:
+    - interval: 5s
+      path: /
+      port: metrics
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  targetLabels:
+    - app.kubernetes.io/name
+    - component
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: flink
+      component: jobmanager
+
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: 'flink-taskmanager'
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+    app: prometheus
+    prometheusApp: prometheus-local
+spec:
+  endpoints:
+    - interval: 5s
+      path: /
+      port: metrics
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  targetLabels:
+    - app.kubernetes.io/name
+    - component
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: flink
+      component: taskmanager

flink/templates/service-jobmanager-rest.yaml

@@ -0,0 +1,24 @@
+{{ if .Values.jobmanager.rest.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: flink-jobmanager-ui
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+    component: jobmanager
+{{- if .Values.jobmanager.service.annotations }}
+  annotations:
+{{ toYaml .Values.jobmanager.service.annotations | indent 4 }}
+{{- end }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.jobmanager.ports.ui }}
+      targetPort: {{ .Values.jobmanager.ports.ui }}
+      protocol: TCP
+      name: ui
+  selector:
+    app.kubernetes.io/name: {{ include "flink.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: jobmanager
+{{- end }}

flink/templates/service-jobmanager.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flink-jobmanager
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+    component: jobmanager
+spec:
+  type: {{ .Values.jobmanager.service.type }}
+  ports:
+    {{- range $name, $port := .Values.jobmanager.ports }}
+    - port: {{ $port }}
+      targetPort: {{ $name }}
+      protocol: TCP
+      name: {{ $name }}
+    {{- end }}
+  selector:
+    app.kubernetes.io/name: {{ include "flink.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: jobmanager

flink/templates/service-taskmanager.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flink-taskmanager
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+    component: taskmanager
+spec:
+  type: {{ .Values.taskmanager.service.type }}
+  ports:
+    {{- range $name, $port := .Values.taskmanager.ports }}
+    - port: {{ $port }}
+      targetPort: {{ $name }}
+      protocol: TCP
+      name: {{ $name }}
+    {{- end }}
+  selector:
+    app.kubernetes.io/name: {{ include "flink.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: taskmanager

flink/templates/statefulset-jobmanager.yaml

@@ -0,0 +1,95 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "flink.fullname" . }}-jobmanager
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+    component: jobmanager
+  annotations:
+    "cluster-autoscaler.kubernetes.io/safe-to-evict": "false"
+spec:
+  replicas: {{ .Values.jobmanager.replicaCount }}
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "flink.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      component: jobmanager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "flink.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        component: jobmanager
+    spec:
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      containers:
+        - name: jobmanager
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          workingDir: {{ .Values.flink.workDir }}
+          command:
+          {{- range .Values.jobmanager.command }}
+            - {{ . | quote }}
+          {{- end }}
+          env:
+          # Redis access
+            - name: REDIS_HOST
+              value: {{ .Release.Namespace }}-redis-master
+            - name: REDIS_PORT
+              value: "6379"
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Namespace }}-redis
+                  key: redis-password
+          #
+          {{- range $key, $value := .Values.jobmanager.extraEnvs }}
+            - name: {{ $key }}
+              value: {{ $value }}
+          {{- end }}
+          {{- range $key, $value := .Values.extraEnvs }}
+            - name: {{ $key }}
+              value: {{ $value }}
+          {{- end }}
+          ports:
+          {{- range $name, $port := .Values.jobmanager.ports }}
+            - containerPort: {{ $port }}
+              name: {{ $name }}
+          {{- end }}
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.jobmanager.ports.rpc }}
+            initialDelaySeconds: 30
+            periodSeconds: 60
+          volumeMounts:
+            - name: flink-config-volume
+              mountPath: {{ .Values.flink.workDir }}/conf
+          resources:
+            {{- toYaml .Values.jobmanager.resources | nindent 12 }}
+
+      volumes:
+        - name: flink-config-volume
+          configMap:
+            name: {{ include "flink.fullname" . }}-config
+            items:
+              - key: flink-conf.yaml
+                path: flink-conf.yaml
+              - key: log4j.properties
+                path: log4j.properties
+
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}

flink/templates/statefulset-taskmanager.yaml

@@ -0,0 +1,95 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "flink.fullname" . }}-taskmanager
+  labels:
+{{ include "flink.labels" . | indent 4 }}
+    component: taskmanager
+  annotations:
+    "cluster-autoscaler.kubernetes.io/safe-to-evict": "false"
+spec:
+  replicas: {{ .Values.taskmanager.replicaCount }}
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "flink.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      component: taskmanager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "flink.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        component: taskmanager
+    spec:
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      containers:
+        - name: taskmanager
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          workingDir: {{ .Values.flink.workDir }}
+          command:
+          {{- range .Values.taskmanager.command }}
+            - {{ . | quote }}
+          {{- end }}
+          env:
+          # Redis access
+            - name: REDIS_HOST
+              value: {{ .Release.Namespace }}-redis-master
+            - name: REDIS_PORT
+              value: "6379"
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Namespace }}-redis
+                  key: redis-password
+          #
+          {{- range $key, $value := .Values.taskmanager.extraEnvs }}
+            - name: {{ $key }}
+              value: {{ $value }}
+          {{- end }}
+          {{- range $key, $value := .Values.extraEnvs }}
+            - name: {{ $key }}
+              value: {{ $value }}
+          {{- end }}
+          ports:
+          {{- range $name, $port := .Values.taskmanager.ports }}
+            - containerPort: {{ $port }}
+              name: {{ $name }}
+          {{- end }}
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.taskmanager.ports.rpc }}
+            initialDelaySeconds: 30
+            periodSeconds: 60
+          volumeMounts:
+            - name: flink-config-volume
+              mountPath: {{ .Values.flink.workDir }}/conf
+          resources:
+            {{- toYaml .Values.taskmanager.resources | nindent 12 }}
+
+      volumes:
+        - name: flink-config-volume
+          configMap:
+            name: {{ include "flink.fullname" . }}-config
+            items:
+              - key: flink-conf.yaml
+                path: flink-conf.yaml
+              - key: log4j.properties
+                path: log4j.properties
+
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}

flink/values.yaml

@@ -0,0 +1,111 @@
+# Default values for flink.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+nameOverride: ""
+fullnameOverride: ""
+
+image:
+  repository: flink
+  tag: 1.9.1-scala_2.12
+  pullPolicy: IfNotPresent
+imagePullSecrets: []
+
+flink:
+  monitoring:
+    enabled: true
+    latency:
+      enabled: false
+      probingInterval: 1000
+    system:
+      enabled: true
+      probingInterval: 5000
+  workDir: /opt/flink
+
+extraEnvs: {}
+
+jobmanager:
+  extraEnvs: {}
+  ports:
+    rpc: 6123
+    blob: 6124
+    ui: 8081
+    metrics: 9999
+  replicaCount: 1
+  # g for Gigabytes, m for Megabytes
+  heapSize: 6g
+  resources:
+    limits:
+      cpu: 3800m
+      # Gi for Gigabytes, Mi for Megabytes
+      memory: 8000Mi
+  command: ["/bin/bash", "-c", "wget \
+  https://repo1.maven.org/maven2/org/apache/flink/flink-metrics-prometheus_2.12/1.9.1/flink-metrics-prometheus_2.12-1.9.1.jar \
+  -O /opt/flink/lib/flink-metrics-prometheus_2.12-1.9.1.jar && \
+  wget https://repo1.maven.org/maven2/com/github/oshi/oshi-core/3.4.0/oshi-core-3.4.0.jar \
+  -O /opt/flink/lib/oshi-core-3.4.0.jar && \
+  wget https://repo1.maven.org/maven2/net/java/dev/jna/jna/5.4.0/jna-5.4.0.jar \
+  -O /opt/flink/lib/jna-5.4.0.jar && \
+  wget https://repo1.maven.org/maven2/net/java/dev/jna/jna-platform/5.4.0/jna-platform-5.4.0.jar \
+  -O /opt/flink/lib/jna-platform-5.4.0.jar && \
+  $FLINK_HOME/bin/jobmanager.sh start;\
+      while :;
+      do
+        if [[ -f $(find log -name '*jobmanager*.log' -print -quit) ]];
+          then tail -f -n +1 log/*jobmanager*.log;
+        fi;
+      done"]
+  service:
+    type: ClusterIP
+    annotations: {}
+  rest:
+    enabled: true
+
+taskmanager:
+  extraEnvs: {}
+  ports:
+    rpc: 6122
+    metrics: 9999
+  replicaCount: 4
+  numberOfTaskSlots: 9
+  # g for Gigabytes, m for Megabytes
+  heapSize: 6g
+  resources:
+    limits:
+      cpu: 3800m
+      # Gi for Gigabytes, Mi for Megabytes
+      memory: 8000Mi
+  command: ["/bin/bash", "-c", "wget \
+  https://repo1.maven.org/maven2/org/apache/flink/flink-metrics-prometheus_2.12/1.9.1/flink-metrics-prometheus_2.12-1.9.1.jar \
+  -O /opt/flink/lib/flink-metrics-prometheus_2.12-1.9.1.jar && \
+  wget https://repo1.maven.org/maven2/com/github/oshi/oshi-core/3.4.0/oshi-core-3.4.0.jar \
+  -O /opt/flink/lib/oshi-core-3.4.0.jar && \
+  wget https://repo1.maven.org/maven2/net/java/dev/jna/jna/5.4.0/jna-5.4.0.jar \
+  -O /opt/flink/lib/jna-5.4.0.jar && \
+  wget https://repo1.maven.org/maven2/net/java/dev/jna/jna-platform/5.4.0/jna-platform-5.4.0.jar \
+  -O /opt/flink/lib/jna-platform-5.4.0.jar && \
+  $FLINK_HOME/bin/taskmanager.sh start; \
+      while :;
+      do
+        if [[ -f $(find log -name '*taskmanager*.log' -print -quit) ]];
+          then tail -f -n +1 log/*taskmanager*.log;
+        fi;
+      done"]
+  service:
+    type: ClusterIP
+
+ingress:
+  enabled: false
+  annotations: {}
+  path: /
+  hosts: []
+  tls: []
+
+nodeSelector: {}
+
+tolerations:
+  - key: instance
+    operator: Exists
+    effect: NoSchedule
+
+affinity: {}