diff --git a/helm/chart/Chart.yaml b/helm/chart/Chart.yaml index 8ce2c38..faaa5c8 100644 --- a/helm/chart/Chart.yaml +++ b/helm/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: flink-kube-operator description: Helm chart for flink kube operator type: application -version: 1.0.1 +version: 1.1.0 appVersion: "0.1.1" dependencies: - name: minio diff --git a/helm/chart/templates/flink/config.yaml b/helm/chart/templates/flink/config.yaml index 3e51828..2255237 100644 --- a/helm/chart/templates/flink/config.yaml +++ b/helm/chart/templates/flink/config.yaml @@ -18,9 +18,13 @@ high-availability.type: kubernetes kubernetes.namespace: {{ .Release.Namespace }} kubernetes.cluster-id: {{ .Values.clusterId | default (print .Release.Name "-cluster") }} - execution.checkpointing.interval: {{ .Values.flink.checkpoint.interval }} - execution.checkpointing.mode: {{ .Values.flink.checkpoint.mode }} + execution.checkpointing.interval: {{ .Values.flink.state.checkpoint.interval }} + execution.checkpointing.mode: {{ .Values.flink.state.checkpoint.mode }} + {{- if eq .Values.flink.state.checkpoint.storageType "filesystem" }} + state.checkpoints.dir: /opt/flink/checkpoints/ + {{- else if eq .Values.flink.state.checkpoint.storageType "s3" }} state.checkpoints.dir: s3://flink/checkpoints/ + {{- end }} state.backend.rocksdb.localdir: /opt/flink/rocksdb high-availability.storageDir: /opt/flink/ha state.savepoints.dir: s3://flink/savepoints/ diff --git a/helm/chart/templates/flink/ha.pvc.yaml b/helm/chart/templates/flink/ha.pvc.yaml index ee3b672..9f137bb 100644 --- a/helm/chart/templates/flink/ha.pvc.yaml +++ b/helm/chart/templates/flink/ha.pvc.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ .Release.Name }}-{{ .Values.flink.state.ha.pvcName }} + name: {{ .Release.Name }}-flink-ha-pvc spec: accessModes: - ReadWriteOnce diff --git a/helm/chart/templates/flink/job-manager-deploy.yaml b/helm/chart/templates/flink/job-manager-deploy.yaml index 017bef4..210d4e3 100644 --- a/helm/chart/templates/flink/job-manager-deploy.yaml +++ b/helm/chart/templates/flink/job-manager-deploy.yaml @@ -64,12 +64,19 @@ spec: volumeMounts: - name: flink-ha mountPath: {{ .Values.flink.state.ha.dir }} - + {{- if eq .Values.flink.state.checkpoint.storageType "filesystem" }} + - name: flink-checkpoint + mountPath: /opt/flink/checkpoints + {{- end }} volumes: - name: flink-ha persistentVolumeClaim: - claimName: {{ .Release.Name }}-{{ .Values.flink.state.ha.pvcName }} - + claimName: {{ .Release.Name }}-flink-ha-pvc + {{- if eq .Values.flink.state.checkpoint.storageType "filesystem" }} + - name: flink-checkpoint + persistentVolumeClaim: + claimName: {{ .Release.Name }}-flink-checkpoint-pvc + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/chart/templates/flink/savepoint.pvc.yaml b/helm/chart/templates/flink/savepoint.pvc.yaml new file mode 100644 index 0000000..ef03774 --- /dev/null +++ b/helm/chart/templates/flink/savepoint.pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-flink-checkpoint-pvc +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{ .Values.flink.state.checkpoint.size }} diff --git a/helm/chart/templates/flink/task-manager-statefulset.yaml b/helm/chart/templates/flink/task-manager-statefulset.yaml index cd260a9..c3e46bc 100644 --- a/helm/chart/templates/flink/task-manager-statefulset.yaml +++ b/helm/chart/templates/flink/task-manager-statefulset.yaml @@ -46,8 +46,18 @@ spec: volumeMounts: - name: rocksdb-storage mountPath: /opt/flink/rocksdb + {{- if eq .Values.flink.state.checkpoint.storageType "filesystem" }} + - name: flink-checkpoint + mountPath: /opt/flink/checkpoints + {{- end }} resources: {{- toYaml .Values.flink.taskManager.resources | nindent 10 }} + {{- if eq .Values.flink.state.checkpoint.storageType "filesystem" }} + volumes: + - name: flink-checkpoint + persistentVolumeClaim: + claimName: {{ .Release.Name }}-flink-checkpoint-pvc + {{- end }} volumeClaimTemplates: - metadata: name: rocksdb-storage diff --git a/helm/chart/values.yaml b/helm/chart/values.yaml index 0f5c7cd..937792b 100644 --- a/helm/chart/values.yaml +++ b/helm/chart/values.yaml @@ -121,10 +121,6 @@ flink: parallelism: default: 1 # Default parallelism for Flink jobs - - checkpoint: - interval: 5min - mode: EXACTLY_ONCE state: backend: rocksdb # Use RocksDB for state backend @@ -133,6 +129,11 @@ flink: dir: "/opt/flink/ha" # Directory to store ha data pvcName: flink-ha-pvc # PVC for ha size: 10Gi # PVC size for ha + checkpoint: + storageType: s3 # s3 / filesystem + interval: 5min + mode: EXACTLY_ONCE + size: 8Gi jobManager: processMemory: 4096m # Size of job manager process memory diff --git a/helm/flink-kube-operator-1.1.0.tgz b/helm/flink-kube-operator-1.1.0.tgz new file mode 100644 index 0000000..e90ba40 Binary files /dev/null and b/helm/flink-kube-operator-1.1.0.tgz differ diff --git a/helm/index.yaml b/helm/index.yaml index 874eca3..fc4dcc9 100644 --- a/helm/index.yaml +++ b/helm/index.yaml @@ -1,6 +1,20 @@ apiVersion: v1 entries: flink-kube-operator: + - apiVersion: v2 + appVersion: 0.1.1 + created: "2025-04-13T10:00:10.79626445+03:30" + dependencies: + - name: minio + repository: https://charts.bitnami.com/bitnami + version: 16.0.2 + description: Helm chart for flink kube operator + digest: 36e78032b5b381dc5b99e647e6c42fe8c270e1141c271d2326d2117cc8700fc8 + name: flink-kube-operator + type: application + urls: + - flink-kube-operator-1.1.0.tgz + version: 1.1.0 - apiVersion: v2 appVersion: 0.1.1 created: "2025-04-12T23:13:39.394371646+03:30" @@ -179,4 +193,4 @@ entries: urls: - flink-kube-operator-0.1.0.tgz version: 0.1.0 -generated: "2025-04-12T23:13:39.378113533+03:30" +generated: "2025-04-13T10:00:10.782977799+03:30"