From 37936c8c5817ec7d9e87262b0e1f84836468157d Mon Sep 17 00:00:00 2001 From: Mohammadreza Khani Date: Fri, 20 Dec 2024 17:12:51 +0330 Subject: [PATCH] feat: separate ha and checkpoint pvc --- Dockerfile | 4 +-- helm/chart/Chart.yaml | 2 +- helm/chart/templates/flink/data.pvc.yaml | 10 +++++++ helm/chart/templates/flink/deploy.yaml | 28 ++++++++++-------- .../flink/{pvc.yaml => savepoint.pvc.yaml} | 2 +- helm/chart/values.yaml | 17 ++++++----- helm/flink-kube-operator-0.1.2.tgz | Bin 0 -> 5481 bytes helm/index.yaml | 16 ++++++++-- 8 files changed, 53 insertions(+), 26 deletions(-) create mode 100644 helm/chart/templates/flink/data.pvc.yaml rename helm/chart/templates/flink/{pvc.yaml => savepoint.pvc.yaml} (65%) create mode 100644 helm/flink-kube-operator-0.1.2.tgz diff --git a/Dockerfile b/Dockerfile index ef568c7..beb8f96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,8 @@ COPY . . # Build ENV GOCACHE=/root/.cache/go-build -RUN --mount=type=cache,target="/go" --mount=type=cache,target="/root/.cache/go-build" CGO_ENABLED=1 GOOS=linux go build -ldflags '-s -w' -o /flink-kube-operator ./cmd/operator -RUN upx -q -5 /flink-kube-operator +RUN --mount=type=cache,target="/go" --mount=type=cache,target="/root/.cache/go-build" CGO_ENABLED=1 GOOS=linux go build -ldflags '-s -w' -o /flink-kube-operator ./cmd/operator \ + && upx -q -9 /flink-kube-operator FROM public.ecr.aws/docker/library/busybox:1.37.0 AS final diff --git a/helm/chart/Chart.yaml b/helm/chart/Chart.yaml index 0758bd4..fba0251 100644 --- a/helm/chart/Chart.yaml +++ b/helm/chart/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: flink-kube-operator description: Helm chart for flink kube operator type: application -version: 0.1.1 +version: 0.1.2 appVersion: "0.1.0" diff --git a/helm/chart/templates/flink/data.pvc.yaml b/helm/chart/templates/flink/data.pvc.yaml new file mode 100644 index 0000000..e09bdea --- /dev/null +++ b/helm/chart/templates/flink/data.pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.flink.state.data.pvcName }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.flink.state.data.size }} # Use size defined in values.yaml diff --git a/helm/chart/templates/flink/deploy.yaml b/helm/chart/templates/flink/deploy.yaml index fc5123a..ae86c57 100644 --- a/helm/chart/templates/flink/deploy.yaml +++ b/helm/chart/templates/flink/deploy.yaml @@ -39,29 +39,33 @@ spec: taskmanager.numberOfTaskSlots: {{ .Values.flink.taskManager.numberOfTaskSlots }} parallelism.default: {{ .Values.flink.parallelism.default }} state.backend: {{ .Values.flink.state.backend }} - state.savepoints.dir: {{ .Values.flink.state.savepoints.dir }} rest.port: 8081 rootLogger.level = DEBUG rootLogger.appenderRef.console.ref = ConsoleAppender - web.upload.dir: /opt/flink/data/web-upload - state.checkpoints.dir: file:///tmp/flink-checkpoints high-availability.type: kubernetes - high-availability.storageDir: file:///opt/flink/ha - kubernetes.cluster-id: cluster-one kubernetes.namespace: {{ .Release.Namespace }} + kubernetes.cluster-id: cluster-one + web.upload.dir: file://{{ .Values.flink.state.data.dir }}/web-upload + state.checkpoints.dir: file://{{ .Values.flink.state.data.dir }}/checkpoints + state.backend.rocksdb.localdir: file://{{ .Values.flink.state.data.dir }}/rocksdb + high-availability.storageDir: file://{{ .Values.flink.state.data.dir }} + state.savepoints.dir: file://{{ .Values.flink.state.savepoints.dir }} volumeMounts: - name: flink-data - mountPath: /opt/flink/data - subPath: data + mountPath: {{ .Values.flink.state.data.dir }} + - name: flink-ha + mountPath: {{ .Values.flink.state.ha.dir }} - name: flink-savepoints - mountPath: /opt/flink/savepoints - - name: flink-savepoints - mountPath: /opt/flink/ha - subPath: ha + mountPath: {{ .Values.flink.state.savepoints.dir }} + volumes: - name: flink-data - emptyDir: {} # Temporary storage for internal data + persistentVolumeClaim: + claimName: {{ .Values.flink.state.data.pvcName }} # PVC for savepoints persistence - name: flink-savepoints persistentVolumeClaim: claimName: {{ .Values.flink.state.savepoints.pvcName }} # PVC for savepoints persistence + - name: flink-ha + persistentVolumeClaim: + claimName: {{ .Values.flink.state.ha.pvcName }} # PVC for savepoints persistence \ No newline at end of file diff --git a/helm/chart/templates/flink/pvc.yaml b/helm/chart/templates/flink/savepoint.pvc.yaml similarity index 65% rename from helm/chart/templates/flink/pvc.yaml rename to helm/chart/templates/flink/savepoint.pvc.yaml index dff425d..6906af1 100644 --- a/helm/chart/templates/flink/pvc.yaml +++ b/helm/chart/templates/flink/savepoint.pvc.yaml @@ -7,4 +7,4 @@ spec: - ReadWriteOnce resources: requests: - storage: {{ .Values.flink.persistence.size }} # Use size defined in values.yaml + storage: {{ .Values.flink.state.savepoints.size }} # Use size defined in values.yaml diff --git a/helm/chart/values.yaml b/helm/chart/values.yaml index fade56f..c1fd7f6 100644 --- a/helm/chart/values.yaml +++ b/helm/chart/values.yaml @@ -125,14 +125,17 @@ flink: state: backend: rocksdb # Use RocksDB for state backend savepoints: - dir: "file:///opt/flink/savepoints" # Directory to store savepoints + dir: "/opt/flink/savepoints" # Directory to store savepoints pvcName: flink-savepoints-pvc # PVC for savepoints persistence + size: 10Gi # PVC size for savepoints storage + data: + dir: "/opt/flink/data" # Directory to store checkpoints/web-upload/rocksdb + pvcName: flink-data-pvc # PVC for checkpoints/web-upload/rocksdb + size: 10Gi # PVC size for checkpoints/web-upload/rocksdb + ha: + dir: "/opt/flink/ha" # Directory to store ha data + pvcName: flink-ha-pvc # PVC for ha + size: 10Gi # PVC size for ha taskManager: numberOfTaskSlots: 100 # Number of task slots for TaskManager - - persistence: - enabled: true - size: 10Gi # PVC size for savepoints storage - accessModes: - - ReadWriteOnce diff --git a/helm/flink-kube-operator-0.1.2.tgz b/helm/flink-kube-operator-0.1.2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..83e36f374382d22c66f831fc5473a983bc461bc8 GIT binary patch literal 5481 zcmV-v6_)BBiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH<$Z`-)C`&qwYuG@p%?XBcT(zFmzTx64MPtIu*BWb%hSQG^< z(KZ`e)RL4_Z<_ntFZhr|J^V{woEgz$N|G!8DjQ*+rI;hhc=u?E zxC|C(Mt^va((Cnl2mAZ_f3Mf8|L+a@`#<#epYQbt&j-Eddq4F0&w9`He}LY@bicY} zLL&Y{@2mT&5AJVLP)gooE(l9Pm=8RZrp0Y9=m$M7#v?F`YrAmIsU(2fcVe zEgr}hkC&hs4xwdPd2*5B5Kx*@5+Mc8o4Z{Nyzfnm|Bt1O|L2HiSUj8nSk3?a{=u^b z|L+ZU{J)j*2#)a>WmJNh^^&iviz4tI!P_YjKm;K8&*7Wb{+RI@k`l)-CKRg*$2g*h zW0)gO(1>CIlEDZ=iiE&1kVGuNa9oB4Nfj z29lUA2}>YCiGn<4SrWq(a~#4{%2b5iuELuqSV1SO8?#7sBbG!sm7;si_!VU+){i7- zlwvNrWp`cAULUD$L+BUy2`q)iO*KR$!5qj8P4I)ER=^cq4OnzN{Xg_T(Sr#h8DE4z zBOXvTArYFTfyB|YOUZ~MzUa0IU4T0NX+|lejFM=fC);!uc%a`5^@|V==`~si-TZux zIVUkT&CQ@Cn!xpx2@J7aKEzT-D~D2zD7X?$LeSGKJ!cuI&Q%kEBd$oIA1EA1G69s> zp2nQb071&h97*g@;%f;nk+{-SACpPOv7fToFXo`D@bEPTzU|uQctV8a3p+2vjM7Vu zx(MO-56-|O17>OF&_pT5K&!``pdoZR-Z-PQ@z8r@=Pc3#8)byX0$3(3!T6j+7*O=E z%pq`;aUvI8+ktO85M9qcIW!MKy)G_O9FZ}>0_V4qw04*BqnxJfVvV#^kSr zVzCes&zz;pv*b{mu|!pLcf zUX)WjqAa>;uoY7%_){jN#zZlgx(SV%_3>vkODPU0ixAcC0a7HVwz6rU z9)!>>o|r0c_3AiP&1X0<*jptRsl`K$qStKgTeHOsxze>Ssj$3XqWLKAx(!Q@W5F^W z;Zh6zh9MJ~R)`!&Y&OG5taXiKAT&;k!t^RyYb7KZP>klv^ia0CGS?%Ph={PvpcPG- z;>_Iq$^_OUxg_PAUr$C96)QYMcm8T%G zXsWt@LlUB)c!7_*$T%)8i5~o>~n9&+jXUr&N*Cd%}=#(U?3W`5vLcgD>4iX$; zArN0Y)#K-Q#^%=W$~axXh$Bs#vBVsHEJ^)iV1dp^IY&{NX(>Ci_cJ`xR(F5!;tjDw z{3F9+(-u{;P+)19my$V76!#jGQB3q*%Gn5iwftFa+Lq6AB>VyaU+sovmih*fu9sIC zs(UqL)$J=R4L7-bWMr3m?G@ne)T(CmwrICZG7%xYKX~{Jp##0^IXA81Zbp&|yJ=LL zM1UFk>^$i8^tOjjusj;RdnXAce`+*`m`6&qOt2R7YQ>mpZ}h5JUehzPrjXIwyu*~d zZx=8a@j*_}oV|Ng=A(0EJ|A{XZMBX~vG2vK+QuU6-; z+OZeol(L25MbC8O0Tf%(J8epJeZ39r2ZLVF^A&B;$3f5^_@B@m^`HAQl90$KW>0BC z#1W;K5-~HqSz#AKUlA>LS~%4-@&@rwY$OyAA=O4hHy@$s3MX+0oJCh69w{L21cnQB zd;E)8YN~o(Q@0Xmj#EYwS!j=#@DMs(mP#|SZds+H5hGlaf1%l3s#RLn$kow2I&-|F z<$FJ!M+(UB{gFYacmt{0P78^X2wNzE{E5|4dqwgl>PfwwLIF)on;1#7bR6pCvJq%I z+yLxefl8lqF= zX#pUv-k=0cFgHjg*=&UQ`S>joSC^DYBPtY9XZnraaw2gBLcP?G-j*=FiGRG!|66$H zx@YHaPc8%bS$p)?Fbm zD8mdTu?W4J8z0CRg7+p;kQ?$=O~UQ1uE9|sv~Z0K-tB&w_&$?b7Q*FaN+j$=EeXkdoWeK`Xv=u^!Xr49sed;4<@ z&C_8GN!5k87pg?!nZ{D7%uEC*j@7ha{mkCnb~FOflA%$2q-Pg;W`;Qn(Z%t9WeWRx#I|_e2nxPkDdvJm>|zps#i< zJ?l<4VL?X&bv%F^^Nlc8SomYcuMv+O^i?0><0?^H4e-77@AodN?Z1yxOx4aKkm*;X z0<5$Dl(AQ}|DNw3?CtEoZIqjv?i26ibE-XuwaAzD`vu;Uu9d(s9+PCHjPuKydV3z2 zR5C5jauMwYJK+mRo+S|+?CDD~yUfO8@)VCOB3i z#70_w!v?mlFI5j}JOV~ILQ3KYHp6aEN{?LoM9#R z3B#WQ?{o~@h$I7~&`@RI7l&0VQ-ftX-F&IFUse{NGqZ|w zY$BVQRXw|QshClWqo)`#U%++5o<3;3dQW|S`M{C5XBA3JwvEjF96#9%8n z!`9R_=uR=61!CIOBM8@;ns%UpL1lGaVO^T9ZFhmQvMd=6i-Z&Pu4!cznTE=KXf*i( ze`HMJ8d;i9X4(-G`=dozF+^;tLMvHwf9#zITOYCCijGMZ*xdST0g+puVZ*RmwfgId zqCG`o)1kU)9%sv>TeVtXGc9WB=BG_}H$N|fo46x)5ZHTFiwQ7&M ziDs%zZtKKybu2AQ@^ITmFa~#mfs=WCNQUPBI{)S4o5QoiS0@)AFNQ}S508&8 zPA)Gib-`%K5IQ8dKJ^S~^;l=szMNTW3n-tsXDjQr zBn32A> zmx8m=#F42*D55D*^H{{H<~GI2Cvgb*HA`@#$u%AYCcP!li7aD6u?cc7wR>a2;r7-^ zj9D^~P8Q8wVLMG%uady#OgFZSH;5Zs`EV=lO-yfi)#*dTPUb_mU!m2xNtn*;?^5Blrc9Oddk3*|oD z{nEPF#HrSj{RhoxrRNuAv&wj(B?@t`S$g66588%ckAA4tT8(N0SvO?S=%ju)O9`iO zY|Gb>>p?>$Imqe>6Rqf#q^zA=(T?}3=u4Rk<;H{7;IafCu@ss9pd}s?LdW68HQ}8y zu=Ay!d|1i*>a2?TG~kt|9aZsbjRqNl>Ccp*QfgtE72hya+ohaI7BL#a+oRzoVH*=Q zIUJsTe0TA>J!Il^ZoU={oKJ>; zvm+bxubm&XALpa$vX=X$_5FJLF8Y5%`)`_}ZA=6^lQQRDSi?-DQYINuXCZD94hVFzpRb-z(m4vcUF zwM%fPj&0-~DzU6lS;scQl9R|KSo`#@KAs|%P`=&CURX8YRrNI_|?G%KE%4qOFe zKso(6@6a4`G{Mfxr9<8f^O_uAONkXHaNLs^CV606^wft?(by- ztkeJf{r!6UXK%36|63_*PIB;T%D+4oF1UJOzO2dpMW?3eUq|J)`3CIpP~DNzaH1@+ z-1i&KBG)C=+E;Nho@r_yAC>3jDXHG#gIY2(GWVWHmqPaNjAeE6L-J}{6Whu;h{F`$ z;N*N3<+iH1LCRC`RHO2~RS#Egct-KsF;tMtkNQ(R>0>?`DC&>Q3v_D-|1CRcq#Soz zq_C7nhg4geInIv@tk)=y5iDo5+P)=4F3+s1?dqM=HLK;WT=lp$Ly&RV1f8Ezq`^O4 z@5*Q~vUUc>->Gb<|2d=IEdIOKAJp}K@1Wn?>HlpMSGn^Miqytpsy6dl88(#s1*2_Y z;jeIfZ;(%`4@fKSucw%sPf*pO1R|WH#UI9SDD;vKC?)z3=t7D3CIntR>`)&&K0Jkq zs+xZktB)f57aPIR#j#Cesrg(jEdLW52}c5(Bgj*laYT#HZ1wplQ6?mt6@WCDIp!k= z;snd$ni5go7$Oz7SsJ^yF{aqLO_7YI#oe{NbIAd}cRC+x!-C&C#_v2sspvd~PRe5a zKR>|AwlFBG+8QTSV}hmnr!ZFkb&u-8psB75h8n7V`JssrN>kB>l+FqoXH;O>K%uqc z7T7lD$TCD*FJ)S#NodtMlM7Z19h{EjegM?-rMY4W zeQ%w%E1Q=M?Z2-|0bHB^*MHXV|MdGi`)?cNj`@GzIeH_Dt@YCL{#x<_*9PU?tU)Kn z-xi~R_)wT46M)pt@Hoj;Q4Ozm4(;hDb`x&5wwgNmu(V`=fJepOvEM%KSuN{$oj5YJMh3OfjV}p=@MwWJxl4 z3LH};$sB86xbqw(vG)iPJTb=={FHJ$CZBO^nDf7W4&XeY3t)+EsZfFxbN!nyffpQK ze!P^7WA72zgX-QNT|!K_@PY}EUH#Xf?**ejc~}38lmVaOZE4@ z!g~^k>(qM^jL?<$B#<+8%{ZBOPyW|?1n&_iEE8~gd?LIc