logicamp/flink-kube-operator
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: retry upload on jar not found

Browse Source
This commit is contained in:
Mohamad Khani 2024-12-08 01:24:14 +03:30
parent 5abc044d69
commit 91ccfebfeb
3 changed files with 79 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
internal/managed_job/cycle.go
View File

@ -2,6 +2,7 @@ package managed_job
import (
"flink-kube-operator/internal/crd/v1alpha1"
"strings"
"time"
"gitea.com/logicamp/lc"
@ -32,6 +33,8 @@ func (job *ManagedJob) Cycle() {
// Init job
if job.def.Status.JobStatus == "" {
if job.def.Status.LastSavepointPath == nil {
if job.def.Status.JarId == nil {
err := job.upload()
if err != nil {
job.crd.Patch(job.def.UID, map[string]interface{}{
@ -41,8 +44,22 @@ func (job *ManagedJob) Cycle() {
})
return
}
err = job.run()
}
for {
err := job.run()
if err != nil {
if strings.ContainsAny(err.Error(), ".jar does not exist") {
err := job.upload()
if err != nil {
job.crd.Patch(job.def.UID, map[string]interface{}{
"status": map[string]interface{}{
"error": "[upload-error] " + err.Error(),
},
})
return
}
continue
}
job.crd.Patch(job.def.UID, map[string]interface{}{
"status": map[string]interface{}{
"error": "[run-error] " + err.Error(),
@ -52,6 +69,11 @@ func (job *ManagedJob) Cycle() {
}
return
}
} else {
job.restore()
}
return
}
// job.crd.SetJobStatus(job.def.UID, v1alpha1.FlinkJobStatus{
// JobStatus: job.def.Status.JobStatus,
// })

32
internal/managed_job/restore.go
View File

@ -3,6 +3,7 @@ package managed_job
import (
"errors"
"flink-kube-operator/internal/crd/v1alpha1"
"strings"
"time"
"gitea.com/logicamp/lc"
@ -18,10 +19,12 @@ func (job *ManagedJob) restore() error {
}
if job.def.Status.JarId == nil {
err := errors.New("missing jar id")
lc.Logger.Error("[managed-job] [run]", zap.Error(err))
lc.Logger.Error("[managed-job] [restore]", zap.Error(err))
return err
}
lc.Logger.Debug("[managed-job] [restore] restoring", zap.String("savepointPath", *job.def.Status.LastSavepointPath))
lc.Logger.Info("[managed-job] [restore] restoring job", zap.String("name", job.def.GetName()), zap.String("savepointPath", *job.def.Status.LastSavepointPath))
var jobId *string
for {
runJarResp, err := job.client.RunJar(api.RunOpts{
JarID: *job.def.Status.JarId,
AllowNonRestoredState: true,
@ -29,25 +32,40 @@ func (job *ManagedJob) restore() error {
SavepointPath: *job.def.Status.LastSavepointPath,
})
if err != nil {
lc.Logger.Error("[managed-job] [run]", zap.Error(err))
if strings.ContainsAny(err.Error(), ".jar does not exist") {
err := job.upload()
if err != nil {
job.crd.Patch(job.def.UID, map[string]interface{}{
"status": map[string]interface{}{
"error": "[upload-error] " + err.Error(),
},
})
return nil
}
continue
}
lc.Logger.Error("[managed-job] [restore]", zap.Error(err))
return err
}
jobId = &runJarResp.JobId
lc.Logger.Debug("[main] after run jar", zap.Any("run-jar-resp", runJarResp))
break
}
// job.def.Status.JobId = &runJarResp.JobId
// job.def.Status.JobStatus = v1alpha1.JobStatusCreating
// job.def.Status.Error = nil
job.crd.Patch(job.def.UID, map[string]interface{}{
"status": map[string]interface{}{
"jobId": &runJarResp.JobId,
"jobId": jobId,
"jobStatus": v1alpha1.JobStatusCreating,
"lifeCycleStatus": v1alpha1.LifeCycleStatusRestoring,
"lastRestoredSavepointDate": job.def.Status.LastRestoredSavepointDate,
"lastRestoredSavepointDate": job.def.Status.LastSavepointDate,
"restoredCount": job.def.Status.RestoredCount + 1,
"lastRestoredSavepointRestoreDate": time.Now().Format(time.RFC3339),
"lastRestoredSavepointRestoredDate": time.Now().Format(time.RFC3339),
"error": nil,
},
})
return err
return nil
}

1
internal/managed_job/run.go
View File

@ -41,6 +41,7 @@ func (job *ManagedJob) run() error {
lc.Logger.Error("[managed-job] [run]", zap.Error(err))
return err
}
lc.Logger.Info("[managed-job] [run] starting job", zap.String("name", job.def.GetName()))
runJarResp, err := job.client.RunJar(api.RunOpts{
JarID: *job.def.Status.JarId,
AllowNonRestoredState: true,