logicamp/flink-kube-operator
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add manager

Browse Source
This commit is contained in:
Mohamad Khani 2024-11-30 23:36:44 +03:30
parent d8b87ac6ee
commit 6bd197b812
14 changed files with 109 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/settings.json vendored
View File

@ -1,7 +1,9 @@
{
"cSpell.words": [
"apiextensions",
"buntdb",
"deepcopy",
"flink"
"flink",
"repsert"
]
}

35
cmd/operator/main.go
View File

@ -1,9 +1,8 @@
package main
import (
"flink-kube-operator/internal/config"
"flink-kube-operator/internal/crd"
"flink-kube-operator/internal/managed_job"
"flink-kube-operator/internal/manager"
"fmt"
"log"
"os"
@ -51,22 +50,24 @@ func main() {
}
lc.Logger.Debug("[main] jobs", zap.Any("jobs", jobs))
config := lc.LoadYamlConfig[config.Config]("./config.yaml")
for _, jobDef := range config.Jobs {
managed_job.NewManagedJob(c, db, jobDef)
}
manager.Setup(c, db)
for _, job := range jobs.Jobs {
job, err := c.Job(job.ID)
if err != nil {
lc.Logger.Error("error getting job info", zap.Error(err))
continue
}
if job.State == "RUNNING" {
lc.Logger.Debug("[main] running job", zap.String("jobId", job.ID))
}
// lc.Logger.Debug("[main]", zap.Any("job", job))
}
//config := lc.LoadYamlConfig[config.Config]("./config.yaml")
// for _, jobDef := range config.Jobs {
// managed_job.NewManagedJob(c, db, jobDef)
// }
// for _, job := range jobs.Jobs {
// job, err := c.Job(job.ID)
// if err != nil {
// lc.Logger.Error("error getting job info", zap.Error(err))
// continue
// }
// if job.State == "RUNNING" {
// lc.Logger.Debug("[main] running job", zap.String("jobId", job.ID))
// }
// // lc.Logger.Debug("[main]", zap.Any("job", job))
// }
cancelChan := make(chan os.Signal, 1)
sig := <-cancelChan

5
config.yaml
View File

@ -1,6 +1 @@
jobs:
- key: price-processor
name: Price Processor
entryClass: tech.logicamp.Main
jarURI: http://price-processor.bz2/price-processor-v0.0.1.jar
savepointInterval: 3m

2
go.mod
View File

@ -5,7 +5,7 @@ go 1.23.2
require (
gitea.com/logicamp/lc v1.14.6
github.com/dgraph-io/badger/v4 v4.5.0
github.com/logi-camp/go-flink-client v0.1.0
github.com/logi-camp/go-flink-client v0.1.1
github.com/matoous/go-nanoid/v2 v2.1.0
github.com/tidwall/buntdb v1.3.2
go.uber.org/zap v1.27.0

2
go.sum
View File

@ -262,6 +262,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/logi-camp/go-flink-client v0.1.0 h1:uzBV6RGkyzZVdSQ8zAlUGbJ5hdXRspaPjzzYJUQ2aJU=
github.com/logi-camp/go-flink-client v0.1.0/go.mod h1:A79abedX6wGQI0FoICdZI7SRoGHj15QwMwWowgsKYFI=
github.com/logi-camp/go-flink-client v0.1.1 h1:N/XLXE/WZGPKgJo8pN+Q3JrHBOoZHwUoditd399coRo=
github.com/logi-camp/go-flink-client v0.1.1/go.mod h1:A79abedX6wGQI0FoICdZI7SRoGHj15QwMwWowgsKYFI=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=

14
internal/config/config.type.go
View File

@ -1,15 +1 @@
package config
import "time"
type JobDef struct {
Key string `yaml:"key"`
Name string `yaml:"name"`
EntryClass string `yaml:"entryClass"`
JarURI string `yaml:"jarURI"`
SavepointInterval time.Duration `yaml:"savepointInterval"`
}
type Config struct {
Jobs []JobDef `yaml:"jobs"`
}

26
internal/crd/repo.go Normal file
View File

@ -0,0 +1,26 @@
package crd
import (
"flink-kube-operator/internal/crd/v1alpha1"
"maps"
"k8s.io/apimachinery/pkg/types"
)
var jobs = map[types.UID]*v1alpha1.FlinkJob{}
func (crd Crd) repsert(job *v1alpha1.FlinkJob) {
jobs[job.GetUID()] = job
}
func GetJob(uid types.UID) v1alpha1.FlinkJob {
return *jobs[uid].DeepCopy()
}
func GetAllJobKeys() []types.UID {
keys := []types.UID{}
for k := range maps.Keys(jobs) {
keys = append(keys, k)
}
return keys
}

16
internal/crd/watch.go
View File

@ -40,25 +40,13 @@ func (crd Crd) watchFlinkJobs() {
case watch.Modified:
fmt.Printf("FlinkJob updated: %s\n", job.GetName())
// Handle the new FlinkJob
handleNewFlinkJob(job)
crd.repsert(job)
case watch.Added:
fmt.Printf("New FlinkJob created: %s\n", job.GetName())
// Handle the new FlinkJob
handleNewFlinkJob(job)
crd.repsert(job)
case watch.Deleted:
}
}
}
func handleNewFlinkJob(job *v1alpha1.FlinkJob) {
// Extract job details
name := job.GetName()
namespace := job.GetNamespace()
// Process job specification
fmt.Printf("Processing FlinkJob %s in namespace %s kind: %s \n", name, namespace, job.Kind)
lc.Logger.Debug("[crd] [watch]", zap.Any("spec", job), zap.Any("name", job.Spec.Name))
// Add your custom logic here
}

4
internal/jar/jar.go
View File

@ -29,7 +29,7 @@ func NewJarFile(URI string) (*JarFile, error) {
return jarFile, nil
}
func (JarFile JarFile) Upload(flinkClient *api.Client) (fileName string, err error) {
func (JarFile *JarFile) Upload(flinkClient *api.Client) (fileName string, err error) {
resp, err := flinkClient.UploadJar(JarFile.filePath)
if err != nil {
@ -65,6 +65,6 @@ func (jarFile *JarFile) Download() error {
return nil
}
func (jarFile JarFile) Delete() error {
func (jarFile *JarFile) Delete() error {
return os.Remove(jarFile.filePath)
}

4
internal/managed_job/cycle.go
View File

@ -28,7 +28,7 @@ func (job *ManagedJob) startCycle() {
}
func (job *ManagedJob) cycle() {
lc.Logger.Debug("[managed-job] [new] check cycle", zap.String("jobKey", job.def.Key))
lc.Logger.Debug("[managed-job] [new] check cycle", zap.String("jobKey", string(job.def.UID)))
// Init job
if job.state == nil {
@ -58,7 +58,7 @@ func (job *ManagedJob) cycle() {
if errors.Is(err, ErrNoJobId) {
job.state = nil
}
if job.state.LastSavepointDate == nil || time.Now().Add(-job.def.SavepointInterval).After(*job.state.LastSavepointDate) {
if job.state.LastSavepointDate == nil || time.Now().Add(-job.def.Spec.SavepointInterval.Duration).After(*job.state.LastSavepointDate) {
if job.state.SavepointTriggerId == nil {
job.createSavepoint()
} else {

6
internal/managed_job/new.go
View File

@ -1,21 +1,21 @@
package managed_job
import (
"flink-kube-operator/internal/config"
"flink-kube-operator/internal/crd/v1alpha1"
api "github.com/logi-camp/go-flink-client"
"github.com/tidwall/buntdb"
)
type ManagedJob struct {
def config.JobDef
def v1alpha1.FlinkJob
client *api.Client
jarId string
db *buntdb.DB
state *jobState
}
func NewManagedJob(client *api.Client, db *buntdb.DB, def config.JobDef) *ManagedJob {
func NewManagedJob(client *api.Client, db *buntdb.DB, def v1alpha1.FlinkJob) *ManagedJob {
job := &ManagedJob{
def: def,
client: client,

4
internal/managed_job/run.go
View File

@ -10,7 +10,7 @@ import (
// upload jar file and set the jarId for later usages
func (job *ManagedJob) upload() error {
jarFile, err := jar.NewJarFile(job.def.JarURI)
jarFile, err := jar.NewJarFile(job.def.Spec.JarURI)
if err != nil {
lc.Logger.Debug("[main] error on download jar", zap.Error(err))
return err
@ -32,7 +32,7 @@ func (job *ManagedJob) run() error {
runJarResp, err := job.client.RunJar(api.RunOpts{
JarID: job.jarId,
AllowNonRestoredState: true,
EntryClass: job.def.EntryClass,
EntryClass: job.def.Spec.EntryClass,
})
if err != nil {
lc.Logger.Error("[managed-job] [run]", zap.Error(err))

12
internal/managed_job/state.go
View File

@ -13,7 +13,7 @@ import (
func (job *ManagedJob) loadState() {
err := job.db.View(
func(tx *buntdb.Tx) error {
if val, err := tx.Get(job.def.Key); err != nil {
if val, err := tx.Get(string(job.def.GetUID())); err != nil {
return err
} else {
return json.Unmarshal([]byte(val), job.state)
@ -30,15 +30,15 @@ func (job *ManagedJob) updateState(state jobState) {
value, _ := json.Marshal(job.state)
job.db.Update(func(tx *buntdb.Tx) error {
_, _, err := tx.Set(job.def.Key, string(value), nil)
if err != nil {
return err
}
return tx.Commit()
_, _, err := tx.Set(string(job.def.GetUID()), string(value), nil)
return err
})
}
func (job *ManagedJob) setError(errMsg string) {
if job.state == nil {
job.state = &jobState{}
}
job.state.Error = &errMsg
job.state.Status = JobStatusError
job.updateState(*job.state)

42
internal/manager/manager.go Normal file
View File

@ -0,0 +1,42 @@
package manager
import (
"flink-kube-operator/internal/crd"
"flink-kube-operator/internal/managed_job"
"time"
api "github.com/logi-camp/go-flink-client"
"github.com/tidwall/buntdb"
"k8s.io/apimachinery/pkg/types"
)
var managedJobs = map[types.UID]managed_job.ManagedJob{}
func Setup(client *api.Client, db *buntdb.DB) {
ticker := time.NewTicker(5 * time.Second)
quit := make(chan struct{})
go func() {
for {
select {
case <-ticker.C:
cycle(client, db)
case <-quit:
ticker.Stop()
return
}
}
}()
}
func cycle(client *api.Client, db *buntdb.DB) {
for _, uid := range crd.GetAllJobKeys() {
job := crd.GetJob(uid)
_, ok := managedJobs[uid]
if ok {
} else {
managed_job.NewManagedJob(client, db, job)
}
}
}