refactor: remove gorm, use pgx for postgres

client/cmd/commands.go

@@ -10,7 +10,6 @@ import (
 	"github.com/odpf/optimus/client/cmd/extension"
 	"github.com/odpf/optimus/client/cmd/initialize"
 	"github.com/odpf/optimus/client/cmd/job"
-	"github.com/odpf/optimus/client/cmd/migration"
 	"github.com/odpf/optimus/client/cmd/namespace"
 	"github.com/odpf/optimus/client/cmd/playground"
 	"github.com/odpf/optimus/client/cmd/plugin"
@@ -64,7 +63,6 @@ func New() *cli.Command {
 		deploy.NewDeployCommand(),
 		initialize.NewInitializeCommand(),
 		job.NewJobCommand(),
-		migration.NewMigrationCommand(),
 		namespace.NewNamespaceCommand(),
 		project.NewProjectCommand(),
 		replay.NewReplayCommand(),

config.sample.yaml

@@ -30,7 +30,7 @@ log:
 #    # database connection string
 #    dsn: postgres://user:password@localhost:5432/database?sslmode=disable
 #
-#    max_idle_connection: 5
+#    min_open_connection: 5
 #    max_open_connection: 10
 
 # optimus supports multiple scheduler types

config/config_server.go

@@ -38,7 +38,7 @@ type Deployer struct {
 
 type DBConfig struct {
 	DSN               string `mapstructure:"dsn"`                              // data source name e.g.: postgres://user:password@host:123/database?sslmode=disable
-	MaxIdleConnection int    `mapstructure:"max_idle_connection" default:"10"` // maximum allowed idle DB connections
+	MinOpenConnection int    `mapstructure:"min_open_connection" default:"5"`  // minimum open DB connections
 	MaxOpenConnection int    `mapstructure:"max_open_connection" default:"20"` // maximum allowed open DB connections
 }
 

config/loader_test.go

@@ -271,7 +271,7 @@ func (s *ConfigTestSuite) initExpectedServerConfig() {
 	s.expectedServerConfig.Serve.Deployer.QueueCapacity = 10
 	s.expectedServerConfig.Serve.DB = config.DBConfig{}
 	s.expectedServerConfig.Serve.DB.DSN = "postgres://user:password@localhost:5432/database?sslmode=disable"
-	s.expectedServerConfig.Serve.DB.MaxIdleConnection = 5
+	s.expectedServerConfig.Serve.DB.MinOpenConnection = 5
 	s.expectedServerConfig.Serve.DB.MaxOpenConnection = 10
 
 	s.expectedServerConfig.Scheduler = config.SchedulerConfig{}

core/scheduler/job_run.go

@@ -35,10 +35,12 @@ func (i JobRunID) IsEmpty() bool {
 type JobRun struct {
 	ID uuid.UUID
 
-	JobName JobName
-	Tenant  tenant.Tenant
-
+	JobName   JobName
+	Tenant    tenant.Tenant
+	State     State
 	StartTime time.Time
+	SLAAlert  bool
+	EndTime   time.Time
 }
 
 type OperatorRun struct {

ext/scheduler/airflow/__lib.py

@@ -189,7 +189,7 @@ def execute(self, context):
 
             if len(pod_list.items) == 1:
                 try_numbers_match = self._try_numbers_match(context, pod_list.items[0])
-                final_state, result = self.handle_pod_overlap(labels, try_numbers_match, launcher, pod_list.items[0])
+                final_state, _, result = self.handle_pod_overlap(labels, try_numbers_match, launcher, pod_list.items[0])
             else:
                 final_state, _, result = self.create_new_pod_for_operator(labels, launcher)
 

go.mod

@@ -6,7 +6,6 @@ require (
 	cloud.google.com/go/bigquery v1.44.0
 	github.com/AlecAivazis/survey/v2 v2.2.7
 	github.com/MakeNowJust/heredoc v1.0.0
-	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/PagerDuty/go-pagerduty v1.5.1
 	github.com/briandowns/spinner v1.18.0
 	github.com/charmbracelet/bubbles v0.13.0
@@ -25,6 +24,7 @@ require (
 	github.com/hashicorp/go-getter v1.6.2
 	github.com/hashicorp/go-hclog v0.14.1
 	github.com/hashicorp/go-plugin v1.4.1
+	github.com/jackc/pgx/v5 v5.2.0
 	github.com/kushsharma/parallel v0.2.1
 	github.com/lib/pq v1.10.4
 	github.com/mattn/go-isatty v0.0.16
@@ -57,9 +57,6 @@ require (
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	gorm.io/datatypes v1.0.0
-	gorm.io/driver/postgres v1.0.8
-	gorm.io/gorm v1.21.16
 )
 
 require (
@@ -68,8 +65,6 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	cloud.google.com/go/iam v0.7.0 // indirect
 	cloud.google.com/go/storage v1.27.0 // indirect
-	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/alecthomas/chroma v0.8.2 // indirect
 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
@@ -104,21 +99,12 @@ require (
 	github.com/hashicorp/go-version v1.3.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.11.0 // indirect
-	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.10.0 // indirect
-	github.com/jackc/pgx/v4 v4.15.0 // indirect
+	github.com/jackc/puddle/v2 v2.1.2 // indirect
 	github.com/jeremywohl/flatten v1.0.1 // indirect
 	github.com/jhump/protoreflect v1.9.1-0.20210817181203-db1a327a393e // indirect
-	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.15.1 // indirect
@@ -127,16 +113,13 @@ require (
 	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
-	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mcuadros/go-defaults v1.2.0 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/microcosm-cc/bluemonday v1.0.6 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
-	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
-	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
@@ -149,7 +132,7 @@ require (
 	github.com/prometheus/common v0.30.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -160,10 +143,11 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel/metric v0.30.0 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/ratelimit v0.2.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
-	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
+	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
+	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.4.0 // indirect