469 lines
16 KiB
Go
469 lines
16 KiB
Go
package server
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"git.myservermanager.com/varakh/upda/util"
|
|
"github.com/adrg/xdg"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zapcore"
|
|
"gorm.io/driver/postgres"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/logger"
|
|
"log"
|
|
"moul.io/zapgorm2"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
type appConfig struct {
|
|
timeZone string
|
|
isDevelopment bool
|
|
isDebug bool
|
|
}
|
|
|
|
type serverConfig struct {
|
|
port int
|
|
listen string
|
|
tlsEnabled bool
|
|
tlsCertPath string
|
|
tlsKeyPath string
|
|
timeout time.Duration
|
|
corsAllowOrigin []string
|
|
corsAllowMethods []string
|
|
corsAllowHeaders []string
|
|
}
|
|
|
|
type authConfig struct {
|
|
adminUser string
|
|
adminPassword string
|
|
}
|
|
|
|
type taskConfig struct {
|
|
updateCleanStaleEnabled bool
|
|
updateCleanStaleInterval string
|
|
updateCleanStaleMaxAge time.Duration
|
|
eventCleanStaleEnabled bool
|
|
eventCleanStaleInterval string
|
|
eventCleanStaleMaxAge time.Duration
|
|
actionsEnqueueEnabled bool
|
|
actionsEnqueueInterval string
|
|
actionsEnqueueBatchSize int
|
|
actionsInvokeEnabled bool
|
|
actionsInvokeInterval string
|
|
actionsInvokeBatchSize int
|
|
actionsInvokeMaxRetries int
|
|
actionsCleanStaleEnabled bool
|
|
actionsCleanStaleInterval string
|
|
actionsCleanStaleMaxAge time.Duration
|
|
prometheusRefreshInterval string
|
|
}
|
|
|
|
type lockConfig struct {
|
|
redisEnabled bool
|
|
redisUrl string
|
|
}
|
|
|
|
type webhookConfig struct {
|
|
tokenLength int
|
|
}
|
|
|
|
type prometheusConfig struct {
|
|
enabled bool
|
|
path string
|
|
secureTokenEnabled bool
|
|
secureToken string
|
|
}
|
|
|
|
type Environment struct {
|
|
appConfig *appConfig
|
|
authConfig *authConfig
|
|
serverConfig *serverConfig
|
|
taskConfig *taskConfig
|
|
lockConfig *lockConfig
|
|
webhookConfig *webhookConfig
|
|
prometheusConfig *prometheusConfig
|
|
db *gorm.DB
|
|
}
|
|
|
|
func bootstrapEnvironment() *Environment {
|
|
var err error
|
|
|
|
// bootstrap logging (configured independently and required before any other action)
|
|
loggingLevel := os.Getenv(envLoggingLevel)
|
|
if loggingLevel == "" {
|
|
if err = os.Setenv(envLoggingLevel, loggingLevelDefault); err != nil {
|
|
log.Fatalf("Cannot set logging level: %v", err)
|
|
}
|
|
loggingLevel = os.Getenv(envLoggingLevel)
|
|
}
|
|
var level zap.AtomicLevel
|
|
if level, err = zap.ParseAtomicLevel(loggingLevel); err != nil {
|
|
log.Fatalf("Cannot parse logging level: %v", err)
|
|
}
|
|
loggingEncoding := os.Getenv(envLoggingEncoding)
|
|
if loggingEncoding == "" {
|
|
if err = os.Setenv(envLoggingEncoding, loggingEncodingDefault); err != nil {
|
|
log.Fatalf("Cannot set logging encoding: %v", err)
|
|
}
|
|
loggingEncoding = os.Getenv(envLoggingEncoding)
|
|
}
|
|
if loggingEncoding != "json" && loggingEncoding != "console" {
|
|
log.Fatalf("Cannot parse logging level: %v", errors.New("only 'json' and 'console' are allowed logging encodings"))
|
|
}
|
|
isDebug := level.Level() == zap.DebugLevel
|
|
isDevelopment := os.Getenv(envDevelopment) == "true"
|
|
var loggingEncoderConfig zapcore.EncoderConfig
|
|
if loggingEncoding == "json" {
|
|
loggingEncoderConfig = zap.NewProductionEncoderConfig()
|
|
} else {
|
|
loggingEncoderConfig = zap.NewDevelopmentEncoderConfig()
|
|
}
|
|
|
|
logPaths := []string{"stderr"}
|
|
loggingDirectory := os.Getenv(envLoggingDirectory)
|
|
|
|
if loggingDirectory != "" {
|
|
logFile := filepath.Join(loggingDirectory, loggingFileNameDefault)
|
|
|
|
if err = util.CreateFileWithParent(logFile); err != nil {
|
|
log.Fatalf("Log file '%s' cannot be created: %v", loggingDirectory, err)
|
|
}
|
|
|
|
logPaths = append(logPaths, logFile)
|
|
}
|
|
|
|
var zapConfig *zap.Config
|
|
if isDebug {
|
|
zapConfig = &zap.Config{
|
|
Level: level,
|
|
Development: isDevelopment,
|
|
Encoding: loggingEncoding,
|
|
EncoderConfig: loggingEncoderConfig,
|
|
OutputPaths: logPaths,
|
|
ErrorOutputPaths: logPaths,
|
|
}
|
|
} else {
|
|
zapConfig = &zap.Config{
|
|
Level: level,
|
|
Development: isDevelopment,
|
|
Sampling: &zap.SamplingConfig{
|
|
Initial: 100,
|
|
Thereafter: 100,
|
|
},
|
|
Encoding: loggingEncoding,
|
|
EncoderConfig: loggingEncoderConfig,
|
|
OutputPaths: logPaths,
|
|
ErrorOutputPaths: logPaths,
|
|
}
|
|
}
|
|
|
|
zapLogger := zap.Must(zapConfig.Build())
|
|
defer zapLogger.Sync()
|
|
zap.ReplaceGlobals(zapLogger)
|
|
|
|
// assign defaults from given environment variables and validate
|
|
bootstrapFromEnvironmentAndValidate()
|
|
|
|
// parse environment variables in actual configuration structs
|
|
// app config
|
|
ac := &appConfig{
|
|
timeZone: os.Getenv(envTZ),
|
|
isDebug: isDebug,
|
|
isDevelopment: isDevelopment,
|
|
}
|
|
|
|
// server config
|
|
var sc *serverConfig
|
|
|
|
var serverPort int
|
|
if serverPort, err = strconv.Atoi(os.Getenv(envServerPort)); err != nil {
|
|
zap.L().Sugar().Fatalf("Invalid server port. Reason: %v", err)
|
|
}
|
|
|
|
serverTlsEnabled := os.Getenv(envServerTlsEnabled) == "true"
|
|
|
|
if serverTlsEnabled {
|
|
failIfEnvKeyNotPresent(envServerTlsCertPath)
|
|
failIfEnvKeyNotPresent(envServerTlsKeyPath)
|
|
}
|
|
|
|
var serverTimeout time.Duration
|
|
var errParse error
|
|
if serverTimeout, errParse = time.ParseDuration(os.Getenv(envServerTimeout)); errParse != nil {
|
|
zap.L().Sugar().Fatalf("Could not parse timeout. Reason: %s", errParse.Error())
|
|
}
|
|
|
|
sc = &serverConfig{
|
|
port: serverPort,
|
|
timeout: serverTimeout,
|
|
listen: os.Getenv(envServerListen),
|
|
tlsEnabled: serverTlsEnabled,
|
|
tlsCertPath: os.Getenv(envServerTlsCertPath),
|
|
tlsKeyPath: os.Getenv(envServerTlsKeyPath),
|
|
corsAllowOrigin: []string{os.Getenv(envCorsAllowOrigin)},
|
|
corsAllowMethods: []string{os.Getenv(envCorsAllowMethods)},
|
|
corsAllowHeaders: []string{os.Getenv(envCorsAllowHeaders)},
|
|
}
|
|
|
|
authC := &authConfig{
|
|
adminUser: os.Getenv(envAdminUser),
|
|
adminPassword: os.Getenv(envAdminPassword),
|
|
}
|
|
|
|
// task config
|
|
var tc *taskConfig
|
|
|
|
var updateCleanStaleMaxAge time.Duration
|
|
if updateCleanStaleMaxAge, errParse = time.ParseDuration(os.Getenv(envTaskUpdateCleanStaleMaxAge)); errParse != nil {
|
|
zap.L().Sugar().Fatalf("Could not parse max age for cleaning stale updates. Reason: %s", errParse.Error())
|
|
}
|
|
|
|
var eventCleanStaleMaxAge time.Duration
|
|
if eventCleanStaleMaxAge, errParse = time.ParseDuration(os.Getenv(envTaskEventCleanStaleMaxAge)); errParse != nil {
|
|
zap.L().Sugar().Fatalf("Could not parse max age for cleaning stale events. Reason: %s", errParse.Error())
|
|
}
|
|
|
|
var actionsEnqueueBatchSize int
|
|
if actionsEnqueueBatchSize, err = strconv.Atoi(os.Getenv(envTaskActionsEnqueueBatchSize)); err != nil {
|
|
zap.L().Sugar().Fatalf("Invalid actions enqueue batch size. Reason: %v", err)
|
|
}
|
|
if actionsEnqueueBatchSize <= 0 {
|
|
zap.L().Sugar().Fatalf("Invalid actions enqueue batch size, must be a positive number.")
|
|
}
|
|
|
|
var actionsInvokeBatchSize int
|
|
if actionsInvokeBatchSize, err = strconv.Atoi(os.Getenv(envTaskActionsInvokeBatchSize)); err != nil {
|
|
zap.L().Sugar().Fatalf("Invalid actions invoke batch size. Reason: %v", err)
|
|
}
|
|
if actionsInvokeBatchSize <= 0 {
|
|
zap.L().Sugar().Fatalf("Invalid actions invoke batch size, must be a positive number.")
|
|
}
|
|
|
|
var actionsInvokeMaxRetries int
|
|
if actionsInvokeMaxRetries, err = strconv.Atoi(os.Getenv(envTaskActionsInvokeMaxRetries)); err != nil {
|
|
zap.L().Sugar().Fatalf("Invalid actions invoke max retries. Reason: %v", err)
|
|
}
|
|
if actionsInvokeMaxRetries <= 0 {
|
|
zap.L().Sugar().Fatalf("Invalid actions invoke max retries, must be a positive number.")
|
|
}
|
|
|
|
var actionsCleanStaleMaxAge time.Duration
|
|
if actionsCleanStaleMaxAge, errParse = time.ParseDuration(os.Getenv(envTaskActionsCleanStaleMaxAge)); errParse != nil {
|
|
zap.L().Sugar().Fatalf("Could not parse max age for cleaning stale actions. Reason: %s", errParse.Error())
|
|
}
|
|
|
|
tc = &taskConfig{
|
|
updateCleanStaleEnabled: os.Getenv(envTaskUpdateCleanStaleEnabled) == "true",
|
|
updateCleanStaleInterval: os.Getenv(envTaskUpdateCleanStaleInterval),
|
|
updateCleanStaleMaxAge: updateCleanStaleMaxAge,
|
|
eventCleanStaleEnabled: os.Getenv(envTaskEventCleanStaleEnabled) == "true",
|
|
eventCleanStaleInterval: os.Getenv(envTaskEventCleanStaleInterval),
|
|
eventCleanStaleMaxAge: eventCleanStaleMaxAge,
|
|
actionsEnqueueEnabled: os.Getenv(envTaskActionsEnqueueEnabled) == "true",
|
|
actionsEnqueueInterval: os.Getenv(envTaskActionsEnqueueInterval),
|
|
actionsEnqueueBatchSize: actionsEnqueueBatchSize,
|
|
actionsInvokeEnabled: os.Getenv(envTaskActionsInvokeEnabled) == "true",
|
|
actionsInvokeInterval: os.Getenv(envTaskActionsInvokeInterval),
|
|
actionsInvokeBatchSize: actionsInvokeBatchSize,
|
|
actionsInvokeMaxRetries: actionsInvokeMaxRetries,
|
|
actionsCleanStaleEnabled: os.Getenv(envTaskActionsCleanStaleEnabled) == "true",
|
|
actionsCleanStaleInterval: os.Getenv(envTaskActionsCleanStaleInterval),
|
|
actionsCleanStaleMaxAge: actionsCleanStaleMaxAge,
|
|
prometheusRefreshInterval: os.Getenv(envTaskPrometheusRefreshInterval),
|
|
}
|
|
|
|
var lc *lockConfig
|
|
lc = &lockConfig{
|
|
redisEnabled: os.Getenv(envLockRedisEnabled) == "true",
|
|
redisUrl: os.Getenv(envLockRedisUrl),
|
|
}
|
|
|
|
webhookTokenLength := 32
|
|
if webhookTokenLength, err = strconv.Atoi(os.Getenv(envWebhooksTokenLength)); err != nil {
|
|
zap.L().Sugar().Fatalf("Invalid webhook token length. Reason: %v", err)
|
|
}
|
|
if webhookTokenLength <= 0 {
|
|
zap.L().Sugar().Fatalln("Invalid webhook token length. Reason: must be a positive number")
|
|
}
|
|
|
|
wc := &webhookConfig{
|
|
tokenLength: webhookTokenLength,
|
|
}
|
|
|
|
pc := &prometheusConfig{
|
|
enabled: os.Getenv(envPrometheusEnabled) == "true",
|
|
path: os.Getenv(envPrometheusMetricsPath),
|
|
secureTokenEnabled: os.Getenv(envPrometheusSecureTokenEnabled) == "true",
|
|
secureToken: os.Getenv(envPrometheusSecureToken),
|
|
}
|
|
|
|
if pc.enabled && pc.secureTokenEnabled {
|
|
failIfEnvKeyNotPresent(envPrometheusSecureToken)
|
|
}
|
|
|
|
// database setup
|
|
gormConfig := &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}
|
|
if isDebug && isDevelopment {
|
|
gormZapLogger := zap.Must(zapConfig.Build())
|
|
defer gormZapLogger.Sync()
|
|
gormLogger := zapgorm2.New(gormZapLogger)
|
|
gormConfig = &gorm.Config{Logger: gormLogger}
|
|
}
|
|
|
|
var db *gorm.DB
|
|
zap.L().Sugar().Infof("Using database type '%s'", os.Getenv(envDbType))
|
|
|
|
if os.Getenv(envDbType) == dbTypeSqlite {
|
|
if os.Getenv(envDbSqliteFile) == "" {
|
|
var defaultDbFile string
|
|
if defaultDbFile, err = xdg.DataFile(Name + "/" + dbTypeSqliteDbNameDefault); err != nil {
|
|
zap.L().Sugar().Fatalf("Database file '%s' could not be created. Reason: %v", defaultDbFile, err)
|
|
}
|
|
setEnvKeyDefault(envDbSqliteFile, defaultDbFile)
|
|
}
|
|
|
|
dbFile := os.Getenv(envDbSqliteFile)
|
|
zap.L().Sugar().Infof("Using database file '%s'", dbFile)
|
|
|
|
if err = util.CreateFileWithParent(dbFile); err != nil {
|
|
zap.L().Sugar().Fatalf("Database file '%s' cannot be created: %v", dbFile, err)
|
|
}
|
|
|
|
if db, err = gorm.Open(sqlite.Open(dbFile), gormConfig); err != nil {
|
|
zap.L().Sugar().Fatalf("Could not setup database: %v", err)
|
|
}
|
|
|
|
if res := db.Exec("PRAGMA foreign_keys = ON"); res.Error != nil {
|
|
zap.L().Sugar().Fatalf("Could not execute foreign key for SQLite: %v", res.Error)
|
|
}
|
|
|
|
sqlDb, _ := db.DB()
|
|
sqlDb.SetMaxOpenConns(1)
|
|
zap.L().Sugar().Infof("SQLite: restricting max connections to '1'")
|
|
} else if os.Getenv(envDbType) == dbTypePostgres {
|
|
host := os.Getenv(envDbPostgresHost)
|
|
port := os.Getenv(envDbPostgresPort)
|
|
dbUser := os.Getenv(envDbPostgresUser)
|
|
dbPass := os.Getenv(envDbPostgresPassword)
|
|
dbName := os.Getenv(envDbPostgresName)
|
|
dbTZ := os.Getenv(envDbPostgresTimeZone)
|
|
|
|
if host == "" || port == "" || dbUser == "" || dbPass == "" || dbName == "" || dbTZ == "" {
|
|
zap.L().Sugar().Fatalf("Some configuration for database type '%s' is missing", os.Getenv(envDbType))
|
|
}
|
|
|
|
dsn := fmt.Sprintf("host=%v user=%v password=%v dbname=%v port=%v sslmode=disable TimeZone=%v", host, dbUser, dbPass, dbName, port, dbTZ)
|
|
if db, err = gorm.Open(postgres.Open(dsn), gormConfig); err != nil {
|
|
zap.L().Sugar().Fatalf("Could not setup database: %v", err)
|
|
}
|
|
} else {
|
|
zap.L().Sugar().Fatalf("Database type '%s' or '%s' is required", dbTypeSqlite, dbTypePostgres)
|
|
}
|
|
|
|
if db == nil {
|
|
zap.L().Sugar().Fatalf("Could not setup database")
|
|
}
|
|
|
|
env := &Environment{appConfig: ac,
|
|
authConfig: authC,
|
|
serverConfig: sc,
|
|
taskConfig: tc,
|
|
lockConfig: lc,
|
|
webhookConfig: wc,
|
|
prometheusConfig: pc,
|
|
db: db}
|
|
|
|
if err = env.db.AutoMigrate(&Update{}, &Webhook{}, &Event{}, &Secret{}, &Action{}, &ActionInvocation{}); err != nil {
|
|
zap.L().Sugar().Fatalf("Could not migrate database schema: %s", err)
|
|
}
|
|
|
|
zap.L().Sugar().Infof("appConfig %+v", env.appConfig)
|
|
zap.L().Sugar().Infof("serverConfig %+v", env.serverConfig)
|
|
zap.L().Sugar().Infof("taskConfig %+v", env.taskConfig)
|
|
zap.L().Sugar().Infof("webhookConfig %+v", env.webhookConfig)
|
|
|
|
return env
|
|
}
|
|
|
|
func bootstrapFromEnvironmentAndValidate() {
|
|
// app
|
|
setEnvKeyDefault(envTZ, tzDefault)
|
|
|
|
failIfEnvKeyNotPresent(envSecret)
|
|
failIfEnvKeyNotPresent(envAdminUser)
|
|
failIfEnvKeyNotPresent(envAdminPassword)
|
|
|
|
// webhook
|
|
setEnvKeyDefault(envWebhooksTokenLength, webhooksTokenLengthDefault)
|
|
|
|
// lock
|
|
setEnvKeyDefault(envLockRedisEnabled, redisEnabledDefault)
|
|
|
|
// task
|
|
setEnvKeyDefault(envTaskUpdateCleanStaleEnabled, taskUpdateCleanStaleEnabledDefault)
|
|
setEnvKeyDefault(envTaskUpdateCleanStaleInterval, taskUpdateCleanStaleIntervalDefault)
|
|
setEnvKeyDefault(envTaskUpdateCleanStaleMaxAge, taskUpdateCleanStaleMaxAgeDefault)
|
|
|
|
setEnvKeyDefault(envTaskEventCleanStaleEnabled, taskEventCleanStaleEnabledDefault)
|
|
setEnvKeyDefault(envTaskEventCleanStaleInterval, taskEventCleanStaleIntervalDefault)
|
|
setEnvKeyDefault(envTaskEventCleanStaleMaxAge, taskEventCleanStaleMaxAgeDefault)
|
|
|
|
setEnvKeyDefault(envTaskActionsEnqueueEnabled, taskActionsEnqueueEnabledDefault)
|
|
setEnvKeyDefault(envTaskActionsEnqueueInterval, taskActionsEnqueueIntervalDefault)
|
|
setEnvKeyDefault(envTaskActionsEnqueueBatchSize, taskActionsEnqueueBatchSizeDefault)
|
|
|
|
setEnvKeyDefault(envTaskActionsInvokeEnabled, taskActionsInvokeEnabledDefault)
|
|
setEnvKeyDefault(envTaskActionsInvokeInterval, taskActionsInvokeIntervalDefault)
|
|
setEnvKeyDefault(envTaskActionsInvokeBatchSize, taskActionsInvokeBatchSizeDefault)
|
|
setEnvKeyDefault(envTaskActionsInvokeMaxRetries, taskActionsInvokeMaxRetriesDefault)
|
|
|
|
setEnvKeyDefault(envTaskActionsCleanStaleEnabled, taskActionsCleanStaleEnabledDefault)
|
|
setEnvKeyDefault(envTaskActionsCleanStaleInterval, taskActionsCleanStaleIntervalDefault)
|
|
setEnvKeyDefault(envTaskActionsCleanStaleMaxAge, taskActionsCleanStaleMaxAgeDefault)
|
|
|
|
setEnvKeyDefault(envTaskPrometheusRefreshInterval, taskPrometheusRefreshDefault)
|
|
|
|
// prometheus
|
|
setEnvKeyDefault(envPrometheusEnabled, prometheusEnabledDefault)
|
|
setEnvKeyDefault(envPrometheusMetricsPath, prometheusMetricsPathDefault)
|
|
setEnvKeyDefault(envPrometheusSecureTokenEnabled, prometheusSecureTokenEnabledDefault)
|
|
|
|
// db
|
|
setEnvKeyDefault(envDbType, dbTypeSqlite)
|
|
|
|
if os.Getenv(envDbType) == dbTypePostgres {
|
|
setEnvKeyDefault(envDbPostgresHost, dbTypePostgresHostDefault)
|
|
setEnvKeyDefault(envDbPostgresPort, dbTypePostgresPortDefault)
|
|
setEnvKeyDefault(envDbPostgresTimeZone, dbTypePostgresTZDefault)
|
|
}
|
|
|
|
// server
|
|
setEnvKeyDefault(envServerPort, serverPortDefault)
|
|
setEnvKeyDefault(envServerListen, serverListenDefault)
|
|
setEnvKeyDefault(envServerTlsEnabled, serverTlsEnabledDefault)
|
|
setEnvKeyDefault(envCorsAllowOrigin, corsAllowOriginDefault)
|
|
setEnvKeyDefault(envCorsAllowMethods, corsAllowMethodsDefault)
|
|
setEnvKeyDefault(envCorsAllowHeaders, corsAllowHeadersDefault)
|
|
setEnvKeyDefault(envServerTimeout, serverTimeoutDefault)
|
|
}
|
|
|
|
func failIfEnvKeyNotPresent(key string) {
|
|
if os.Getenv(key) == "" {
|
|
zap.L().Sugar().Fatalf("Not all required ENV variables given. Please set '%s'", key)
|
|
}
|
|
}
|
|
|
|
func setEnvKeyDefault(key string, defaultValue string) {
|
|
var err error
|
|
if os.Getenv(key) == "" {
|
|
if err = os.Setenv(key, defaultValue); err != nil {
|
|
zap.L().Sugar().Fatalf("Could not set default value for ENV variable '%s'", key)
|
|
}
|
|
|
|
zap.L().Sugar().Infof("Set '%s' to '%s'", key, defaultValue)
|
|
}
|
|
}
|