From 5763d770451c0737e7c82337978e30b29dec35b9 Mon Sep 17 00:00:00 2001 From: Varakh Date: Fri, 22 Dec 2023 13:13:44 +0100 Subject: [PATCH] Added locking, improve prometheus handling and change to more reasonable configuration defaults for background tasks - Disable cleaning up stale updates and events by default - Change Prometheus exporter behavior - Return -1 for deleted updates in Prometheus which are evicted on next application restart - Ignore PROMETHEUS_METRICS_PATH (defaults to /metrics) in application metrics - Improve template Grafana Dashboard - Add locking for background task in non-distributed environments to avoid overlaps --- CHANGELOG.md | 8 ++- README.md | 89 +++++++++++++------------ _doc/upda-grafana-dashboard.json | 68 +++++++------------ go.mod | 4 +- go.sum | 51 +++++++------- server/app.go | 6 +- server/constants_env.go | 10 +-- server/environment.go | 21 ++++-- server/service_lock.go | 9 +++ server/service_lock_mem.go | 52 +++++++++++++++ server/service_prometheus.go | 3 +- server/service_task.go | 77 +++++++++++++++++++-- server/service_update.go | 16 +++-- util/locker_memory.go | 111 +++++++++++++++++++++++++++++++ 14 files changed, 384 insertions(+), 141 deletions(-) create mode 100644 server/service_lock.go create mode 100644 server/service_lock_mem.go create mode 100644 util/locker_memory.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ed262b..98eaa4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,13 @@ Changes adhere to [semantic versioning](https://semver.org). ## [1.0.1] - UNRELEASED -* ... +* Disable cleaning up stale updates and events by default +* Change Prometheus exporter behavior + * Return `-1` for deleted updates in Prometheus which are evicted on next application restart + * Ignore `PROMETHEUS_METRICS_PATH` (defaults to `/metrics`) in application metrics +* Introduce locking for periodic background tasks + * Rename `TASK_LOCK_REDIS_ENABLED` to `LOCK_REDIS_ENABLED` which still defaults to `false` (disabled) + * Rename `TASK_LOCK_REDIS_URL` to `LOCK_REDIS_URL` ## [1.0.0] - 2023/12/21 diff --git a/README.md b/README.md index 2607b83..1888d1f 100644 --- a/README.md +++ b/README.md @@ -127,49 +127,50 @@ via web interface or API. The following environment variables can be used to modify application behavior. -| Variable | Purpose | Default/Description | -|:-----------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| -| `TZ` | The time zone (**recommended** to set it properly, background tasks depend on it) | Defaults to `Europe/Berlin`, can be any time zone according to _tz database_ | -| `ADMIN_USER` | Admin user name for login | Not set by default, you need to explicitly set it to user name | -| `ADMIN_PASSWORD` | Admin password for login | Not set by default, you need to explicitly set it to a secure random | -| | | | -| `DB_TYPE` | The database type (Postgres is **recommended**) | Defaults to `sqlite`, possible values are `sqlite` or `postgres` | -| `DB_SQLITE_FILE` | Path to the SQLITE file | Defaults to `/upda/upda.db`, e.g. `~/.local/share/upda/upda.db` | -| `DB_POSTGRES_HOST` | The postgres host | Postgres host address, defaults to `localhost` | -| `DB_POSTGRES_PORT` | The postgres port | Postgres port, defaults to `5432` | -| `DB_POSTGRES_NAME` | The postgres database name | Postgres database name, needs to be set | -| `DB_POSTGRES_TZ` | The postgres time zone | Postgres time zone settings, defaults to `Europe/Berlin` | -| `DB_POSTGRES_USER` | The postgres user | Postgres user name, needs to be set | -| `DB_POSTGRES_PASSWORD` | The postgres password | Postgres user password, needs to be set | -| | | | -| `SERVER_PORT` | Port | Defaults to `8080` | -| `SERVER_LISTEN` | Server's listen address | Defaults to empty which equals `0.0.0.0` | -| `SERVER_TLS_ENABLED` | If server uses TLS | Defaults `false` | -| `SERVER_TLS_CERT_PATH` | When TLS enabled, provide the certificate path | | -| `SERVER_TLS_KEY_PATH` | When TLS enabled, provide the key path | | -| `SERVER_TIMEOUT` | Timeout the server waits before shutting down to end any pending tasks | Defaults to `1s` (1 second), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | -| `CORS_ALLOW_ORIGIN` | CORS configuration | Defaults to `*` | -| `CORS_ALLOW_METHODS` | CORS configuration | Defaults to `GET, POST, PUT, PATCH, DELETE, OPTIONS` | -| `CORS_ALLOW_HEADERS` | CORS configuration | Defaults to `Authorization, Content-Type` | -| | | | -| `LOGGING_LEVEL` | Logging level. Possible are `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal` | Defaults to `info` | -| | | | -| `WEBHOOKS_TOKEN_LENGTH` | The length of the token | Defaults to `16`, positive number | -| | | | -| `TASK_UPDATE_CLEAN_STALE_ENABLED` | If background task should run to do housekeeping of stale (ignored/approved) updates from the database | Defaults to `true` | -| `TASK_UPDATE_CLEAN_STALE_INTERVAL` | Interval at which a background task does housekeeping by deleting stale (ignored/approved) updates from the database | Defaults to `1h` (1 hour), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | -| `TASK_UPDATE_CLEAN_STALE_MAX_AGE` | Number defining at which age stale (ignored/approved) updates are deleted by the background task (_updatedAt_ attribute decides) | Defaults to `168h` (168 hours = 1 week), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | -| `TASK_EVENT_CLEAN_STALE_ENABLED` | If background task should run to do housekeeping of stale (old) events from the database | Defaults to `true` | -| `TASK_EVENT_CLEAN_STALE_INTERVAL` | Interval at which a background task does housekeeping by deleting stale (old) events from the database | Defaults to `8h` (8 hours), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | -| `TASK_EVENT_CLEAN_STALE_MAX_AGE` | Number defining at which age stale (old) events are deleted by the background task (_updatedAt_ attribute decides) | Defaults to `2190h` (2190 hours = 3 months), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | -| `TASK_PROMETHEUS_REFRESH_INTERVAL` | Interval at which a background task updates custom metrics | Defaults to `60s` (60 seconds), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | -| `TASK_LOCK_REDIS_ENABLED` | If task locking (multiple instances) is enabled. Requires REDIS. | Defaults to `false` | -| `TASK_LOCK_REDIS_URL` | If task locking via REDIS is enabled, this should point to a resolvable REDIS instance, e.g. `redis://:@localhost:6379/`. | | -| | | | -| `PROMETHEUS_ENABLED` | If Prometheus metrics are exposed | Defaults to `false` | -| `PROMETHEUS_METRICS_PATH` | Defines the metrics endpoint path | Defaults to `/metrics` | -| `PROMETHEUS_SECURE_TOKEN_ENABLED` | If Prometheus metrics endpoint is protected by a token when enabled (**recommended**) | Defaults to `true` | -| `PROMETHEUS_SECURE_TOKEN` | The token securing the metrics endpoint when enabled (**recommended**) | Not set by default, you need to explicitly set it to a secure random | +| Variable | Purpose | Default/Description | +|:-----------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| +| `TZ` | The time zone (**recommended** to set it properly, background tasks depend on it) | Defaults to `Europe/Berlin`, can be any time zone according to _tz database_ | +| `ADMIN_USER` | Admin user name for login | Not set by default, you need to explicitly set it to user name | +| `ADMIN_PASSWORD` | Admin password for login | Not set by default, you need to explicitly set it to a secure random | +| | | | +| `DB_TYPE` | The database type (Postgres is **recommended**) | Defaults to `sqlite`, possible values are `sqlite` or `postgres` | +| `DB_SQLITE_FILE` | Path to the SQLITE file | Defaults to `/upda/upda.db`, e.g. `~/.local/share/upda/upda.db` | +| `DB_POSTGRES_HOST` | The postgres host | Postgres host address, defaults to `localhost` | +| `DB_POSTGRES_PORT` | The postgres port | Postgres port, defaults to `5432` | +| `DB_POSTGRES_NAME` | The postgres database name | Postgres database name, needs to be set | +| `DB_POSTGRES_TZ` | The postgres time zone | Postgres time zone settings, defaults to `Europe/Berlin` | +| `DB_POSTGRES_USER` | The postgres user | Postgres user name, needs to be set | +| `DB_POSTGRES_PASSWORD` | The postgres password | Postgres user password, needs to be set | +| | | | +| `SERVER_PORT` | Port | Defaults to `8080` | +| `SERVER_LISTEN` | Server's listen address | Defaults to empty which equals `0.0.0.0` | +| `SERVER_TLS_ENABLED` | If server uses TLS | Defaults `false` | +| `SERVER_TLS_CERT_PATH` | When TLS enabled, provide the certificate path | | +| `SERVER_TLS_KEY_PATH` | When TLS enabled, provide the key path | | +| `SERVER_TIMEOUT` | Timeout the server waits before shutting down to end any pending tasks | Defaults to `1s` (1 second), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | +| `CORS_ALLOW_ORIGIN` | CORS configuration | Defaults to `*` | +| `CORS_ALLOW_METHODS` | CORS configuration | Defaults to `GET, POST, PUT, PATCH, DELETE, OPTIONS` | +| `CORS_ALLOW_HEADERS` | CORS configuration | Defaults to `Authorization, Content-Type` | +| | | | +| `LOGGING_LEVEL` | Logging level. Possible are `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal` | Defaults to `info` | +| | | | +| `WEBHOOKS_TOKEN_LENGTH` | The length of the token | Defaults to `16`, positive number | +| | | | +| `TASK_UPDATE_CLEAN_STALE_ENABLED` | If background task should run to do housekeeping of stale (ignored/approved) updates from the database | Defaults to `false` | +| `TASK_UPDATE_CLEAN_STALE_INTERVAL` | Interval at which a background task does housekeeping by deleting stale (ignored/approved) updates from the database | Defaults to `1h` (1 hour), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | +| `TASK_UPDATE_CLEAN_STALE_MAX_AGE` | Number defining at which age stale (ignored/approved) updates are deleted by the background task (_updatedAt_ attribute decides) | Defaults to `168h` (168 hours = 1 week), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | +| `TASK_EVENT_CLEAN_STALE_ENABLED` | If background task should run to do housekeeping of stale (old) events from the database | Defaults to `false` | +| `TASK_EVENT_CLEAN_STALE_INTERVAL` | Interval at which a background task does housekeeping by deleting stale (old) events from the database | Defaults to `8h` (8 hours), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | +| `TASK_EVENT_CLEAN_STALE_MAX_AGE` | Number defining at which age stale (old) events are deleted by the background task (_updatedAt_ attribute decides) | Defaults to `2190h` (2190 hours = 3 months), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | +| `TASK_PROMETHEUS_REFRESH_INTERVAL` | Interval at which a background task updates custom metrics | Defaults to `60s` (60 seconds), qualifier can be `s = second`, `m = minute`, `h = hour` prefixed with a positive number | +| | | | +| `LOCK_REDIS_ENABLED` | If locking via REDIS (multiple instances) is enabled. Requires REDIS. Otherwise uses in-memory locks. | Defaults to `false` | +| `LOCK_REDIS_URL` | If locking via REDIS is enabled, this should point to a resolvable REDIS instance, e.g. `redis://:@localhost:6379/`. | | +| | | | +| `PROMETHEUS_ENABLED` | If Prometheus metrics are exposed | Defaults to `false` | +| `PROMETHEUS_METRICS_PATH` | Defines the metrics endpoint path | Defaults to `/metrics` | +| `PROMETHEUS_SECURE_TOKEN_ENABLED` | If Prometheus metrics endpoint is protected by a token when enabled (**recommended**) | Defaults to `true` | +| `PROMETHEUS_SECURE_TOKEN` | The token securing the metrics endpoint when enabled (**recommended**) | Not set by default, you need to explicitly set it to a secure random | ## 3rd party integrations @@ -221,7 +222,7 @@ Custom exposed metrics are exposed under the `upda_` namespace. Examples: ```shell -# HELP upda_updates details for all updates, 0=pending, 1=approved, 2=ignored +# HELP upda_updates details for all updates, -1=deleted (deleted next restart), 0=pending, 1=approved, 2=ignored upda_updates{application="codeberg.org/forgejo/forgejo",host="myserver",provider="oci"} 0 upda_updates{application="docker.io/library/mysql",host="myserver",provider="oci"} 2 upda_updates{application="quay.io/navidys/prometheus-podman-exporter",host="myserver",provider="oci"} 1 diff --git a/_doc/upda-grafana-dashboard.json b/_doc/upda-grafana-dashboard.json index a0f7de8..b6abe69 100644 --- a/_doc/upda-grafana-dashboard.json +++ b/_doc/upda-grafana-dashboard.json @@ -2,6 +2,7 @@ "__inputs": [ { "name": "DS_PROMETHEUS", + "label": "datasource", "description": "", "type": "datasource", "pluginId": "prometheus", @@ -199,17 +200,23 @@ "options": { "0": { "color": "blue", - "index": 0, + "index": 1, "text": "pending" }, "1": { "color": "green", - "index": 1, + "index": 2, "text": "approved" }, "2": { "color": "red", - "index": 2 + "index": 3, + "text": "ignored" + }, + "-1": { + "color": "transparent", + "index": 0, + "text": "deleted" } }, "type": "value" @@ -429,10 +436,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -449,10 +452,10 @@ "id": 28, "links": [], "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -460,8 +463,7 @@ "fields": "", "values": false }, - "showUnfilled": true, - "valueMode": "color" + "textMode": "auto" }, "pluginVersion": "10.1.5", "targets": [ @@ -474,13 +476,13 @@ "expr": "promhttp_metric_handler_requests_total{job=\"$job\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "HTTP Status {{code}}", + "legendFormat": "{{code}}", "range": true, "refId": "A" } ], - "title": "Request Count", - "type": "bargauge" + "title": "Request Count per status code", + "type": "stat" }, { "datasource": { @@ -545,7 +547,7 @@ "expr": "upda_request_duration_count{job=\"$job\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{method}} {{path}}", + "legendFormat": "{{method}} | {{path}}", "range": true, "refId": "A" } @@ -570,14 +572,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -637,14 +635,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, @@ -704,10 +698,6 @@ { "color": "green", "value": null - }, - { - "color": "yellow", - "value": 1 } ] }, @@ -817,7 +807,7 @@ "expr": "upda_requests_total{job=\"$job\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 1, - "legendFormat": "{{method}} {{path}} | Http Status {{code}}", + "legendFormat": "{{code}} | {{method}} | {{path}}", "range": true, "refId": "A" } @@ -842,10 +832,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -909,10 +895,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -976,10 +958,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 1 } ] }, @@ -2218,6 +2196,6 @@ "timezone": "", "title": "upda", "uid": "CgCw8jKZ8", - "version": 19, + "version": 4, "weekStart": "" } \ No newline at end of file diff --git a/go.mod b/go.mod index 775643b..ba05272 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/go-co-op/gocron v1.37.0 github.com/go-co-op/gocron-redis-lock v1.3.0 github.com/go-playground/validator/v10 v10.16.0 + github.com/go-resty/resty/v2 v2.10.0 github.com/google/uuid v1.5.0 github.com/redis/go-redis/v9 v9.3.1 github.com/stretchr/testify v1.8.4 @@ -35,8 +36,7 @@ require ( github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-redsync/redsync/v4 v4.10.0 // indirect - github.com/go-resty/resty/v2 v2.10.0 // indirect + github.com/go-redsync/redsync/v4 v4.11.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 732715f..d7878d9 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/Depado/ginprom v1.8.0 h1:zaaibRLNI1dMiiuj1MKzatm8qrcHzikMlCc1anqOdyo= github.com/Depado/ginprom v1.8.0/go.mod h1:XBaKzeNBqPF4vxJpNLincSQZeMDnZp1tIbU0FU0UKgg= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= -github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= +github.com/Microsoft/hcsshim v0.11.1 h1:hJ3s7GbWlGK4YVV92sO88BQSyF4ZLVy7/awqOlPxFbA= +github.com/Microsoft/hcsshim v0.11.1/go.mod h1:nFJmaO4Zr5Y7eADdFOpYswDDlNVbvcIJJNJLECr5JQg= github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= @@ -33,8 +33,10 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= -github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= -github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= +github.com/containerd/containerd v1.7.7 h1:QOC2K4A42RQpcrZyptP6z9EJZnlHfHJUfZrAAHe15q4= +github.com/containerd/containerd v1.7.7/go.mod h1:3c4XZv6VeT9qgf9GMTxNTMFxGJrGpI2vz1yk4ye+YY8= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -47,8 +49,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= -github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -65,8 +67,6 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0= github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY= -github.com/go-co-op/gocron-redis-lock v1.2.0 h1:c5aGtxGxqWfln50Fdx9WpwYwtX7bK8i+pw3aIu2feao= -github.com/go-co-op/gocron-redis-lock v1.2.0/go.mod h1:En1WRsLSXsWiRul1GNyKiqniGe6UnrcEs9bOzDBya04= github.com/go-co-op/gocron-redis-lock v1.3.0 h1:PKwtuc/BhrDll/DxJfnXoW/+D1VXubd47xcGaB9pDuM= github.com/go-co-op/gocron-redis-lock v1.3.0/go.mod h1:9+H7ZfqVtJfx94uEAELwH+uHkn1UpM6lRM99wOBTGtg= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -85,8 +85,8 @@ github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1 github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= -github.com/go-redsync/redsync/v4 v4.10.0 h1:hTeAak4C73mNBQSTq6KCKDFaiIlfC+z5yTTl8fCJuBs= -github.com/go-redsync/redsync/v4 v4.10.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc= +github.com/go-redsync/redsync/v4 v4.11.0 h1:OPEcAxHBb95EzfwCKWM93ksOwHd5bTce2BD4+R14N6k= +github.com/go-redsync/redsync/v4 v4.11.0/go.mod h1:ZfayzutkgeBmEmBlUR3j+rF6kN44UUGtEdfzhBFZTPc= github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -151,8 +151,8 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= -github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -166,8 +166,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= -github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= +github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= @@ -188,8 +188,6 @@ github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdO github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= -github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/redis/go-redis/v9 v9.3.1 h1:KqdY8U+3X6z+iACvumCNxnoluToB+9Me+TvyFa21Mds= github.com/redis/go-redis/v9 v9.3.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= @@ -202,12 +200,12 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil/v3 v3.23.8 h1:xnATPiybo6GgdRoC4YoGnxXZFRc3dqQTGi73oLvvBrE= -github.com/shirou/gopsutil/v3 v3.23.8/go.mod h1:7hmCaBn+2ZwaZOr6jmPBZDfawwMGuo1id3C6aM8EDqQ= +github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E= +github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -221,10 +219,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= -github.com/testcontainers/testcontainers-go v0.25.0 h1:erH6cQjsaJrH+rJDU9qIf89KFdhK0Bft0aEZHlYC3Vs= -github.com/testcontainers/testcontainers-go v0.25.0/go.mod h1:4sC9SiJyzD1XFi59q8umTQYWxnkweEc5OjVtTUlJzqQ= -github.com/testcontainers/testcontainers-go/modules/redis v0.25.0 h1:Oml2QVZtDfLB8gosT7fRdWsWYSM8vihWKVSl7XZZtv0= -github.com/testcontainers/testcontainers-go/modules/redis v0.25.0/go.mod h1:xmCaWbWOQoWzhb5isnZw4GGy4aIaDjq8lSCXYFZz1ME= +github.com/testcontainers/testcontainers-go v0.26.0 h1:uqcYdoOHBy1ca7gKODfBd9uTHVK3a7UL848z09MVZ0c= +github.com/testcontainers/testcontainers-go v0.26.0/go.mod h1:ICriE9bLX5CLxL9OFQ2N+2N+f+803LNJ1utJb1+Inx0= +github.com/testcontainers/testcontainers-go/modules/redis v0.26.0 h1:GLN70++1KrLmFZWEvqqf8dnO6KzZ5ANg6lPUurR/n88= +github.com/testcontainers/testcontainers-go/modules/redis v0.26.0/go.mod h1:rEFRs/2LoFtRbHO/8c78rD8S0LwOOM6Kkygw1I/zdGQ= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -315,6 +313,7 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -330,8 +329,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= +google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= diff --git a/server/app.go b/server/app.go index 13498a5..2515d51 100644 --- a/server/app.go +++ b/server/app.go @@ -46,12 +46,14 @@ func Start() { webhookRepo := newWebhookDbRepo(env.db) eventRepo := newEventDbRepo(env.db) + lockService := newLockMemService() + eventService := newEventService(eventRepo) - updateService := newUpdateService(updateRepo, eventService) + updateService := newUpdateService(updateRepo, eventService, prometheusService) webhookService := newWebhookService(webhookRepo, env.webhookConfig, eventService) webhookInvocationService := newWebhookInvocationService(webhookService, updateService, env.webhookConfig) - taskService := newTaskService(updateService, eventService, webhookService, prometheusService, env.appConfig, env.taskConfig, env.prometheusConfig) + taskService := newTaskService(updateService, eventService, webhookService, lockService, prometheusService, env.appConfig, env.taskConfig, env.lockConfig, env.prometheusConfig) taskService.init() taskService.start() diff --git a/server/constants_env.go b/server/constants_env.go index 15ba733..0ace0dc 100644 --- a/server/constants_env.go +++ b/server/constants_env.go @@ -60,18 +60,18 @@ const ( envTaskUpdateCleanStaleEnabled = "TASK_UPDATE_CLEAN_STALE_ENABLED" envTaskUpdateCleanStaleInterval = "TASK_UPDATE_CLEAN_STALE_INTERVAL" envTaskUpdateCleanStaleMaxAge = "TASK_UPDATE_CLEAN_STALE_MAX_AGE" - taskUpdateCleanStaleEnabledDefault = "true" + taskUpdateCleanStaleEnabledDefault = "false" taskUpdateCleanStaleIntervalDefault = "1h" taskUpdateCleanStaleMaxAgeDefault = "168h" envTaskEventCleanStaleEnabled = "TASK_EVENT_CLEAN_STALE_ENABLED" envTaskEventCleanStaleInterval = "TASK_EVENT_CLEAN_STALE_INTERVAL" envTaskEventCleanStaleMaxAge = "TASK_EVENT_CLEAN_STALE_MAX_AGE" - taskEventCleanStaleEnabledDefault = "true" + taskEventCleanStaleEnabledDefault = "false" taskEventCleanStaleIntervalDefault = "8h" taskEventCleanStaleMaxAgeDefault = "2190h" - envTaskLockRedisEnabled = "TASK_LOCK_REDIS_ENABLED" - envTaskLockRedisUrl = "TASK_LOCK_REDIS_URL" - taskLockRedisEnabledDefault = "false" + envLockRedisEnabled = "LOCK_REDIS_ENABLED" + envLockRedisUrl = "LOCK_REDIS_URL" + redisEnabledDefault = "false" ) diff --git a/server/environment.go b/server/environment.go index 037c591..2d5aca7 100644 --- a/server/environment.go +++ b/server/environment.go @@ -44,8 +44,11 @@ type taskConfig struct { eventCleanStaleInterval string eventCleanStaleMaxAge time.Duration prometheusRefreshInterval string - lockRedisEnabled bool - lockRedisUrl string +} + +type lockConfig struct { + redisEnabled bool + redisUrl string } type webhookConfig struct { @@ -64,6 +67,7 @@ type Environment struct { authConfig *authConfig serverConfig *serverConfig taskConfig *taskConfig + lockConfig *lockConfig webhookConfig *webhookConfig prometheusConfig *prometheusConfig db *gorm.DB @@ -159,8 +163,12 @@ func bootstrapEnvironment() *Environment { eventCleanStaleInterval: os.Getenv(envTaskEventCleanStaleInterval), eventCleanStaleMaxAge: eventCleanStaleMaxAge, prometheusRefreshInterval: os.Getenv(envTaskPrometheusRefreshInterval), - lockRedisEnabled: os.Getenv(envTaskLockRedisEnabled) == "true", - lockRedisUrl: os.Getenv(envTaskLockRedisUrl), + } + + var lc *lockConfig + lc = &lockConfig{ + redisEnabled: os.Getenv(envLockRedisEnabled) == "true", + redisUrl: os.Getenv(envLockRedisUrl), } webhookTokenLength := 32 @@ -244,6 +252,7 @@ func bootstrapEnvironment() *Environment { authConfig: authConfig, serverConfig: sc, taskConfig: tc, + lockConfig: lc, webhookConfig: webhookConfig, prometheusConfig: prometheusConfig, db: db} @@ -270,6 +279,9 @@ func bootstrapFromEnvironmentAndValidate() { // webhook setEnvKeyDefault(envWebhooksTokenLength, webhooksTokenLengthDefault) + // lock + setEnvKeyDefault(envLockRedisEnabled, redisEnabledDefault) + // task setEnvKeyDefault(envTaskUpdateCleanStaleEnabled, taskUpdateCleanStaleEnabledDefault) setEnvKeyDefault(envTaskUpdateCleanStaleInterval, taskUpdateCleanStaleIntervalDefault) @@ -280,7 +292,6 @@ func bootstrapFromEnvironmentAndValidate() { setEnvKeyDefault(envTaskEventCleanStaleMaxAge, taskEventCleanStaleMaxAgeDefault) setEnvKeyDefault(envTaskPrometheusRefreshInterval, taskPrometheusRefreshDefault) - setEnvKeyDefault(envTaskLockRedisEnabled, taskLockRedisEnabledDefault) // prometheus setEnvKeyDefault(envPrometheusEnabled, prometheusEnabledDefault) diff --git a/server/service_lock.go b/server/service_lock.go new file mode 100644 index 0000000..9b7baf1 --- /dev/null +++ b/server/service_lock.go @@ -0,0 +1,9 @@ +package server + +type lockService interface { + init() error + tryLock(resource string) error + release(resource string) error + exists(resource string) bool + stop() +} diff --git a/server/service_lock_mem.go b/server/service_lock_mem.go new file mode 100644 index 0000000..b627fa6 --- /dev/null +++ b/server/service_lock_mem.go @@ -0,0 +1,52 @@ +package server + +import ( + "git.myservermanager.com/varakh/upda/util" + "go.uber.org/zap" +) + +type lockMemService struct { + registry *util.InMemoryLockRegistry +} + +func newLockMemService() lockService { + return &lockMemService{registry: util.NewInMemoryLockRegistry()} +} + +func (s *lockMemService) init() error { + zap.L().Info("Initialized in-memory locking service") + return nil +} + +func (s *lockMemService) tryLock(resource string) error { + if resource == "" { + return errorValidationNotBlank + } + + zap.L().Sugar().Debugf("Trying to lock '%s'", resource) + s.registry.Lock(resource) + zap.L().Sugar().Debugf("Locked '%s'", resource) + + return nil +} + +func (s *lockMemService) release(resource string) error { + if resource == "" { + return errorValidationNotBlank + } + + zap.L().Sugar().Debugf("Releasing lock '%s'", resource) + err := s.registry.Unlock(resource) + zap.L().Sugar().Debugf("Released lock '%s'", resource) + + return err +} + +func (s *lockMemService) exists(resource string) bool { + return s.registry.Exists(resource) +} + +func (s *lockMemService) stop() { + zap.L().Info("Clearing in-memory locking service") + s.registry.Clear() +} diff --git a/server/service_prometheus.go b/server/service_prometheus.go index 9dc556f..83154b8 100644 --- a/server/service_prometheus.go +++ b/server/service_prometheus.go @@ -22,6 +22,7 @@ func newPrometheusService(r *gin.Engine, c *prometheusConfig) *prometheusService ginprom.Namespace(Name), ginprom.Subsystem(""), ginprom.Path(c.path), + ginprom.Ignore(c.path), ginprom.Token(c.secureToken), ) } else { @@ -29,8 +30,8 @@ func newPrometheusService(r *gin.Engine, c *prometheusConfig) *prometheusService ginprom.Engine(r), ginprom.Namespace(Name), ginprom.Subsystem(""), + ginprom.Ignore(c.path), ginprom.Path(c.path), - ginprom.Token(c.secureToken), ) } } diff --git a/server/service_task.go b/server/service_task.go index e26f732..dd0b33c 100644 --- a/server/service_task.go +++ b/server/service_task.go @@ -13,14 +13,22 @@ type taskService struct { updateService *updateService eventService *eventService webhookService *webhookService + lockService lockService prometheusService *prometheusService appConfig *appConfig taskConfig *taskConfig + lockConfig *lockConfig prometheusConfig *prometheusConfig scheduler *gocron.Scheduler } -func newTaskService(u *updateService, e *eventService, w *webhookService, p *prometheusService, ac *appConfig, tc *taskConfig, pc *prometheusConfig) *taskService { +const ( + taskLockNameUpdatesCleanStale = "updates_clean_stale" + taskLockNameEventsCleanStale = "events_clean_stale" + taskLockNamePrometheusUpdate = "prometheus_update" +) + +func newTaskService(u *updateService, e *eventService, w *webhookService, l lockService, p *prometheusService, ac *appConfig, tc *taskConfig, lc *lockConfig, pc *prometheusConfig) *taskService { location, err := time.LoadLocation(ac.timeZone) if err != nil { @@ -33,12 +41,12 @@ func newTaskService(u *updateService, e *eventService, w *webhookService, p *pro scheduler := gocron.NewScheduler(location) - if tc.lockRedisEnabled { + if lc.redisEnabled { var redisOptions *redis.Options - redisOptions, err = redis.ParseURL(tc.lockRedisUrl) + redisOptions, err = redis.ParseURL(lc.redisUrl) if err != nil { - zap.L().Sugar().Fatalf("Cannot parse REDIS URL '%s' to set up locking. Reason: %s", tc.lockRedisUrl, err.Error()) + zap.L().Sugar().Fatalf("Cannot parse REDIS URL '%s' to set up locking. Reason: %s", lc.redisUrl, err.Error()) } redisClient := redis.NewClient(redisOptions) locker, err := redislock.NewRedisLocker(redisClient, redislock.WithTries(1)) @@ -52,9 +60,11 @@ func newTaskService(u *updateService, e *eventService, w *webhookService, p *pro updateService: u, eventService: e, webhookService: w, + lockService: l, prometheusService: p, appConfig: ac, taskConfig: tc, + lockConfig: lc, prometheusConfig: pc, scheduler: scheduler, } @@ -69,6 +79,7 @@ func (s *taskService) init() { func (s *taskService) stop() { zap.L().Sugar().Infof("Stopping %d periodic tasks...", len(s.scheduler.Jobs())) s.scheduler.Stop() + s.lockService.stop() zap.L().Info("Stopped all periodic tasks") } @@ -81,9 +92,27 @@ func (s *taskService) configureCleanupStaleUpdatesTask() { if !s.taskConfig.updateCleanStaleEnabled { return } - + initialDelay := time.Now().Add(10 * time.Second) _, err := s.scheduler.Every(s.taskConfig.updateCleanStaleInterval). + StartAt(initialDelay). Do(func() { + resource := taskLockNameUpdatesCleanStale + // distributed lock handled via gocron-redis-lock for tasks + if !s.lockConfig.redisEnabled { + // skip execution if lock already exists, wait otherwise + if lockExists := s.lockService.exists(resource); lockExists { + zap.L().Sugar().Debugf("Skipping task execution because task lock '%s' exists", resource) + return + } + _ = s.lockService.tryLock(resource) + defer func(lockService lockService, resource string) { + err := lockService.release(resource) + if err != nil { + zap.L().Sugar().Warnf("Could not release task lock '%s'", resource) + } + }(s.lockService, resource) + } + t := time.Now() t = t.Add(-s.taskConfig.updateCleanStaleMaxAge) @@ -112,8 +141,27 @@ func (s *taskService) configureCleanupStaleEventsTask() { return } + initialDelay := time.Now().Add(5 * time.Second) _, err := s.scheduler.Every(s.taskConfig.eventCleanStaleInterval). + StartAt(initialDelay). Do(func() { + resource := taskLockNameEventsCleanStale + // distributed lock handled via gocron-redis-lock for tasks + if !s.lockConfig.redisEnabled { + // skip execution if lock already exists, wait otherwise + if lockExists := s.lockService.exists(resource); lockExists { + zap.L().Sugar().Debugf("Skipping task execution because task lock '%s' exists", resource) + return + } + _ = s.lockService.tryLock(resource) + defer func(lockService lockService, resource string) { + err := lockService.release(resource) + if err != nil { + zap.L().Sugar().Warnf("Could not release task lock '%s'", resource) + } + }(s.lockService, resource) + } + t := time.Now() t = t.Add(-s.taskConfig.eventCleanStaleMaxAge) @@ -142,8 +190,27 @@ func (s *taskService) configurePrometheusRefreshTask() { return } + initialDelay := time.Now().Add(10 * time.Second) _, err := s.scheduler.Every(s.taskConfig.prometheusRefreshInterval). + StartAt(initialDelay). Do(func() { + resource := taskLockNamePrometheusUpdate + // distributed lock handled via gocron-redis-lock for tasks + if !s.lockConfig.redisEnabled { + // skip execution if lock already exists, wait otherwise + if lockExists := s.lockService.exists(resource); lockExists { + zap.L().Sugar().Debugf("Skipping task execution because task lock '%s' exists", resource) + return + } + _ = s.lockService.tryLock(resource) + defer func(lockService lockService, resource string) { + err := lockService.release(resource) + if err != nil { + zap.L().Sugar().Warnf("Could not release task lock '%s'", resource) + } + }(s.lockService, resource) + } + // updates with labels and collect stats about state updates, updatesError := s.updateService.getAll() diff --git a/server/service_update.go b/server/service_update.go index 1798202..93b0453 100644 --- a/server/service_update.go +++ b/server/service_update.go @@ -8,14 +8,16 @@ import ( ) type updateService struct { - repo updateRepository - eventService *eventService + repo updateRepository + eventService *eventService + prometheusService *prometheusService } -func newUpdateService(r updateRepository, e *eventService) *updateService { +func newUpdateService(r updateRepository, e *eventService, p *prometheusService) *updateService { return &updateService{ - repo: r, - eventService: e, + repo: r, + eventService: e, + prometheusService: p, } } @@ -116,6 +118,10 @@ func (s *updateService) delete(id string) error { s.eventService.createUpdateDeleted(e) + if err = s.prometheusService.setGauge(metricUpdates, []string{e.Application, e.Provider, e.Host}, -1); err != nil { + zap.L().Sugar().Errorf("Could not refresh updates prometheus metric for deleted update '%v'. Reason: %v", e.ID, err) + } + zap.L().Sugar().Infof("Deleted update '%v'", id) return nil } diff --git a/util/locker_memory.go b/util/locker_memory.go new file mode 100644 index 0000000..b48fc8f --- /dev/null +++ b/util/locker_memory.go @@ -0,0 +1,111 @@ +package util + +import "sync" + +import ( + "errors" + "sync/atomic" +) + +// ErrorNoSuchLock is returned when the requested lock does not exist +var ErrorNoSuchLock = errors.New("no such lock") + +// InMemoryLockRegistry provides a locking mechanism based on the passed in reference name +type InMemoryLockRegistry struct { + mu sync.Mutex + locks map[string]*lockCtr +} + +// lockCtr is used by InMemoryLockRegistry to represent a lock with a given name. +type lockCtr struct { + mu sync.Mutex + // waiters is the number of waiters waiting to acquire the lock + // this is int32 instead of uint32, so we can add `-1` in `dec()` + waiters int32 +} + +// inc increments the number of waiters waiting for the lock +func (l *lockCtr) inc() { + atomic.AddInt32(&l.waiters, 1) +} + +// dec decrements the number of waiters waiting on the lock +func (l *lockCtr) dec() { + atomic.AddInt32(&l.waiters, -1) +} + +// count gets the current number of waiters +func (l *lockCtr) count() int32 { + return atomic.LoadInt32(&l.waiters) +} + +// Lock locks the mutex +func (l *lockCtr) Lock() { + l.mu.Lock() +} + +// Unlock unlocks the mutex +func (l *lockCtr) Unlock() { + l.mu.Unlock() +} + +// NewInMemoryLockRegistry creates a new InMemoryLockRegistry +func NewInMemoryLockRegistry() *InMemoryLockRegistry { + return &InMemoryLockRegistry{ + locks: make(map[string]*lockCtr), + } +} + +// Clear clears all locks by initializing a new map +func (l *InMemoryLockRegistry) Clear() { + l.locks = make(map[string]*lockCtr) +} + +// Exists exists a lock by name +func (l *InMemoryLockRegistry) Exists(name string) bool { + _, exists := l.locks[name] + return exists +} + +// Lock locks a mutex with the given name. If it doesn't exist, one is created +func (l *InMemoryLockRegistry) Lock(name string) { + l.mu.Lock() + if l.locks == nil { + l.locks = make(map[string]*lockCtr) + } + + nameLock, exists := l.locks[name] + if !exists { + nameLock = &lockCtr{} + l.locks[name] = nameLock + } + + // increment the nameLock waiters while inside the main mutex + // this makes sure that the lock isn't deleted if `Lock` and `Unlock` are called concurrently + nameLock.inc() + l.mu.Unlock() + + // Lock the nameLock outside the main mutex, so we don't block other operations + // once locked then we can decrement the number of waiters for this lock + nameLock.Lock() + nameLock.dec() +} + +// Unlock unlocks the mutex with the given name +// If the given lock is not being waited on by any other callers, it is deleted +func (l *InMemoryLockRegistry) Unlock(name string) error { + l.mu.Lock() + nameLock, exists := l.locks[name] + if !exists { + l.mu.Unlock() + return ErrorNoSuchLock + } + + if nameLock.count() == 0 { + delete(l.locks, name) + } + nameLock.Unlock() + + l.mu.Unlock() + return nil +}