diff --git a/.golangci.yml b/.golangci.yml
index f6521d2ced4..84691a304a5 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -68,6 +68,7 @@ linters:
- error
- empty
- stdlib
+ - generic # allow generic types to be returned
- github.com/percona/pmm/admin/commands.Result
- github.com/percona/pmm/agent/runner/actions.Action
- github.com/percona/pmm/managed/services/telemetry.DataSource
diff --git a/.mockery.yaml b/.mockery.yaml
index 5b8e2ef3996..64bf0ef535d 100644
--- a/.mockery.yaml
+++ b/.mockery.yaml
@@ -9,6 +9,9 @@ filename: "mock_{{ .InterfaceName | snakecase }}_test.go"
mockname: "mock{{ .InterfaceName | camelcase | firstUpper }}"
packages:
# managed
+ github.com/percona/pmm/managed/services/agents:
+ interfaces:
+ Limiter:
github.com/percona/pmm/managed/services/backup:
interfaces:
agentService:
@@ -23,6 +26,10 @@ packages:
github.com/percona/pmm/managed/services/checks:
interfaces:
agentsRegistry:
+ github.com/percona/pmm/managed/services/grafana:
+ interfaces:
+ grafanaAuthUserGetter:
+ accessControl:
github.com/percona/pmm/managed/services/inventory:
interfaces:
agentService:
@@ -65,6 +72,7 @@ packages:
interfaces:
agentsRegistry:
agentsStateUpdater:
+ Limiter:
github.com/percona/pmm/managed/services/scheduler:
interfaces:
backupService:
diff --git a/build/ansible/roles/nginx/files/conf.d/pmm.conf b/build/ansible/roles/nginx/files/conf.d/pmm.conf
index 6ae7efbe7bc..a33d7bcf899 100644
--- a/build/ansible/roles/nginx/files/conf.d/pmm.conf
+++ b/build/ansible/roles/nginx/files/conf.d/pmm.conf
@@ -6,7 +6,7 @@
upstream managed-json {
server 127.0.0.1:7772;
keepalive 32;
- keepalive_requests 100;
+ keepalive_requests 10000;
keepalive_timeout 75s;
}
@@ -18,28 +18,76 @@
upstream qan-api-json {
server 127.0.0.1:9922;
keepalive 32;
- keepalive_requests 100;
+ keepalive_requests 10000;
keepalive_timeout 75s;
}
upstream vmproxy {
server 127.0.0.1:8430;
keepalive 32;
- keepalive_requests 100;
+ keepalive_requests 10000;
keepalive_timeout 75s;
}
+ upstream victoriametrics {
+ server 127.0.0.1:9090;
+ # Allocates a pool of idle TCP connections to the backend.
+ # It's bigger than for the rest of upstreams because it has more incoming traffic to handle.
+ keepalive 64;
+ # Prevent connection churn during high-throughput metric shipping.
+ # The NGINX default (100 or 1000) is insufficient for observability data streams.
+ # Increasing this prevents frequent socket teardowns.
+ keepalive_requests 100000;
+ # Ensure connections survive standard 30s-60s scrape intervals
+ keepalive_timeout 75s;
+}
+
+ upstream vmalert {
+ server 127.0.0.1:8880;
+ # Maintain a small pool of idle connections
+ keepalive 32;
+ # Keep connections alive across multiple scrapes/queries
+ keepalive_requests 10000;
+ keepalive_timeout 60s;
+}
+
upstream nomad-server-json {
server 127.0.0.1:4646;
keepalive 32;
- keepalive_requests 100;
+ keepalive_requests 10000;
keepalive_timeout 75s;
}
upstream grafana {
server 127.0.0.1:3000;
+ keepalive 32;
+ keepalive_requests 10000;
+ keepalive_timeout 75s;
}
+ # Sets up a 1MB memory zone named 'STATIC' and a max disk footprint of 1GB.
+ # It is used for static assets caching.
+ # Params:
+ # levels=1:2 - Specifies a subtree of folders to store cache files in.
+ # 1:2 means to create 1-character subdirectories and 2-character sub-subdirectories
+ # (e.g. /srv/nginx/cache/c/29/4f...). This prevents filesystem performance degradation
+ # due to thousands of files in a single folder.
+ #
+ # keys_zone=STATIC:1m - Allocates a 10 MB area in RAM called STATIC.
+ # Only cache keys (MD5 hashes) and metadata are stored in RAM.
+ # 1 MB holds ~8,000 keys.
+ #
+ # inactive=24h - If a cached file has not been accessed for 24 hours, NGINX deletes
+ # it from disk, regardless of the time-to-live (TTL) specified in proxy_cache_valid.
+ #
+ # max_size=1g - Upper limit of disk space for cache (1 GB). When this limit is reached,
+ # the cache manager process deletes the least used data (LRU algorithm).
+ proxy_cache_path /srv/nginx/cache/static levels=1:2 keys_zone=STATIC:1m inactive=24h max_size=1g;
+
+ # Sets up a 1MB memory zone named 'AUTH_CACHE' and a max disk footprint of 10M.
+ # It is used for auth response caching to 'push metrics' URI.
+ proxy_cache_path /srv/nginx/cache/auth levels=1:2 keys_zone=AUTH_CACHE:1m inactive=10m max_size=128m;
+
server {
listen 8080;
listen 8443 ssl;
@@ -49,6 +97,11 @@
absolute_redirect off;
+ # Low Latency: Accelerates the transfer of small amounts of data -
+ # REST/JSON API, gRPC, headers or WebSocket frames.
+ # Keep-Alive Optimization: Improves responsiveness when using reopened TCP connections.
+ tcp_nodelay on;
+
# allow huge requests
large_client_header_buffers 128 64k;
@@ -59,6 +112,11 @@
ssl_trusted_certificate /srv/nginx/ca-certs.pem;
ssl_dhparam /srv/nginx/dhparam.pem;
+ # Enable passing of the remote user's IP address to all
+ # proxied services using the X-Forwarded-For header
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ ############### MAINTENANCE BLOCK ###################
# this block checks for maintenance.html file and, if it exists, it redirects all requests to the maintenance page
# there are two exceptions for it /v1/updates/Status and /auth_request endpoints
set $maintenance_mode 0;
@@ -83,39 +141,60 @@
rewrite ^(.*)$ /maintenance.html break;
}
-
- # Enable passing of the remote user's IP address to all
- # proxied services using the X-Forwarded-For header
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ ############### AUTHENTICATION BLOCK #################
# Enable auth_request for all locations, including root
- # (but excluding /auth_request).
- auth_request /auth_request;
+ # (but excluding /auth_request_no_cache).
+ auth_request /auth_request_no_cache;
+
+ # Extract custom headers sent by the auth backend
+ # (e.g., X-Auth-Code, X-Auth-Error, X-Auth-Message).
+ # These headers are set by auth backend in case authentication failure (401, 403 status codes).
+ auth_request_set $auth_code $upstream_http_x_auth_code;
+ auth_request_set $auth_error $upstream_http_x_auth_error;
+ auth_request_set $auth_message $upstream_http_x_auth_message;
# Store the value of X-Proxy-Filter header of auth_request subrequest response in the variable.
auth_request_set $auth_request_proxy_filter $upstream_http_x_proxy_filter;
- proxy_set_header X-Proxy-Filter $auth_request_proxy_filter;
-
- # nginx completely ignores auth_request subrequest response body.
- # We use that directive to send the same request to the same location as a normal request
- # to get a response body or redirect and return it to the client.
- # 401 is re-run through /auth_request (below) to produce a response body. 403
- # (insufficient role) is served a static body by @access_denied without re-running:
- # the re-run is always a GET, which would wrongly pass method-specific rules.
- # proxy_intercept_errors is off, so backend API 403s pass through unchanged.
- error_page 401 = /auth_request;
+ proxy_set_header X-Proxy-Filter $auth_request_proxy_filter;
+
+ # 401 is handled by @auth_failed (below) to produce a response body.
+ # It construct JSON body based on values extracted from auth response headers
+ # (see above `auth_request_set $auth_code $upstream_http_x_auth_code;`)
+ # 403 (insufficient role) is served as static body by @access_denied.
+
+ error_page 401 = @auth_failed;
error_page 403 = @access_denied;
# Internal location for authentication via pmm-managed/Grafana.
- # First, nginx sends request there to authenticate it. If it is not authenticated by pmm-managed/Grafana,
- # it is sent to this location for the second time (as a normal request) by error_page directive above.
- location /auth_request {
+ # It doesn't use cache for request authentication results - a subrequest will be sent
+ # to auth backend for authentication for each incoming request that requies authentication.
+ location = /auth_request_no_cache {
internal;
-
auth_request off;
-
proxy_pass http://managed-json/auth_request;
+ # nginx always strips body from authentication subrequests.
+ # Overwrite Content-Length to avoid problems on Go side and to keep connection alive.
+ proxy_pass_request_body off;
+ proxy_set_header Content-Length 0;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+
+ # Those headers are set for both subrequest and normal request.
+ proxy_set_header X-Original-Uri $request_uri;
+ proxy_set_header X-Original-Method $request_method;
+ proxy_buffering off;
+ }
+
+ # Internal location for authentication via pmm-managed/Grafana.
+ # It uses cache for request authentication results. It incoming request matches the
+ # Cache entry key - cached response will be used immidiatly (if it exists), otherwise
+ # a subrequest to auth backend is sent and it's response is cached.
+ location = /auth_request_cached {
+ internal;
+ auth_request off;
+ proxy_pass http://managed-json/auth_request;
# nginx always strips body from authentication subrequests.
# Overwrite Content-Length to avoid problems on Go side and to keep connection alive.
proxy_pass_request_body off;
@@ -127,6 +206,35 @@
# Those headers are set for both subrequest and normal request.
proxy_set_header X-Original-Uri $request_uri;
proxy_set_header X-Original-Method $request_method;
+ proxy_buffering on;
+
+ # Auth responses cache set up.
+ proxy_cache AUTH_CACHE;
+ # If the client request method is listed in this directive then the response will be cached.
+ # “GET” and “HEAD” methods are always added to the list,
+ # though it is recommended to specify them explicitly.
+ proxy_cache_methods GET HEAD POST;
+ # Cache entry key = bearer/basic auth header|original request method|original request URL path.
+ # Same pair reuses one cached auth result.
+ proxy_cache_key "$http_authorization|$request_method|$request_uri";
+ # Successful auth responses are cached for 5 minutes.
+ proxy_cache_valid 200 5m;
+ # Denied auth responses are cached for 15 seconds
+ # (negative caching to reduce auth backend load).
+ proxy_cache_valid 401 403 30s;
+ # NGINX ignores upstream cache-control and cookie instructions,
+ # forcing cache behavior from local config.
+ # NOTE: our pmm-managed doesn't set any of those headers,
+ # but this is a good practice to have it here.
+ proxy_ignore_headers Cache-Control Expires Set-Cookie;
+ }
+
+ location @auth_failed {
+ auth_request off;
+ default_type application/json;
+
+ # Construct JSON payload using the extracted variables
+ return 401 '{"code": $auth_code, "error": "$auth_error", "message": "$auth_message"}';
}
# Static body matching pmm-managed's PermissionDenied response (see error_page 403 above).
@@ -136,13 +244,66 @@
return 403 '{"code":7,"error":"Access denied","message":"Access denied"}';
}
+ ############### PMM UI BLOCK ########################
+
# PMM UI
- location /pmm-ui {
+ rewrite ^/pmm-ui$ /pmm-ui/;
+ location ^~ /pmm-ui/ {
# Will redirect on FE to login page if user is not authenticated
auth_request off;
- alias /usr/share/pmm-ui;
+ alias /usr/share/pmm-ui/;
try_files $uri /index.html break;
+
+ # Optimizations for local static file serving
+ # Bypass user-space buffers, copy directly between file descriptors
+ sendfile on;
+ # Send HTTP response headers in the same packet as the file data
+ tcp_nopush on;
+ # Disable Nagle's algorithm for faster small packet delivery
+ tcp_nodelay on;
+ }
+
+ # All PMM UI assets are dynamic - bypass authentication and cache on browser side.
+ location ^~ /pmm-ui/assets/ {
+ auth_request off;
+
+ alias /usr/share/pmm-ui/assets/;
+ try_files $uri =404;
+ # Add caching headers to further reduce container load
+ expires 30d;
+ add_header Cache-Control "public, max-age=2592000, immutable" always;
+
+ add_header X-Cache-Status "LOCAL-FILE" always;
+
+ # open_file_cache stores file metadata in RAM, rather than their contents.
+ # This significantly reduces the number of open(), stat(), and close() system calls
+ # in the operating system when distributing static files, which reduces the load
+ # on the CPU and disk subsystem.
+ #
+ # Params:
+ # max=10000 - The maximum number of items (handles) in the cache. When the cache fills up,
+ # NGINX removes the least used items using the LRU (Least Recently Used) algorithm.
+ # inactive=60s - The amount of time an item remains in the cache if it has not been accessed.
+ # After this time, the item is removed (default 60s).
+ open_file_cache max=10000 inactive=60s;
+ # The time interval after which NGINX will check at the OS level whether the file
+ # has changed and whether it still exists (default 60s).
+ open_file_cache_valid 60s;
+ # The minimum number of file requests during the inactive time required to keep
+ # the handle open in the cache (default 1).
+ open_file_cache_min_uses 2;
+ # Enables or disables caching of file search error information
+ # (allows to avoid constant disk requests for files that do not exist).
+ open_file_cache_errors on;
+
+ # Optimizations for local static file serving.
+ # Bypass user-space buffers, copy directly between file descriptors
+ sendfile on;
+ # Send HTTP response headers in the same packet as the file data
+ tcp_nopush on;
+ # Disable Nagle's algorithm for faster small packet delivery
+ tcp_nodelay on;
}
# Grafana
@@ -150,17 +311,48 @@
auth_request off;
return 302 /graph/;
}
+
rewrite ^/graph$ /graph/;
- location /graph {
+ location ^~ /graph/ {
proxy_cookie_path / "/;";
proxy_pass http://grafana;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
proxy_read_timeout 600;
proxy_set_header Host $http_host;
proxy_set_header X-Proxy-Filter $auth_request_proxy_filter;
}
+ # Grafana Static Assets - shall bypass authentication.
+ location ^~ /graph/public/ {
+ auth_request off;
+ proxy_pass http://grafana;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+ proxy_read_timeout 600;
+ proxy_set_header Host $http_host;
+ # Add caching headers to further reduce container load
+ expires 30d;
+ add_header Cache-Control "public, no-transform";
+
+ # Enable caching using the 'STATIC' zone.
+ proxy_cache STATIC;
+ # Cache successful responses for 24 hours
+ proxy_cache_valid 200 24h;
+ # Add a header to debug cache HITs or MISSes.
+ add_header X-Cache-Status $upstream_cache_status;
+
+ # Optimizations for local static file serving
+ # Bypass user-space buffers, copy directly between file descriptors
+ sendfile on;
+ # Send HTTP response headers in the same packet as the file data
+ tcp_nopush on;
+ # Disable Nagle's algorithm for faster small packet delivery
+ tcp_nodelay on;
+ }
+
# See https://grafana.com/tutorials/run-grafana-behind-a-proxy/
- location /graph/api/live/ {
+ location ^~ /graph/api/live/ {
proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
@@ -173,45 +365,68 @@
return 301 /graph/dashboard/snapshots;
}
- # Prometheus
- location /prometheus {
- proxy_pass http://127.0.0.1:9090;
+ ############### METRICS AND ALERTS BLOCK ##############
+
+ location ^~ /prometheus {
+ proxy_pass http://victoriametrics;
proxy_read_timeout 600;
proxy_http_version 1.1;
proxy_set_header Connection "";
+
+ # Disable body size limits for large remote_write payloads
+ client_max_body_size 0;
+ client_body_buffer_size 10m;
}
- location /prometheus/api/v1 {
+
+ location ^~ /prometheus/api/v1 {
proxy_pass http://vmproxy;
proxy_read_timeout 600;
proxy_http_version 1.1;
proxy_set_header Connection "";
+ proxy_buffering off;
}
# VictoriaMetrics
- location /victoriametrics/ {
+ location ^~ /victoriametrics/ {
proxy_pass http://vmproxy/;
proxy_read_timeout 600;
proxy_http_version 1.1;
proxy_set_header Connection "";
client_body_buffer_size 10m;
+ proxy_buffering off;
+ }
+
+ # This URI is used by vm-agent to push metrics.
+ # This is most heavily used URI and it requires auth cache usage to decrease the
+ # load on auch backend.
+ location = /victoriametrics/api/v1/write {
+ auth_request /auth_request_cached;
+ proxy_pass http://victoriametrics/prometheus/api/v1/write;
+ proxy_read_timeout 600;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+ client_body_buffer_size 10m;
+ proxy_buffering off;
}
# VMAlert
- location /prometheus/rules {
- proxy_pass http://127.0.0.1:8880/api/v1/rules;
+ location ^~ /prometheus/rules {
+ proxy_pass http://vmalert/api/v1/rules;
proxy_read_timeout 600;
proxy_http_version 1.1;
proxy_set_header Connection "";
+ proxy_buffering off;
}
- location /prometheus/alerts {
- proxy_pass http://127.0.0.1:8880/api/v1/alerts;
+ location ^~ /prometheus/alerts {
+ proxy_pass http://vmalert/api/v1/alerts;
proxy_read_timeout 600;
proxy_http_version 1.1;
proxy_set_header Connection "";
+ proxy_buffering off;
}
- # Nomad
- location /nomad/ {
+ ############### NOMAD BLOCK ########################
+ location ^~ /nomad/ {
rewrite /nomad/(.*) /$1 break;
proxy_pass https://nomad-server-json;
@@ -229,33 +444,37 @@
# will be rejected by Nomad. It must be rewritten to be the
# host address instead.
proxy_set_header Origin "${scheme}://${proxy_host}";
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
}
+ ############### SWAGGER BLOCK ######################
# Swagger UI
rewrite ^/swagger/swagger.json$ /swagger.json permanent;
rewrite ^/swagger/(.*)$ /swagger permanent;
- location /swagger {
+ location ^~ /swagger {
root /usr/share/pmm-managed/swagger;
- try_files $uri /index.html break;
+ try_files $uri /index.html =404;
}
+ ############### API BLOCK ##########################
# pmm-managed gRPC APIs
- location /agent. {
+ location ^~ /agent. {
grpc_pass grpc://managed-grpc;
# Disable request body size check for gRPC streaming, see https://trac.nginx.org/nginx/ticket/1642.
# pmm-managed uses grpc.MaxRecvMsgSize for that.
client_max_body_size 0;
}
- location /inventory. {
+ location ^~ /inventory. {
grpc_pass grpc://managed-grpc;
}
- location /management. {
+ location ^~ /management. {
grpc_pass grpc://managed-grpc;
}
- location /server. {
+ location ^~ /server. {
grpc_pass grpc://managed-grpc;
}
- location /realtimeanalytics. {
+ location ^~ /realtimeanalytics. {
grpc_pass grpc://managed-grpc;
# Disable request body size check for gRPC streaming, see https://trac.nginx.org/nginx/ticket/1642.
# pmm-managed uses grpc.MaxRecvMsgSize for that.
@@ -267,54 +486,63 @@
proxy_pass http://managed-json/v1/;
proxy_http_version 1.1;
proxy_set_header Connection "";
+ proxy_buffering off;
}
# qan-api gRPC APIs should not be exposed
# qan-api JSON APIs
- location /v1/qan {
+ location ^~ /v1/qan {
proxy_pass http://qan-api-json/v1/qan;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header X-Proxy-Filter $auth_request_proxy_filter;
+ proxy_buffering off;
}
- # compatibility with PMM 1.x
- rewrite ^/ping$ /v1/server/readyz;
# compatibility with PMM 2.x
- rewrite ^/v1/readyz$ /v1/server/readyz;
rewrite ^/v1/version$ /v1/server/version;
- rewrite ^/logs.zip$ /v1/server/logs.zip;
+ rewrite ^/logs.zip$ /v1/server/logs.zip;
# logs.zip in both PMM 1.x and 2.x variants
- location /v1/server/logs.zip {
+ location = /v1/server/logs.zip {
proxy_pass http://managed-json;
proxy_http_version 1.1;
proxy_set_header Connection "";
+ proxy_buffering off;
}
+ ############### FS BLOCK ###########################
# pmm-dump artifacts
- location /dump {
+ location ^~ /dump/ {
alias /srv/dump/;
+ try_files $uri =404;
+
+ # Optimizations for local static file serving
+ # Bypass user-space buffers, copy directly between file descriptors
+ sendfile on;
+ # Send HTTP response headers in the same packet as the file data
+ tcp_nopush on;
+ # Disable Nagle's algorithm for faster small packet delivery
+ tcp_nodelay on;
}
- # This location stores static content for general pmm-server purposes.
- # Ex.: local-rss.xml - contains Percona's news when no internet connection.
- location /pmm-static {
- auth_request off;
- alias /usr/share/pmm-server/static;
- }
+ ############ UNPROTECTED ENDPOINTS BLOCK ##############
+ # Must be available without authentication for health checking
- # proxy requests to the Percona's blog feed
- # fallback to local rss if pmm-server is isolated from internet.
- # https://jira.percona.com/browse/PMM-6153
- location = /percona-blog/feed {
- auth_request off;
- proxy_ssl_server_name on;
+ # compatibility with PMM 1.x
+ rewrite ^/ping$ /v1/server/readyz;
+ # compatibility with PMM 2.x
+ rewrite ^/v1/readyz$ /v1/server/readyz;
- set $feed https://www.percona.com/blog/feed/;
- proxy_pass $feed;
- proxy_set_header User-Agent "$http_user_agent pmm-server/3.x";
- error_page 500 502 503 504 /pmm-static/local-rss.xml;
+ # Unprotected Health Checks and Version Info
+ location ~ ^/v1/server/(readyz|leaderHealthCheck)$ {
+ # Matches /v1/server/readyz, /v1/server/leaderHealthCheck
+ auth_request off;
+ proxy_pass http://managed-json;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+ proxy_buffering off;
}
+
}
diff --git a/build/ansible/roles/nginx/files/nginx.conf b/build/ansible/roles/nginx/files/nginx.conf
index e3ec684fa02..66c961f7ae6 100644
--- a/build/ansible/roles/nginx/files/nginx.conf
+++ b/build/ansible/roles/nginx/files/nginx.conf
@@ -9,6 +9,11 @@ pid /run/nginx.pid;
events {
worker_connections 4096;
+ # Tells the worker process to accept as many new connections as possible
+ # in a single pass after receiving a notification of a new connection.
+ # If disabled, the process accepts only one new connection at a time.
+ # Enabling improves throughput and reduces latency during traffic spikes.
+ multi_accept on;
}
http {
diff --git a/build/docker/server/entrypoint.sh b/build/docker/server/entrypoint.sh
index 737cc2db37a..5e95646586e 100755
--- a/build/docker/server/entrypoint.sh
+++ b/build/docker/server/entrypoint.sh
@@ -133,6 +133,9 @@ unset PLUGINS_SRC PLUGINS_DST PLUGINS_MARKER BUNDLED_VERSION SYNCED_VERSION
echo "Creating nginx temp directories..."
mkdir -p /srv/nginx/tmp/{client,proxy,fastcgi,uwsgi,scgi}
+echo "Creating nginx cache directories..."
+mkdir -p /srv/nginx/cache/{static,auth}
+
if [ ! -d "/srv/pmm-agent/tmp" ]; then
echo "Creating pmm-agent temp directory..."
install -d -m 770 /srv/pmm-agent/tmp
diff --git a/dashboards/dashboards/PMM Health/PMM_Health.json b/dashboards/dashboards/PMM Health/PMM_Health.json
index 0f04f162cde..3d28ef0b7e5 100644
--- a/dashboards/dashboards/PMM Health/PMM_Health.json
+++ b/dashboards/dashboards/PMM Health/PMM_Health.json
@@ -2189,7 +2189,7 @@
"h": 3,
"w": 6,
"x": 0,
- "y": 3
+ "y": 223
},
"id": 1071,
"options": {
@@ -2252,7 +2252,7 @@
"h": 3,
"w": 6,
"x": 6,
- "y": 3
+ "y": 223
},
"id": 1080,
"options": {
@@ -2315,7 +2315,7 @@
"h": 3,
"w": 4,
"x": 12,
- "y": 3
+ "y": 223
},
"id": 1078,
"options": {
@@ -2382,7 +2382,7 @@
"h": 3,
"w": 4,
"x": 16,
- "y": 3
+ "y": 223
},
"id": 1082,
"options": {
@@ -2446,7 +2446,7 @@
"h": 3,
"w": 4,
"x": 20,
- "y": 3
+ "y": 223
},
"id": 1081,
"options": {
@@ -2544,7 +2544,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 84
+ "y": 226
},
"id": 1072,
"options": {
@@ -2645,7 +2645,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 84
+ "y": 226
},
"id": 1058,
"options": {
@@ -2756,7 +2756,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 92
+ "y": 234
},
"id": 1075,
"options": {
@@ -2849,7 +2849,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 92
+ "y": 234
},
"id": 1077,
"options": {
@@ -4365,6 +4365,9 @@
},
"insertNulls": false,
"lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
@@ -4372,7 +4375,7 @@
},
"showPoints": "auto",
"showValues": false,
- "spanNulls": 60000,
+ "spanNulls": true,
"stacking": {
"group": "A",
"mode": "normal"
@@ -4903,7 +4906,7 @@
},
{
"datasource": "Metrics",
- "description": "Heap and allocation throughput.\n\ngo_memstats_mallocs_total - Total number of heap objects allocated, both live and gc-ed.\n\ngo_memstats_frees_total - Total number of heap objects frees.",
+ "description": "Heap objects allocation throughput.\n\ngo_memstats_mallocs_total - Total number of heap objects allocated, both live and gc-ed.\n\ngo_memstats_frees_total - Total number of heap objects frees.",
"fieldConfig": {
"defaults": {
"color": {
@@ -5031,7 +5034,7 @@
"refId": "C"
}
],
- "title": "Heap Allocations",
+ "title": "Heap Objects Allocations",
"transparent": true,
"type": "timeseries"
},
@@ -5132,7 +5135,7 @@
"sort": "none"
}
},
- "pluginVersion": "12.4.3",
+ "pluginVersion": "12.4.5",
"targets": [
{
"editorMode": "code",
@@ -5250,7 +5253,7 @@
"sort": "none"
}
},
- "pluginVersion": "12.4.3",
+ "pluginVersion": "12.4.5",
"targets": [
{
"expr": "go_goroutines{job=\"pmm-managed\"}",
@@ -5271,7 +5274,7 @@
},
{
"datasource": "Metrics",
- "description": "go_sql_connections_max_open - Maximum number of open connections to the database.\n\ngo_sql_connections_in_use - The number of connections currently in use.\n\nWait rate.\n\ngo_sql_connections_wait_duration_seconds - The total time blocked waiting for a new connection. If the result is 1.0, it means that at any given moment, exactly one goroutine is completely blocked waiting for a DB connection. If it spikes to 5.0, five goroutines are perpetually blocked. It should be 0 ideally.",
+ "description": "SQL connections used for internal system staff (configurations, periodic jobs, etc).\n\ngo_sql_connections_max_open - Maximum number of open connections to the database.\n\ngo_sql_connections_in_use - The number of connections currently in use.\n\nWait rate.\n\ngo_sql_connections_wait_duration_seconds - The total time blocked waiting for a new connection. If the result is 1.0, it means that at any given moment, exactly one goroutine is completely blocked waiting for a DB connection. If it spikes to 5.0, five goroutines are perpetually blocked. It should be 0 ideally.",
"fieldConfig": {
"defaults": {
"color": {
@@ -5370,16 +5373,31 @@
"id": "custom.lineStyle"
}
]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "max"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "green",
+ "mode": "fixed"
+ }
+ }
+ ]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
- "x": 12,
+ "x": 0,
"y": 74
},
- "id": 1123,
+ "id": 1129,
"options": {
"legend": {
"calcs": [],
@@ -5393,11 +5411,11 @@
"sort": "none"
}
},
- "pluginVersion": "12.4.3",
+ "pluginVersion": "12.4.5",
"targets": [
{
"editorMode": "code",
- "expr": "go_sql_connections_max_open{job=\"pmm-managed\", db=\"pmm-managed\"}",
+ "expr": "go_sql_connections_max_open{job=\"pmm-managed\", db=\"pmm-managed/internal\"}",
"legendFormat": "max",
"range": true,
"refId": "A"
@@ -5409,7 +5427,7 @@
},
"editorMode": "code",
"exemplar": false,
- "expr": "go_sql_connections_in_use{job=\"pmm-managed\", db=\"pmm-managed\"}",
+ "expr": "go_sql_connections_in_use{job=\"pmm-managed\", db=\"pmm-managed/internal\"}",
"instant": false,
"legendFormat": "in-use",
"range": true,
@@ -5422,1034 +5440,1199 @@
},
"editorMode": "code",
"exemplar": false,
- "expr": "rate(go_sql_connections_wait_duration_seconds{db=\"pmm-managed\",driver=\"postgres\"}[$__interval])",
+ "expr": "rate(go_sql_connections_wait_duration_seconds{db=\"pmm-managed/internal\",driver=\"postgres\"}[$__interval])",
"instant": false,
"legendFormat": "blocked go-routines",
"range": true,
"refId": "C"
}
],
- "title": "SQL connections pool",
+ "title": "Internal SQL connections pool",
"transparent": true,
"type": "timeseries"
},
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 82
- },
- "id": 1033,
- "panels": [],
- "title": "Grafana",
- "type": "row"
- },
{
"datasource": "Metrics",
- "description": "Is Grafana UP and running",
+ "description": "SQL connections used for handling:\n- connections and events from PMM Agents.\n- incoming gRPC/REST requests.\n\ngo_sql_connections_max_open - Maximum number of open connections to the database.\n\ngo_sql_connections_in_use - The number of connections currently in use.\n\nWait rate.\n\ngo_sql_connections_wait_duration_seconds - The total time blocked waiting for a new connection. If the result is 1.0, it means that at any given moment, exactly one goroutine is completely blocked waiting for a DB connection. If it spikes to 5.0, five goroutines are perpetually blocked. It should be 0 ideally.",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
},
- "mappings": [
- {
- "options": {
- "0": {
- "color": "red",
- "index": 1,
- "text": "DOWN"
- },
- "1": {
- "color": "green",
- "index": 0,
- "text": "UP"
- }
- },
- "type": "value"
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "series",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "dash": [
+ 10,
+ 10
+ ],
+ "fill": "dash"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": 60000,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
- ],
+ },
+ "decimals": 0,
+ "mappings": [],
+ "min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
- "color": "red",
+ "color": "green",
"value": 0
},
{
- "color": "green",
- "value": 1
+ "color": "red",
+ "value": 80
}
]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 6,
- "x": 0,
- "y": 83
- },
- "id": 1059,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "12.4.3",
- "targets": [
- {
- "datasource": "Metrics",
- "editorMode": "code",
- "expr": "up{job=\"grafana\"}",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Grafana Status",
- "type": "stat"
- },
- {
- "datasource": "Metrics",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
},
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
+ "unit": "short"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "blocked go-routines"
+ },
+ "properties": [
{
- "color": "green",
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "unit",
+ "value": "short"
+ },
+ {
+ "id": "min",
"value": 0
},
{
- "color": "red",
- "value": 80
+ "id": "custom.lineStyle"
}
]
},
- "unit": "none"
- },
- "overrides": []
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "in-use"
+ },
+ "properties": [
+ {
+ "id": "custom.lineStyle"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 3,
- "w": 6,
- "x": 6,
- "y": 83
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 74
},
- "id": 1047,
+ "id": 1123,
"options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
},
- "pluginVersion": "12.4.3",
+ "pluginVersion": "12.4.5",
"targets": [
{
- "datasource": "Metrics",
"editorMode": "code",
- "expr": "rate(grafana_database_conn_open[$__interval])",
- "instant": false,
- "legendFormat": "open conn",
+ "expr": "go_sql_connections_max_open{job=\"pmm-managed\", db=\"pmm-managed/api\"}",
+ "legendFormat": "max",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PA58DA793C7250F1B"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "go_sql_connections_in_use{job=\"pmm-managed\", db=\"pmm-managed/api\"}",
+ "instant": false,
+ "legendFormat": "in-use",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PA58DA793C7250F1B"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "rate(go_sql_connections_wait_duration_seconds{db=\"pmm-managed/api\",driver=\"postgres\"}[$__interval])",
+ "instant": false,
+ "legendFormat": "blocked go-routines",
+ "range": true,
+ "refId": "C"
}
],
- "title": "DB Open Connections",
- "type": "stat"
+ "title": "API SQL connections pool",
+ "transparent": true,
+ "type": "timeseries"
},
{
- "datasource": "Metrics",
- "description": "total amount of orgs",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
+ "collapsed": true,
"gridPos": {
- "h": 3,
- "w": 3,
- "x": 12,
- "y": 83
- },
- "id": 1049,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 82
},
- "pluginVersion": "12.4.3",
- "targets": [
+ "id": 1033,
+ "panels": [
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "grafana_stat_total_orgs{instance=\"$instance\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Organisations Count",
- "type": "stat"
- },
- {
- "datasource": "Metrics",
- "description": "total amount of users",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
+ "description": "Is Grafana UP and running",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
},
- {
- "color": "red",
- "value": 80
+ "mappings": [
+ {
+ "options": {
+ "0": {
+ "color": "red",
+ "index": 1,
+ "text": "DOWN"
+ },
+ "1": {
+ "color": "green",
+ "index": 0,
+ "text": "UP"
+ }
+ },
+ "type": "value"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "red",
+ "value": 0
+ },
+ {
+ "color": "green",
+ "value": 1
+ }
+ ]
}
- ]
+ },
+ "overrides": []
},
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 15,
- "y": 83
- },
- "id": 1029,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "last"
+ "gridPos": {
+ "h": 3,
+ "w": 6,
+ "x": 0,
+ "y": 437
+ },
+ "id": 1059,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "up{job=\"grafana\"}",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
],
- "fields": "",
- "values": false
+ "title": "Grafana Status",
+ "type": "stat"
},
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "12.4.3",
- "targets": [
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "grafana_stat_total_users",
- "format": "time_series",
- "intervalFactor": 2,
- "range": true,
- "refId": "A"
- }
- ],
- "title": "User Count",
- "type": "stat"
- },
- {
- "datasource": "Metrics",
- "description": "total amount of folders",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
},
- "mappings": [
+ "gridPos": {
+ "h": 3,
+ "w": 6,
+ "x": 6,
+ "y": 437
+ },
+ "id": 1047,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "12.4.5",
+ "targets": [
{
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
- },
- "type": "special"
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "rate(grafana_database_conn_open[$__interval])",
+ "instant": false,
+ "legendFormat": "open conn",
+ "range": true,
+ "refId": "A"
}
],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
+ "title": "DB Open Connections",
+ "type": "stat"
},
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 18,
- "y": 83
- },
- "id": 1051,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "12.4.3",
- "targets": [
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "grafana_stat_totals_folder{instance=\"$instance\"}",
- "format": "time_series",
- "intervalFactor": 2,
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Folders Count",
- "type": "stat"
- },
- {
- "datasource": "Metrics",
- "description": "total amount of dashboards",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [
- {
- "options": {
- "match": "null",
- "result": {
- "text": "N/A"
- }
+ "description": "total amount of orgs",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
},
- "type": "special"
- }
- ],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ }
+ ]
},
- {
- "color": "red",
- "value": 80
- }
- ]
+ "unit": "none"
+ },
+ "overrides": []
},
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 3,
- "w": 3,
- "x": 21,
- "y": 83
- },
- "id": 1031,
- "maxDataPoints": 100,
- "options": {
- "colorMode": "none",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "horizontal",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 12,
+ "y": 437
+ },
+ "id": 1049,
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "grafana_stat_total_orgs{instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "range": true,
+ "refId": "A"
+ }
],
- "fields": "",
- "values": false
+ "title": "Organisations Count",
+ "type": "stat"
},
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "12.4.3",
- "targets": [
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "grafana_stat_totals_dashboard",
- "format": "time_series",
- "intervalFactor": 2,
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Dashboard Count",
- "type": "stat"
- },
- {
- "datasource": "Metrics",
- "description": "Average user and system CPU time spent in seconds.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "showValues": false,
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
+ "description": "total amount of users",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
},
- "thresholdsStyle": {
- "mode": "off"
- }
+ "overrides": []
},
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
- },
- {
- "color": "red",
- "value": 80
- }
- ]
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 15,
+ "y": 437
},
- "unit": "s"
+ "id": 1029,
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "grafana_stat_total_users",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "User Count",
+ "type": "stat"
},
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 86
- },
- "id": 1039,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "12.4.3",
- "targets": [
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "avg(rate(process_cpu_seconds_total{job=\"grafana\"}[$__interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "CPU Time",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Average CPU Usage",
- "type": "timeseries"
- },
- {
- "datasource": "Metrics",
- "description": "Virtual and Resident memory size in bytes, averages over 5 min interval",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "showValues": false,
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
+ "description": "total amount of folders",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
},
- "thresholdsStyle": {
- "mode": "off"
- }
+ "overrides": []
},
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
- },
- {
- "color": "red",
- "value": 80
- }
- ]
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 18,
+ "y": 437
},
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 86
- },
- "id": 1037,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
+ "id": 1051,
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "grafana_stat_totals_folder{instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "range": true,
+ "refId": "A"
+ }
],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "title": "Folders Count",
+ "type": "stat"
},
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "12.4.3",
- "targets": [
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "avg(rate(process_resident_memory_bytes{job=\"grafana\"}[$__interval]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Resident Mem",
- "range": true,
- "refId": "A"
+ "description": "total amount of dashboards",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 3,
+ "x": 21,
+ "y": 437
+ },
+ "id": 1031,
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "grafana_stat_totals_dashboard",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Dashboard Count",
+ "type": "stat"
},
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "avg(rate(process_virtual_memory_bytes{job=\"grafana\"}[5m]))",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Virtual Mem",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Average Memory Usage",
- "type": "timeseries"
- },
- {
- "datasource": "Metrics",
- "description": "http response status",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
+ "description": "Average user and system CPU time spent in seconds.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
},
- "showPoints": "never",
- "showValues": false,
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 458
+ },
+ "id": 1039,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
- "thresholdsStyle": {
- "mode": "off"
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
}
},
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 94
- },
- "id": 1041,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "avg(rate(process_cpu_seconds_total{job=\"grafana\"}[$__interval]))",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "CPU Time",
+ "range": true,
+ "refId": "A"
+ }
],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "12.4.5",
- "targets": [
- {
- "datasource": "Metrics",
- "editorMode": "code",
- "expr": "rate(grafana_api_response_status_total[$__interval])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "API {{ code }}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": "Metrics",
- "editorMode": "code",
- "expr": "rate(grafana_page_response_status_total[$__interval])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Page {{ code }}",
- "range": true,
- "refId": "B"
+ "title": "Average CPU Usage",
+ "type": "timeseries"
},
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "rate(grafana_proxy_response_status_total[$__interval])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Proxy {{ code }}",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "Total Response Statuses",
- "type": "timeseries"
- },
- {
- "datasource": "Metrics",
- "description": "api login counters",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
+ "description": "Virtual and Resident memory size in bytes, averages over 5 min interval",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
},
- "showPoints": "never",
- "showValues": false,
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 458
+ },
+ "id": 1037,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
- "thresholdsStyle": {
- "mode": "off"
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
}
},
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 94
- },
- "id": 1043,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "lastNotNull",
- "max",
- "min"
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "avg(rate(process_resident_memory_bytes{job=\"grafana\"}[$__interval]))",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Resident Mem",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "avg(rate(process_virtual_memory_bytes{job=\"grafana\"}[5m]))",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Virtual Mem",
+ "range": true,
+ "refId": "B"
+ }
],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "title": "Average Memory Usage",
+ "type": "timeseries"
},
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "12.4.5",
- "targets": [
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "rate(grafana_api_login_post_total[$__interval])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "POST",
- "range": true,
- "refId": "A"
+ "description": "http response status",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 466
+ },
+ "id": 1041,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "rate(grafana_api_response_status_total[$__interval])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "API {{ code }}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "rate(grafana_page_response_status_total[$__interval])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Page {{ code }}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "rate(grafana_proxy_response_status_total[$__interval])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "Proxy {{ code }}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Total Response Statuses",
+ "type": "timeseries"
},
{
"datasource": "Metrics",
- "editorMode": "code",
- "expr": "rate(grafana_api_login_oauth_total[$__interval])",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "OAuth",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Login Events",
- "type": "timeseries"
- },
- {
- "datasource": "Metrics",
- "description": "go_sql_connections_max_open - Maximum number of open connections to the database.\n\ngo_sql_connections_in_use - The number of connections currently in use.\n\nWait rate.\n\ngo_sql_connections_wait_duration_seconds - The total time blocked waiting for a new connection. If the result is 1.0, it means that at any given moment, exactly one goroutine is completely blocked waiting for a DB connection. If it spikes to 5.0, five goroutines are perpetually blocked. It should be 0 ideally.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "series",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
+ "description": "api login counters",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
},
- "showPoints": "auto",
- "showValues": false,
- "spanNulls": 60000,
- "stacking": {
- "group": "A",
- "mode": "none"
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 466
+ },
+ "id": 1043,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "lastNotNull",
+ "max",
+ "min"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
},
- "thresholdsStyle": {
- "mode": "off"
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
}
},
- "decimals": 0,
- "mappings": [],
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "rate(grafana_api_login_post_total[$__interval])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "POST",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": "Metrics",
+ "editorMode": "code",
+ "expr": "rate(grafana_api_login_oauth_total[$__interval])",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "legendFormat": "OAuth",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Login Events",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "Metrics",
+ "description": "go_sql_connections_max_open - Maximum number of open connections to the database.\n\ngo_sql_connections_in_use - The number of connections currently in use.\n\nWait rate.\n\ngo_sql_connections_wait_duration_seconds - The total time blocked waiting for a new connection. If the result is 1.0, it means that at any given moment, exactly one goroutine is completely blocked waiting for a DB connection. If it spikes to 5.0, five goroutines are perpetually blocked. It should be 0 ideally.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "series",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": 60000,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "decimals": 0,
+ "mappings": [],
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
},
+ "unit": "short"
+ },
+ "overrides": [
{
- "color": "red",
- "value": 80
+ "matcher": {
+ "id": "byName",
+ "options": "max"
+ },
+ "properties": [
+ {
+ "id": "custom.lineStyle",
+ "value": {
+ "dash": [
+ 10,
+ 10
+ ],
+ "fill": "dash"
+ }
+ }
+ ]
}
]
},
- "unit": "short"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "max"
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 474
+ },
+ "id": 1127,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
},
- "properties": [
- {
- "id": "custom.lineStyle",
- "value": {
- "dash": [
- 10,
- 10
- ],
- "fill": "dash"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 102
- },
- "id": 1127,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.4.5",
- "targets": [
- {
- "editorMode": "code",
- "expr": "go_sql_stats_connections_max_open{job=\"grafana\", db_name=\"grafana\"}",
- "legendFormat": "max",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "PA58DA793C7250F1B"
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
},
- "editorMode": "code",
- "exemplar": false,
- "expr": "go_sql_stats_connections_in_use{job=\"grafana\",db_name=\"grafana\"}",
- "instant": false,
- "legendFormat": "in-use",
- "range": true,
- "refId": "B"
+ "pluginVersion": "12.4.5",
+ "targets": [
+ {
+ "editorMode": "code",
+ "expr": "go_sql_stats_connections_max_open{job=\"grafana\", db_name=\"grafana\"}",
+ "legendFormat": "max",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PA58DA793C7250F1B"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "go_sql_stats_connections_in_use{job=\"grafana\",db_name=\"grafana\"}",
+ "instant": false,
+ "legendFormat": "in-use",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "SQL connections pool",
+ "transparent": true,
+ "type": "timeseries"
}
],
- "title": "SQL connections pool",
- "transparent": true,
- "type": "timeseries"
+ "title": "Grafana",
+ "type": "row"
},
{
"collapsed": true,
@@ -6457,7 +6640,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 110
+ "y": 83
},
"id": 1009,
"panels": [
@@ -6506,7 +6689,7 @@
"h": 3,
"w": 5,
"x": 0,
- "y": 230
+ "y": 84
},
"id": 1011,
"maxDataPoints": 100,
@@ -6591,7 +6774,7 @@
"h": 3,
"w": 5,
"x": 5,
- "y": 230
+ "y": 84
},
"id": 1013,
"maxDataPoints": 100,
@@ -6684,7 +6867,7 @@
"h": 3,
"w": 5,
"x": 10,
- "y": 230
+ "y": 84
},
"id": 1015,
"maxDataPoints": 100,
@@ -6766,7 +6949,7 @@
"h": 3,
"w": 4,
"x": 15,
- "y": 230
+ "y": 84
},
"id": 1017,
"maxDataPoints": 100,
@@ -6848,7 +7031,7 @@
"h": 3,
"w": 5,
"x": 19,
- "y": 230
+ "y": 84
},
"id": 1019,
"maxDataPoints": 100,
@@ -6948,13 +7131,38 @@
},
"unit": "ops"
},
- "overrides": []
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "/api/v1/write"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": true,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
- "y": 233
+ "y": 87
},
"id": 1021,
"options": {
@@ -7056,7 +7264,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 233
+ "y": 87
},
"id": 1023,
"options": {
@@ -7156,7 +7364,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 241
+ "y": 95
},
"id": 1025,
"options": {
@@ -7257,7 +7465,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 241
+ "y": 95
},
"id": 1027,
"links": [
@@ -7288,12 +7496,14 @@
"targets": [
{
"datasource": "Metrics",
+ "editorMode": "code",
"expr": "vm_cache_entries{job=\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}",
"format": "time_series",
"hide": true,
"interval": "$interval",
"intervalFactor": 1,
"legendFormat": "Time Series",
+ "range": true,
"refId": "A"
},
{
@@ -7317,7 +7527,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 111
+ "y": 84
},
"id": 1084,
"panels": [
@@ -7366,7 +7576,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 231
+ "y": 681
},
"id": 1094,
"options": {
@@ -7445,7 +7655,7 @@
"h": 3,
"w": 4,
"x": 4,
- "y": 231
+ "y": 681
},
"id": 1095,
"options": {
@@ -7524,7 +7734,7 @@
"h": 3,
"w": 4,
"x": 8,
- "y": 231
+ "y": 681
},
"id": 1096,
"options": {
@@ -7604,7 +7814,7 @@
"h": 3,
"w": 4,
"x": 12,
- "y": 231
+ "y": 681
},
"id": 1099,
"options": {
@@ -7684,7 +7894,7 @@
"h": 3,
"w": 4,
"x": 16,
- "y": 231
+ "y": 681
},
"id": 1098,
"options": {
@@ -7764,7 +7974,7 @@
"h": 3,
"w": 4,
"x": 20,
- "y": 231
+ "y": 681
},
"id": 1097,
"options": {
@@ -7862,7 +8072,7 @@
"h": 5,
"w": 12,
"x": 0,
- "y": 234
+ "y": 684
},
"id": 1101,
"maxDataPoints": 200,
@@ -7960,7 +8170,7 @@
"h": 5,
"w": 12,
"x": 12,
- "y": 234
+ "y": 684
},
"id": 1103,
"maxDataPoints": 200,
@@ -8068,7 +8278,7 @@
"h": 5,
"w": 12,
"x": 0,
- "y": 239
+ "y": 689
},
"id": 1105,
"maxDataPoints": 200,
@@ -8177,7 +8387,7 @@
"h": 5,
"w": 12,
"x": 12,
- "y": 239
+ "y": 689
},
"id": 1107,
"maxDataPoints": 200,
@@ -8221,7 +8431,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 112
+ "y": 85
},
"id": 1007,
"panels": [
@@ -8280,7 +8490,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 263
+ "y": 713
},
"id": 63,
"maxDataPoints": 100,
@@ -8328,7 +8538,7 @@
"h": 3,
"w": 5,
"x": 4,
- "y": 263
+ "y": 713
},
"id": 1001,
"options": {
@@ -8369,7 +8579,7 @@
"h": 3,
"w": 5,
"x": 9,
- "y": 263
+ "y": 713
},
"id": 65,
"options": {
@@ -8445,7 +8655,7 @@
"h": 3,
"w": 5,
"x": 14,
- "y": 263
+ "y": 713
},
"id": 67,
"links": [
@@ -8532,7 +8742,7 @@
"h": 3,
"w": 5,
"x": 19,
- "y": 263
+ "y": 713
},
"id": 69,
"links": [
@@ -8632,7 +8842,7 @@
"h": 3,
"w": 4,
"x": 0,
- "y": 266
+ "y": 716
},
"id": 85,
"links": [
@@ -8721,7 +8931,7 @@
"h": 3,
"w": 5,
"x": 4,
- "y": 266
+ "y": 716
},
"id": 86,
"maxDataPoints": 100,
@@ -8804,7 +9014,7 @@
"h": 3,
"w": 5,
"x": 9,
- "y": 266
+ "y": 716
},
"id": 1005,
"links": [
@@ -8892,7 +9102,7 @@
"h": 3,
"w": 5,
"x": 14,
- "y": 266
+ "y": 716
},
"id": 70,
"links": [
@@ -8980,7 +9190,7 @@
"h": 3,
"w": 5,
"x": 19,
- "y": 266
+ "y": 716
},
"id": 68,
"links": [
@@ -9137,7 +9347,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 269
+ "y": 719
},
"id": 23,
"options": {
@@ -9293,7 +9503,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 269
+ "y": 719
},
"id": 34,
"options": {
@@ -9403,7 +9613,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 277
+ "y": 727
},
"id": 36,
"options": {
@@ -9540,7 +9750,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 277
+ "y": 727
},
"id": 26,
"options": {
@@ -9732,6 +9942,6 @@
"timezone": "",
"title": "PMM Health",
"uid": "pmm-health",
- "version": 6,
+ "version": 4,
"weekStart": ""
}
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index f74e9f2eb3c..9dc31015f43 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -7,6 +7,14 @@ services:
platform: linux/amd64
container_name: pmm-server
hostname: pmm-server
+# deploy:
+# resources:
+# limits:
+# cpus: '6'
+# memory: 16G
+# reservations:
+# cpus: '2'
+# memory: 16G
env_file:
- .env
# enable for delve
diff --git a/go.mod b/go.mod
index 1cb367c8e88..71076506862 100644
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.10.0
github.com/ClickHouse/clickhouse-go/v2 v2.47.0
github.com/DATA-DOG/go-sqlmock v1.5.2
+ github.com/KimMachineGun/automemlimit v0.7.5
github.com/alecthomas/kingpin/v2 v2.4.0
github.com/alecthomas/kong v1.16.0
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
@@ -267,6 +268,7 @@ require (
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/paulmach/orb v0.13.0 // indirect
+ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pelletier/go-toml/v2 v2.3.1 // indirect
github.com/petermattis/goid v0.0.0-20260330135022-df67b199bc81 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
diff --git a/go.sum b/go.sum
index 09c030665cf..ba47a43755b 100644
--- a/go.sum
+++ b/go.sum
@@ -82,6 +82,8 @@ github.com/ClickHouse/clickhouse-go/v2 v2.47.0/go.mod h1:sPj7C7UYQ2MWHcfX+4eGN6n
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk=
+github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE=
@@ -657,6 +659,8 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw=
github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go
index 34fe6c3ec98..e4fc99528ff 100644
--- a/managed/cmd/pmm-managed/main.go
+++ b/managed/cmd/pmm-managed/main.go
@@ -30,6 +30,7 @@ import (
"net/url"
"os"
"os/signal"
+ "runtime"
"sort"
"strconv"
"strings"
@@ -37,6 +38,8 @@ import (
"time"
_ "github.com/ClickHouse/clickhouse-go/v2"
+ // By default, it sets `GOMEMLIMIT` to 90% of cgroup's memory limit.
+ _ "github.com/KimMachineGun/automemlimit"
"github.com/alecthomas/kingpin/v2"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/validator"
@@ -112,6 +115,7 @@ import (
platformClient "github.com/percona/pmm/managed/utils/platform"
pmmerrors "github.com/percona/pmm/utils/errors"
"github.com/percona/pmm/utils/logger"
+ "github.com/percona/pmm/utils/rateLimiter"
"github.com/percona/pmm/utils/sqlmetrics"
"github.com/percona/pmm/version"
)
@@ -142,10 +146,41 @@ const (
distributionInfoFilePath = "/srv/pmm-distribution"
osInfoFilePath = "/proc/version"
+
+ // DB-related consts.
+ dbMaxLifeTime = 0
+ dbMaxIdleTime = 5 * time.Minute
+
+ internalDbMinOpenConns = 20
+ apiDbMinOpenConns = 50
+
+ // Per-P settings keep pool growth proportional to scheduler parallelism.
+ internalDbOpenConnsPerP = 5
+ apiDbOpenConnsPerP = 12
+)
+
+var (
+ // Internal DB params.
+ internalDbMaxOpenConns = max(internalDbMinOpenConns, runtime.GOMAXPROCS(0)*internalDbOpenConnsPerP)
+ internalDbMaxIdleConns = internalDbMaxOpenConns
+
+ // API DB params.
+ // Sized to give DB-bound auth/role/settings paths enough headroom during
+ // a reconnect storm from a fleet of agents, while staying well within
+ // Postgres max_connections (set to 2000 by PMM Server).
+ apiDbMaxOpenConns = max(apiDbMinOpenConns, runtime.GOMAXPROCS(0)*apiDbOpenConnsPerP)
+ apiDbMaxIdleConns = apiDbMaxOpenConns
)
var pprofSemaphore = semaphore.NewWeighted(1)
+// pmmAgentsConnectionLimiter is used to limit the number of concurrent
+// connection attempts from pmm-agents to the API server.
+// Each connection attempt uses a database connection(s), so we limit the number
+// of concurrent connections to avoid exhausting the database connection pool and
+// to prevent the system from degrading during a thundering herd of connection attempts.
+var pmmAgentsConnectionsLimiter = rateLimiter.NewConcurrencyLimiter(int32(apiDbMaxOpenConns * 7 / 10)) //nolint:mnd
+
func addLogsHandler(mux *http.ServeMux, logs *server.Logs) {
l := logrus.WithField("component", "logs.zip")
@@ -326,7 +361,8 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) {
// Register RTA service with in-memory store
rtaStore := realtimeanalytics.NewStore()
- rtaSvc := realtimeanalytics.NewService(deps.db, deps.agentsRegistry, deps.agentsStateUpdater, rtaStore)
+ rtaSvc := realtimeanalytics.NewService(deps.db, deps.agentsRegistry,
+ deps.agentsStateUpdater, rtaStore, pmmAgentsConnectionsLimiter)
rtav1.RegisterRealtimeAnalyticsServiceServer(gRPCServer, rtaSvc)
rtav1.RegisterCollectorServiceServer(gRPCServer, rtaSvc)
@@ -536,7 +572,7 @@ func runDebugServer(ctx context.Context) {
}
type setupDeps struct {
- sqlDB *sql.DB
+ db *reform.DB
ha *ha.Service
supervisord *supervisord.Service
vmdb *victoriametrics.Service
@@ -547,9 +583,6 @@ type setupDeps struct {
// setup performs setup tasks that depend on database.
func setup(ctx context.Context, deps *setupDeps) bool {
- l := reform.NewPrintfLogger(deps.l.Debugf)
- db := reform.NewDB(deps.sqlDB, postgresql.Dialect, l)
-
// log and ignore validation errors; fail on other errors
deps.l.Infof("Updating settings...")
env := os.Environ()
@@ -566,7 +599,7 @@ func setup(ctx context.Context, deps *setupDeps) bool {
}
deps.l.Infof("Updating supervisord configuration...")
- settings, err := models.GetSettings(db.Querier)
+ settings, err := models.GetSettings(deps.db.WithContext(ctx))
if err != nil {
deps.l.Warnf("Failed to get settings: %s.", err)
return false
@@ -870,45 +903,91 @@ func main() { //nolint:gocognit,maintidx,cyclop
l.Panicf("cannot load victoriametrics params problem: %+v", err)
}
- setupParams := models.SetupDBParams{
- Address: *postgresAddrF,
- Name: *postgresDBNameF,
- Username: *postgresDBUsernameF,
- Password: *postgresDBPasswordF,
- SSLMode: *postgresSSLModeF,
- SSLCAPath: *postgresSSLCAPathF,
- SSLKeyPath: *postgresSSLKeyPathF,
- SSLCertPath: *postgresSSLCertPathF,
- HANodeID: *haNodeID,
- HAPeers: nodes,
+ // These params are used for setting up access to the PG database.
+ // This DB connections pool is used by internal services only and MUST NOT be used
+ // by gRPC/REST API handlers. So that if there is a thundering herd from clients
+ // and they occupy all connections from SQL pool - the internal system's serviceses
+ // are still able to communicate with DB and perform tasks to keep system alive
+ // (like update caches, fetch settings, run cleanup tasks, etc).
+ setupInternalDBParams := models.SetupDBParams{
+ Address: *postgresAddrF,
+ Name: *postgresDBNameF,
+ Username: *postgresDBUsernameF,
+ Password: *postgresDBPasswordF,
+ SSLMode: *postgresSSLModeF,
+ SSLCAPath: *postgresSSLCAPathF,
+ SSLKeyPath: *postgresSSLKeyPathF,
+ SSLCertPath: *postgresSSLCertPathF,
+ HANodeID: *haNodeID,
+ HAPeers: nodes,
+ ConnMaxLifetime: dbMaxLifeTime,
+ ConnMaxIdleTime: dbMaxIdleTime,
+ MaxIdleConns: internalDbMaxIdleConns,
+ MaxOpenConns: internalDbMaxOpenConns,
}
- sqlDB, err := models.OpenDB(setupParams)
+ sqlInternalDB, err := models.OpenDB(setupInternalDBParams)
if err != nil {
l.Panicf("Failed to connect to database: %+v", err)
}
- defer sqlDB.Close() //nolint:errcheck
+ defer func() {
+ _ = sqlInternalDB.Close()
+ }()
+
+ migrateDB(ctx, sqlInternalDB, setupInternalDBParams)
+
+ prom.MustRegister(sqlmetrics.NewCollector("postgres", *postgresDBNameF+"/internal", sqlInternalDB))
+ internalReformL := sqlmetrics.NewReform("postgres", *postgresDBNameF+"/internal", logrus.WithField("component", "reform").Tracef)
+ prom.MustRegister(internalReformL)
+ internalDB := reform.NewDB(sqlInternalDB, postgresql.Dialect, internalReformL)
+
+ // These params are used for setting up access to the PG database.
+ // This DB connections pool is used by gRPC/REST API handlers.
+ // So that it doesn't interfere with internal DB connections pool and allow the system
+ // to operate even if the API clients occupy all connections from the API DB pool.
+ setupAPIDBParams := models.SetupDBParams{
+ Address: *postgresAddrF,
+ Name: *postgresDBNameF,
+ Username: *postgresDBUsernameF,
+ Password: *postgresDBPasswordF,
+ SSLMode: *postgresSSLModeF,
+ SSLCAPath: *postgresSSLCAPathF,
+ SSLKeyPath: *postgresSSLKeyPathF,
+ SSLCertPath: *postgresSSLCertPathF,
+ HANodeID: *haNodeID,
+ HAPeers: nodes,
+ ConnMaxLifetime: dbMaxLifeTime,
+ ConnMaxIdleTime: dbMaxIdleTime,
+ MaxIdleConns: apiDbMaxIdleConns,
+ MaxOpenConns: apiDbMaxOpenConns,
+ }
+
+ sqlAPIDB, err := models.OpenDB(setupAPIDBParams)
+ if err != nil {
+ l.Panicf("Failed to connect to database: %+v", err)
+ }
+ defer func() {
+ _ = sqlAPIDB.Close()
+ }()
if *haEnabled {
models.AgentConfigFilePath = "/srv/pmm-agent/config/pmm-agent.yaml"
}
- migrateDB(ctx, sqlDB, setupParams)
-
- prom.MustRegister(sqlmetrics.NewCollector("postgres", *postgresDBNameF, sqlDB))
- reformL := sqlmetrics.NewReform("postgres", *postgresDBNameF, logrus.WithField("component", "reform").Tracef)
- prom.MustRegister(reformL)
- db := reform.NewDB(sqlDB, postgresql.Dialect, reformL)
+ prom.MustRegister(sqlmetrics.NewCollector("postgres", *postgresDBNameF+"/api", sqlAPIDB))
+ apiReformL := sqlmetrics.NewReform("postgres", *postgresDBNameF+"/api", logrus.WithField("component", "reform").Tracef)
+ prom.MustRegister(apiReformL)
+ apiDB := reform.NewDB(sqlAPIDB, postgresql.Dialect, apiReformL)
// Generate unique PMM Server ID if it's not already.
- err = models.SetPMMServerID(db)
+ err = models.SetPMMServerID(internalDB)
if err != nil {
l.Panicf("failed to set PMM Server ID")
}
- cleaner := clean.New(db)
+ cleaner := clean.New(internalDB)
externalRules := vmalert.NewExternalRules()
- vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, vmParams, chParams, haService)
+ vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, internalDB, vmParams, chParams, haService)
if err != nil {
l.Panicf("VictoriaMetrics service problem: %+v", err)
}
@@ -921,9 +1000,9 @@ func main() { //nolint:gocognit,maintidx,cyclop
minioClient := minio.New()
- qanClient := getQANClient(sqlDB, *postgresDBNameF, *qanAPIAddrF)
+ qanClient := getQANClient(sqlAPIDB, *postgresDBNameF, *qanAPIAddrF)
- agentsRegistry := agents.NewRegistry(db, vmParams, haService)
+ agentsRegistry := agents.NewRegistry(apiDB, vmParams, haService)
// TODO remove once PMM cluster is Active-Active
// TODO kick non-pmm-server agents only
@@ -933,11 +1012,11 @@ func main() { //nolint:gocognit,maintidx,cyclop
// func() { agentsRegistry.KickAll(ctx) }))
pbmPITRService := backup.NewPBMPITRService()
- backupRemovalService := backup.NewRemovalService(db, pbmPITRService)
- backupRetentionService := backup.NewRetentionService(db, backupRemovalService)
+ backupRemovalService := backup.NewRemovalService(internalDB, pbmPITRService)
+ backupRetentionService := backup.NewRetentionService(internalDB, backupRemovalService)
prom.MustRegister(agentsRegistry)
- inventoryMetrics := inventory.NewInventoryMetrics(db, agentsRegistry)
+ inventoryMetrics := inventory.NewInventoryMetrics(internalDB, agentsRegistry)
inventoryMetricsCollector := inventory.NewInventoryMetricsCollector(inventoryMetrics)
prom.MustRegister(inventoryMetricsCollector)
@@ -947,7 +1026,7 @@ func main() { //nolint:gocognit,maintidx,cyclop
connectionCheck := agents.NewConnectionChecker(agentsRegistry)
serviceInfoBroker := agents.NewServiceInfoBroker(agentsRegistry)
- updater := server.NewUpdater(db)
+ updater := server.NewUpdater(internalDB)
logs := server.NewLogs(version.FullInfo(), updater, vmParams)
@@ -983,7 +1062,7 @@ func main() { //nolint:gocognit,maintidx,cyclop
platformClient := platformClient.NewClient(platformAddress)
dus := distribution.NewService(distributionInfoFilePath, osInfoFilePath, l)
- telemetry, err := telemetry.NewService(db, platformClient, version.Version, dus, cfg.Config.Services.Telemetry)
+ telemetry, err := telemetry.NewService(internalDB, platformClient, version.Version, dus, cfg.Config.Services.Telemetry)
if err != nil {
l.Fatalf("Could not create telemetry service: %s", err)
}
@@ -998,14 +1077,16 @@ func main() { //nolint:gocognit,maintidx,cyclop
GCMaxAllocs: *nomadGCMaxAllocsF,
GCParallelDestroys: *nomadGCParallelDestroysF,
}
- nomad, err := nomad.New(db, nomadClientConfig)
+ nomad, err := nomad.New(internalDB, nomadClientConfig)
if err != nil {
l.Fatalf("Could not create Nomad client: %s", err)
}
- jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService)
- agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams, nomad)
- agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService)
+ jobsService := agents.NewJobsService(internalDB, agentsRegistry, backupRetentionService)
+ agentsStateUpdater := agents.NewStateUpdater(apiDB, agentsRegistry, vmdb, vmParams, nomad)
+ // Agents service handles pmm-agent <-> pmm-server communication logic.
+ // Shall use apiDB connection pool.
+ agentsHandler := agents.NewHandler(apiDB, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService, pmmAgentsConnectionsLimiter)
actionsService := agents.NewActionsService(qanClient, agentsRegistry)
@@ -1014,16 +1095,19 @@ func main() { //nolint:gocognit,maintidx,cyclop
l.Fatalf("Could not create Victoria Metrics client: %s", err)
}
- clickhouseClient, err := newClickhouseDB(qanDB.DSN, clickhouseMaxIdleConns, clickhouseMaxOpenConns)
+ clickhouseDB, err := newClickhouseDB(qanDB.DSN, clickhouseMaxIdleConns, clickhouseMaxOpenConns)
if err != nil {
l.Fatalf("Could not create Clickhouse client: %s", err)
}
- externalExporterStatusSvc := agents.NewExternalExporterStatusService(db, v1.NewAPI(vmClient))
+ defer func() {
+ _ = clickhouseDB.Close()
+ }()
+ externalExporterStatusSvc := agents.NewExternalExporterStatusService(internalDB, v1.NewAPI(vmClient))
- checksService := checks.New(db, actionsService, v1.NewAPI(vmClient), clickhouseClient)
+ checksService := checks.New(internalDB, actionsService, v1.NewAPI(vmClient), clickhouseDB)
prom.MustRegister(checksService)
- alertingService, err := alerting.NewService(db, grafanaClient)
+ alertingService, err := alerting.NewService(internalDB, grafanaClient)
if err != nil {
l.Fatalf("Could not create alerting service: %s", err)
}
@@ -1032,21 +1116,21 @@ func main() { //nolint:gocognit,maintidx,cyclop
agentService := agents.NewAgentService(agentsRegistry)
versioner := agents.NewVersionerService(agentsRegistry)
- compatibilityService := backup.NewCompatibilityService(db, versioner)
- backupService := backup.NewService(db, jobsService, agentService, compatibilityService, pbmPITRService)
- backupMetricsCollector := backup.NewMetricsCollector(db)
+ compatibilityService := backup.NewCompatibilityService(internalDB, versioner)
+ backupService := backup.NewService(internalDB, jobsService, agentService, compatibilityService, pbmPITRService)
+ backupMetricsCollector := backup.NewMetricsCollector(internalDB)
prom.MustRegister(backupMetricsCollector)
- schedulerService := scheduler.New(db, backupService)
- versionCache := versioncache.New(db, versioner)
+ schedulerService := scheduler.New(internalDB, backupService)
+ versionCache := versioncache.New(internalDB, versioner)
- dumpService := dump.New(db, &dump.URLs{
+ dumpService := dump.New(internalDB, &dump.URLs{
ClickhouseURL: chParams.URL().String(),
VMURL: *victoriaMetricsURLF,
})
serverParams := &server.Params{
- DB: db,
+ DB: internalDB,
VMDB: vmdb,
VMAlert: vmalert,
AgentsStateUpdater: agentsStateUpdater,
@@ -1095,7 +1179,7 @@ func main() { //nolint:gocognit,maintidx,cyclop
// try synchronously once, then retry in the background
deps := &setupDeps{
- sqlDB: sqlDB,
+ db: internalDB,
ha: haService,
supervisord: supervisord,
vmdb: vmdb,
@@ -1123,21 +1207,17 @@ func main() { //nolint:gocognit,maintidx,cyclop
}()
}
- settings, err := models.GetSettings(sqlDB)
+ settings, err := models.GetSettings(sqlInternalDB)
if err != nil {
l.Fatalf("Failed to get settings: %+v.", err)
}
- authServer := grafana.NewAuthServer(grafanaClient, db)
+ authServer := grafana.NewAuthServer(ctx, grafanaClient, apiDB)
prom.MustRegister(authServer)
l.Info("Starting services...")
var wg sync.WaitGroup
- wg.Go(func() {
- authServer.Run(ctx)
- })
-
wg.Go(func() {
vmalert.Run(ctx)
})
@@ -1192,7 +1272,7 @@ func main() { //nolint:gocognit,maintidx,cyclop
compatibilityService: compatibilityService,
config: &cfg.Config,
connectionCheck: connectionCheck,
- db: db,
+ db: apiDB,
dumpService: dumpService,
grafanaClient: grafanaClient,
handler: agentsHandler,
diff --git a/managed/cmd/pmm-managed/packages.dot b/managed/cmd/pmm-managed/packages.dot
index fdcfc42bf83..18e3c5d5702 100644
--- a/managed/cmd/pmm-managed/packages.dot
+++ b/managed/cmd/pmm-managed/packages.dot
@@ -1,5 +1,6 @@
digraph packages {
"/cmd/pmm-managed-init" -> "/models";
+ "/cmd/pmm-managed-init" -> "/services/clickhouse";
"/cmd/pmm-managed-init" -> "/services/supervisord";
"/cmd/pmm-managed-starlark" -> "/pi/check";
"/cmd/pmm-managed-starlark" -> "/pi/starlark";
diff --git a/managed/models/database.go b/managed/models/database.go
index fa839fa3be0..437dc858a00 100644
--- a/managed/models/database.go
+++ b/managed/models/database.go
@@ -1222,13 +1222,10 @@ func OpenDB(params SetupDBParams) (*sql.DB, error) {
return nil, fmt.Errorf("failed to create a connection pool to PostgreSQL: %w", err)
}
- db.SetConnMaxLifetime(0)
- db.SetConnMaxIdleTime(5 * time.Minute) //nolint:mnd
- // Sized to give DB-bound auth/role/settings paths enough headroom during
- // a reconnect storm from a fleet of agents, while staying well within
- // Postgres max_connections (set to 2000 by PMM Server).
- db.SetMaxIdleConns(50) //nolint:mnd
- db.SetMaxOpenConns(50) //nolint:mnd
+ db.SetConnMaxLifetime(params.ConnMaxLifetime)
+ db.SetConnMaxIdleTime(params.ConnMaxIdleTime)
+ db.SetMaxIdleConns(params.MaxIdleConns)
+ db.SetMaxOpenConns(params.MaxOpenConns)
return db, nil
}
@@ -1258,6 +1255,10 @@ type SetupDBParams struct {
HAPeers []string
SetupFixtures SetupFixturesMode
MigrationVersion *int
+ ConnMaxLifetime time.Duration
+ ConnMaxIdleTime time.Duration
+ MaxIdleConns int
+ MaxOpenConns int
}
// SetupDB checks minimal required PostgreSQL version and runs database migrations. Optionally creates database and adds initial data.
diff --git a/managed/services/agents/deps.go b/managed/services/agents/deps.go
index dfca631a75b..e7da2d5eb42 100644
--- a/managed/services/agents/deps.go
+++ b/managed/services/agents/deps.go
@@ -23,6 +23,7 @@ import (
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/prometheus/common/model"
"github.com/sirupsen/logrus"
+ "gopkg.in/reform.v1"
agentv1 "github.com/percona/pmm/api/agent/v1"
qanv1 "github.com/percona/pmm/api/qan/v1"
@@ -38,7 +39,7 @@ type prometheusService interface {
// ForceConfigurationUpdate triggers immediate synchronous configuration update,
// bypassing the batch delay. Use this for critical updates like port changes.
ForceConfigurationUpdate(ctx context.Context) error
- BuildScrapeConfigForVMAgent(ctx context.Context, pmmAgentID string) ([]byte, error)
+ BuildScrapeConfigForVMAgent(q *reform.Querier, pmmAgentID string) ([]byte, error)
}
// qanClient is a subset of methods of qan.Client used by this package.
@@ -84,3 +85,13 @@ type nomad interface {
GetClientKey() (string, error)
GetClientConfig() models.NomadClient
}
+
+// Limiter defines the interface to perform request rate limiting.
+// If TryAcquire function return false, the request will be rejected.
+// Otherwise, the request will pass.
+type Limiter interface {
+ // Try to acquire a free slot to handle incoming request.
+ TryAcquire() bool
+ // Release the used slot.
+ Release()
+}
diff --git a/managed/services/agents/handler.go b/managed/services/agents/handler.go
index 6480afff971..17f28025858 100644
--- a/managed/services/agents/handler.go
+++ b/managed/services/agents/handler.go
@@ -46,11 +46,15 @@ type Handler struct {
qanClient qanClient
state *StateUpdater
jobsService jobsService
+ // PMM Agents connection attempts rate limiter.
+ // Used to prevent the system degradation (exhausted db connections in particular)
+ // during massive agents connections (thundering herd).
+ rateLimiter Limiter
}
// NewHandler creates new agents handler.
func NewHandler(db *reform.DB, qanClient qanClient, vmdb prometheusService, registry *Registry, state *StateUpdater,
- jobsService jobsService,
+ jobsService jobsService, rateLimiter Limiter,
) *Handler {
h := &Handler{
db: db,
@@ -59,6 +63,7 @@ func NewHandler(db *reform.DB, qanClient qanClient, vmdb prometheusService, regi
qanClient: qanClient,
state: state,
jobsService: jobsService,
+ rateLimiter: rateLimiter,
}
return h
}
@@ -69,8 +74,15 @@ func (h *Handler) Run(stream agentv1.AgentService_ConnectServer) error { //nolin
ctx := stream.Context()
l := logger.Get(ctx)
+
+ if !h.rateLimiter.TryAcquire() {
+ disconnectReason = "RESOURCE_EXHAUSTED"
+ return status.Error(codes.ResourceExhausted, "is rejected by ratelimit, please retry later.")
+ }
agent, err := h.r.register(stream)
+ h.rateLimiter.Release()
if err != nil {
+ l.WithError(err).Warn("Failed to register agent.")
disconnectReason = "auth"
return err
}
@@ -123,7 +135,7 @@ func (h *Handler) Run(stream agentv1.AgentService_ConnectServer) error { //nolin
case *agentv1.StateChangedRequest:
pprof.Do(ctx, pprof.Labels("request", "StateChangedRequest"), func(ctx context.Context) {
- err := h.stateChanged(ctx, p)
+ err := h.stateChanged(ctx, agent.id, p)
if err != nil {
l.Errorf("%+v", err)
}
@@ -174,8 +186,7 @@ func (h *Handler) Run(stream agentv1.AgentService_ConnectServer) error { //nolin
}
}
-func (h *Handler) stateChanged(ctx context.Context, req *agentv1.StateChangedRequest) error {
- var PMMAgentID string
+func (h *Handler) stateChanged(ctx context.Context, pmmAgentID string, req *agentv1.StateChangedRequest) error {
var portsChanged bool
l := logger.Get(ctx).WithField("component", "agents/handler")
@@ -183,7 +194,7 @@ func (h *Handler) stateChanged(ctx context.Context, req *agentv1.StateChangedReq
var agentIDs []string
var err error
sAgentID := strings.TrimPrefix(req.AgentId, "/agent_id/")
- PMMAgentID, agentIDs, err = h.r.roster.get(sAgentID)
+ _, agentIDs, err = h.r.roster.get(sAgentID)
if err != nil {
return err
}
@@ -226,15 +237,7 @@ func (h *Handler) stateChanged(ctx context.Context, req *agentv1.StateChangedReq
h.vmdb.RequestConfigurationUpdate()
}
- agent, err := models.FindAgentByID(h.db.Querier, PMMAgentID)
- if err != nil {
- return err
- }
- if agent.PMMAgentID == nil {
- return nil
- }
-
- h.state.RequestStateUpdate(ctx, *agent.PMMAgentID)
+ h.state.RequestStateUpdate(ctx, pmmAgentID)
return nil
}
diff --git a/managed/services/agents/handler_test.go b/managed/services/agents/handler_test.go
index f1096c4e73e..29719a2d4ae 100644
--- a/managed/services/agents/handler_test.go
+++ b/managed/services/agents/handler_test.go
@@ -16,6 +16,8 @@
package agents
import (
+ "context"
+ "errors"
"testing"
"time"
@@ -25,7 +27,9 @@ import (
"gopkg.in/reform.v1"
"gopkg.in/reform.v1/dialects/postgresql"
+ inventoryv1 "github.com/percona/pmm/api/inventory/v1"
"github.com/percona/pmm/managed/models"
+ "github.com/percona/pmm/utils/logger"
)
func TestCheckPortChanged(t *testing.T) {
@@ -302,4 +306,212 @@ func TestCheckPortChanged(t *testing.T) {
require.NoError(t, mock.ExpectationsWereMet())
})
+
+ t.Run("returns false when loading agent fails with unexpected error", func(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE .+ LIMIT 1`).
+ WithArgs("broken-agent-id").
+ WillReturnError(errors.New("db unavailable"))
+
+ changed := checkPortChanged(db.Querier, "broken-agent-id", 8080)
+ assert.False(t, changed)
+
+ require.NoError(t, mock.ExpectationsWereMet())
+ })
+
+ t.Run("treats wrapped uint32 port as unchanged when effective uint16 port matches", func(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE .+ LIMIT 1`).
+ WithArgs("test-agent-wrap").
+ WillReturnRows(sqlmock.NewRows(agentColumns).AddRow(
+ "test-agent-wrap",
+ string(models.PMMAgentType),
+ "test-node-wrap",
+ nil, nil, nil, nil, nil,
+ time.Now(), time.Now(),
+ false, "", 8080, nil, nil, false,
+ nil, nil, nil, false, false, nil,
+ `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`,
+ ))
+
+ changed := checkPortChanged(db.Querier, "test-agent-wrap", uint32(8080+65536))
+ assert.False(t, changed)
+
+ require.NoError(t, mock.ExpectationsWereMet())
+ })
+}
+
+func TestUpdateAgentStatus(t *testing.T) {
+ t.Parallel()
+
+ agentColumns := []string{
+ "agent_id", "agent_type", "runs_on_node_id", "service_id", "node_id",
+ "pmm_agent_id", "custom_labels", "environment_variables", "created_at", "updated_at",
+ "disabled", "status", "listen_port", "version", "process_exec_path", "is_connected",
+ "username", "password", "agent_password", "tls", "tls_skip_verify",
+ "log_level", "exporter_options", "qan_options", "rta_options",
+ "aws_options", "azure_options", "mongo_options", "mysql_options", "postgresql_options", "valkey_options",
+ }
+
+ t.Run("updates enabled agent status and metadata", func(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE .+ LIMIT 1`).
+ WithArgs("agent-update-ok").
+ WillReturnRows(sqlmock.NewRows(agentColumns).AddRow(
+ "agent-update-ok", string(models.PMMAgentType), "node-1",
+ nil, nil, nil, nil, nil,
+ time.Now(), time.Now(),
+ false, inventoryv1.AgentStatus_AGENT_STATUS_UNKNOWN.String(), 9000, "2.0.0", nil, false,
+ nil, nil, nil, false, false, nil,
+ `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`,
+ ))
+
+ mock.ExpectExec(`UPDATE "agents"`).WillReturnResult(sqlmock.NewResult(0, 1))
+
+ processPath := "/usr/bin/pmm-agent"
+ version := "3.0.0"
+ ctx := logger.Set(context.Background(), "test-request")
+ err = updateAgentStatus(
+ ctx,
+ db.Querier,
+ "agent-update-ok",
+ inventoryv1.AgentStatus_AGENT_STATUS_RUNNING,
+ 10000,
+ &processPath,
+ &version,
+ )
+ require.NoError(t, err)
+ require.NoError(t, mock.ExpectationsWereMet())
+ })
+
+ t.Run("returns error for missing agent with terminal stopping status", func(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ ctx := logger.Set(context.Background(), "test-request")
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE .+ LIMIT 1`).
+ WithArgs("missing-stopping-agent").
+ WillReturnError(reform.ErrNoRows)
+
+ err = updateAgentStatus(
+ ctx,
+ db.Querier,
+ "missing-stopping-agent",
+ inventoryv1.AgentStatus_AGENT_STATUS_STOPPING,
+ 9000,
+ nil,
+ nil,
+ )
+ require.Error(t, err)
+ require.ErrorContains(t, err, "failed to select Agent by ID")
+ require.NoError(t, mock.ExpectationsWereMet())
+ })
+
+ t.Run("returns error for missing agent with non terminal status", func(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ ctx := logger.Set(context.Background(), "test-request")
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE .+ LIMIT 1`).
+ WithArgs("missing-running-agent").
+ WillReturnError(reform.ErrNoRows)
+
+ err = updateAgentStatus(
+ ctx,
+ db.Querier,
+ "missing-running-agent",
+ inventoryv1.AgentStatus_AGENT_STATUS_RUNNING,
+ 9000,
+ nil,
+ nil,
+ )
+ require.Error(t, err)
+ require.ErrorContains(t, err, "failed to select Agent by ID")
+ require.NoError(t, mock.ExpectationsWereMet())
+ })
+
+ t.Run("updates disabled agent without returning error", func(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ ctx := logger.Set(context.Background(), "test-request")
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE .+ LIMIT 1`).
+ WithArgs("disabled-agent").
+ WillReturnRows(sqlmock.NewRows(agentColumns).AddRow(
+ "disabled-agent", string(models.PMMAgentType), "node-disabled",
+ nil, nil, nil, nil, nil,
+ time.Now(), time.Now(),
+ true, inventoryv1.AgentStatus_AGENT_STATUS_RUNNING.String(), 9100, "2.0.0", nil, false,
+ nil, nil, nil, false, false, nil,
+ `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`, `{}`,
+ ))
+
+ mock.ExpectExec(`UPDATE "agents"`).WillReturnResult(sqlmock.NewResult(0, 1))
+
+ err = updateAgentStatus(
+ ctx,
+ db.Querier,
+ "disabled-agent",
+ inventoryv1.AgentStatus_AGENT_STATUS_RUNNING,
+ 9200,
+ nil,
+ nil,
+ )
+ require.NoError(t, err)
+ require.NoError(t, mock.ExpectationsWereMet())
+ })
}
diff --git a/managed/services/agents/mock_limiter_test.go b/managed/services/agents/mock_limiter_test.go
new file mode 100644
index 00000000000..6e8d6709db7
--- /dev/null
+++ b/managed/services/agents/mock_limiter_test.go
@@ -0,0 +1,48 @@
+// Code generated by mockery. DO NOT EDIT.
+
+package agents
+
+import mock "github.com/stretchr/testify/mock"
+
+// mockLimiter is an autogenerated mock type for the Limiter type
+type mockLimiter struct {
+ mock.Mock
+}
+
+// Release provides a mock function with no fields
+func (_m *mockLimiter) Release() {
+ _m.Called()
+}
+
+// TryAcquire provides a mock function with no fields
+func (_m *mockLimiter) TryAcquire() bool {
+ ret := _m.Called()
+
+ if len(ret) == 0 {
+ panic("no return value specified for TryAcquire")
+ }
+
+ var r0 bool
+ if rf, ok := ret.Get(0).(func() bool); ok {
+ r0 = rf()
+ } else {
+ r0 = ret.Get(0).(bool)
+ }
+
+ return r0
+}
+
+// newMockLimiter creates a new instance of mockLimiter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+// The first argument is typically a *testing.T value.
+func newMockLimiter(t interface {
+ mock.TestingT
+ Cleanup(func())
+},
+) *mockLimiter {
+ mock := &mockLimiter{}
+ mock.Mock.Test(t)
+
+ t.Cleanup(func() { mock.AssertExpectations(t) })
+
+ return mock
+}
diff --git a/managed/services/agents/registry.go b/managed/services/agents/registry.go
index e60707d18b6..15b6088ade3 100644
--- a/managed/services/agents/registry.go
+++ b/managed/services/agents/registry.go
@@ -32,6 +32,7 @@ import (
agentv1 "github.com/percona/pmm/api/agent/v1"
"github.com/percona/pmm/managed/models"
"github.com/percona/pmm/managed/services/agents/channel"
+ "github.com/percona/pmm/utils/cache"
"github.com/percona/pmm/utils/logger"
"github.com/percona/pmm/version"
)
@@ -71,8 +72,10 @@ var (
)
type pmmAgentInfo struct {
- channel *channel.Channel
id string
+ version *version.Parsed
+ runsOnNodeID string
+ channel *channel.Channel
stateChangeChan chan struct{}
kickChan chan struct{}
}
@@ -86,8 +89,8 @@ type haService interface {
type Registry struct {
db *reform.DB
- rw sync.RWMutex
- agents map[string]*pmmAgentInfo // id -> info
+ // Currently registered PMM Agents cache.
+ agentsCache *cache.Cache[pmmAgentInfo] // id -> info
roster *roster
@@ -109,11 +112,10 @@ type Registry struct {
// NewRegistry creates a new registry with given database connection.
func NewRegistry(db *reform.DB, vmParams victoriaMetricsParams, ha haService) *Registry {
- agents := make(map[string]*pmmAgentInfo)
r := &Registry{
db: db,
- agents: agents,
+ agentsCache: cache.NewCache[pmmAgentInfo](),
roster: newRoster(db),
@@ -157,10 +159,7 @@ func NewRegistry(db *reform.DB, vmParams victoriaMetricsParams, ha haService) *R
Name: "connected",
Help: "The current number of connected pmm-agents.",
}, func() float64 {
- r.rw.Lock()
- defer r.rw.Unlock()
-
- return float64(len(agents))
+ return float64(r.agentsCache.Size())
})
// initialize metrics with labels
@@ -175,8 +174,8 @@ func NewRegistry(db *reform.DB, vmParams victoriaMetricsParams, ha haService) *R
func (r *Registry) IsConnected(pmmAgentID string) bool {
if !r.haService.Params().Enabled {
// Non-HA mode: check in-memory registry
- _, err := r.get(pmmAgentID)
- return err == nil
+ _, exists := r.agentsCache.Get(pmmAgentID)
+ return exists
}
// HA mode: check cache first, then database
@@ -225,17 +224,20 @@ func (r *Registry) rebuildConnectionCache() {
r.cacheMu.Unlock()
}
-func (r *Registry) register(stream agentv1.AgentService_ConnectServer) (*pmmAgentInfo, error) {
+func (r *Registry) register(stream agentv1.AgentService_ConnectServer) (pmmAgentInfo, error) {
ctx := stream.Context()
l := logger.Get(ctx)
r.mConnects.Inc()
+ // used to return smth in case of error
+ var zero pmmAgentInfo
agentMD, err := agentv1.ReceiveAgentConnectMetadata(stream)
if err != nil {
- return nil, err
+ return zero, err
}
+
var node *models.Node
- err = r.db.InTransaction(func(tx *reform.TX) error {
+ err = r.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error {
node, err = r.authenticate(agentMD, tx.Querier)
if err != nil {
return err
@@ -244,9 +246,13 @@ func (r *Registry) register(stream agentv1.AgentService_ConnectServer) (*pmmAgen
})
if err != nil {
l.Warnf("Failed to authenticate connected pmm-agent %+v.", agentMD)
- return nil, err
+ return zero, err
}
l.Infof("Connected pmm-agent: %+v.", agentMD)
+ pmmAgentVersion, err := version.Parse(agentMD.Version)
+ if err != nil {
+ return zero, fmt.Errorf("failed to parse PMM agent version %q: %w", agentMD.Version, err)
+ }
serverMD := agentv1.ServerConnectMetadata{
AgentRunsOnNodeID: node.NodeID,
@@ -256,11 +262,11 @@ func (r *Registry) register(stream agentv1.AgentService_ConnectServer) (*pmmAgen
l.Debugf("Sending metadata: %+v.", serverMD)
err = agentv1.SendServerConnectMetadata(stream, &serverMD)
if err != nil {
- return nil, err
+ return zero, err
}
- currentAgent, err := r.get(agentMD.ID)
- if err == nil {
+ currentAgent, exists := r.agentsCache.Get(agentMD.ID)
+ if exists {
// pmm-agent with the same ID can still be connected in two cases:
// 1. Someone uses the same ID by mistake, glitch, or malicious intent.
// 2. pmm-agent detects broken connection and reconnects,
@@ -270,23 +276,23 @@ func (r *Registry) register(stream agentv1.AgentService_ConnectServer) (*pmmAgen
// and proceed with the new one.
err := r.ping(ctx, currentAgent)
if err == nil {
- return nil, status.Errorf(codes.AlreadyExists, "pmm-agent with ID %q is already connected.", agentMD.ID)
+ return zero, status.Errorf(codes.AlreadyExists, "pmm-agent with ID %q is already connected.", agentMD.ID)
}
l.Warningf("Failed to ping pmm-agent with ID %q: %v", agentMD.ID, err)
r.Kick(ctx, agentMD.ID)
l.Warningf("pmm-agent with ID %q is kicked.", agentMD.ID)
}
- r.rw.Lock()
- defer r.rw.Unlock()
- agent := &pmmAgentInfo{
- channel: channel.New(ctx, stream),
+ agent := pmmAgentInfo{
id: agentMD.ID,
+ runsOnNodeID: node.NodeID,
+ version: pmmAgentVersion,
+ channel: channel.New(ctx, stream),
stateChangeChan: make(chan struct{}, 1),
kickChan: make(chan struct{}),
}
- r.agents[agentMD.ID] = agent
+ r.agentsCache.Set(agentMD.ID, agent)
// Only persist is_connected to database when HA is enabled
if r.haService.Params().Enabled {
@@ -303,8 +309,8 @@ func (r *Registry) register(stream agentv1.AgentService_ConnectServer) (*pmmAgen
return nil
})
if err != nil {
- delete(r.agents, agentMD.ID)
- return nil, fmt.Errorf("failed to persist the connection status for agent %s: %w", agentMD.ID, err)
+ r.agentsCache.Delete(agentMD.ID)
+ return zero, fmt.Errorf("failed to persist the connection status for agent %s: %w", agentMD.ID, err)
}
r.cacheMu.Lock()
@@ -369,21 +375,20 @@ func (r *Registry) authenticate(md *agentv1.AgentConnectMetadata, q *reform.Quer
}
// unregister removes pmm-agent with given ID from the registry.
-func (r *Registry) unregister(ctx context.Context, pmmAgentID, disconnectReason string) *pmmAgentInfo {
+func (r *Registry) unregister(ctx context.Context, pmmAgentID, disconnectReason string) pmmAgentInfo {
r.mDisconnects.WithLabelValues(disconnectReason).Inc()
-
- r.rw.Lock()
- defer r.rw.Unlock()
+ // used to return smth in case of error
+ var zero pmmAgentInfo
// We do not check that pmmAgentID is in fact ID of existing pmm-agent because
// it may be already deleted from the database, that's why we unregister it.
- agent := r.agents[pmmAgentID]
- if agent == nil {
- return nil
+ agent, ok := r.agentsCache.Get(pmmAgentID)
+ if !ok {
+ return zero
}
- delete(r.agents, pmmAgentID)
+ r.agentsCache.Delete(pmmAgentID)
r.roster.clear(pmmAgentID)
// Only persist connection status when HA is enabled
@@ -420,7 +425,7 @@ func (r *Registry) unregister(ctx context.Context, pmmAgentID, disconnectReason
// ping sends Ping message to given Agent, waits for Pong and observes round-trip time and clock drift.
// Returns true if pong is received, false if there is no pong or error occurred.
-func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error {
+func (r *Registry) ping(ctx context.Context, agent pmmAgentInfo) error {
l := logger.Get(ctx)
start := time.Now()
resp, err := agent.channel.SendAndWaitResponse(&agentv1.Ping{})
@@ -443,7 +448,7 @@ func (r *Registry) ping(ctx context.Context, agent *pmmAgentInfo) error {
}
// addOrRemoveVMAgent - creates vmAgent agentType if pmm-agent's version supports it and agent does not exist yet,
-// otherwise ensures that vmAgent does not start for pmm-agent when pmm-agent's agents don't have push_metrics mode,
+// otherwise ensures that vmAgent does not start for pmm-agent when pmm-agent's agentsCache don't have push_metrics mode,
// removes it if needed.
func (r *Registry) addOrRemoveVMAgent(q *reform.Querier, pmmAgentID, runsOnNodeID string) error {
return r.addVMAgentToPMMAgent(q, pmmAgentID, runsOnNodeID)
@@ -495,7 +500,7 @@ func (r *Registry) addNomadAgentToPMMAgent(q *reform.Querier, pmmAgentID, runsOn
// Kick unregisters and forcefully disconnects pmm-agent with given ID.
func (r *Registry) Kick(ctx context.Context, pmmAgentID string) {
agent := r.unregister(ctx, pmmAgentID, "kick")
- if agent == nil {
+ if agent.id == "" {
return
}
@@ -509,12 +514,11 @@ func (r *Registry) Kick(ctx context.Context, pmmAgentID string) {
// closing agent.kickChan is enough to exit runStateChangeHandler goroutine.
}
-func (r *Registry) get(pmmAgentID string) (*pmmAgentInfo, error) {
- r.rw.RLock()
- pmmAgent := r.agents[pmmAgentID]
- r.rw.RUnlock()
- if pmmAgent == nil {
- return nil, status.Errorf(codes.FailedPrecondition, "pmm-agent with ID %s is not currently connected", pmmAgentID)
+func (r *Registry) get(pmmAgentID string) (pmmAgentInfo, error) {
+ pmmAgent, ok := r.agentsCache.Get(pmmAgentID)
+ if !ok {
+ var zero pmmAgentInfo
+ return zero, status.Errorf(codes.FailedPrecondition, "pmm-agent with ID %s is not currently connected", pmmAgentID)
}
return pmmAgent, nil
}
@@ -530,9 +534,7 @@ func (r *Registry) Describe(ch chan<- *prom.Desc) {
// Collect implement prometheus.Collector.
func (r *Registry) Collect(ch chan<- prom.Metric) {
- r.rw.RLock()
-
- for _, agent := range r.agents {
+ for _, agent := range r.agentsCache.All() {
m := agent.channel.Metrics()
ch <- prom.MustNewConstMetric(mSentDesc, prom.CounterValue, m.Sent, agent.id)
@@ -540,7 +542,6 @@ func (r *Registry) Collect(ch chan<- prom.Metric) {
ch <- prom.MustNewConstMetric(mResponsesDesc, prom.GaugeValue, m.Responses, agent.id)
ch <- prom.MustNewConstMetric(mRequestsDesc, prom.GaugeValue, m.Requests, agent.id)
}
- r.rw.RUnlock()
r.mAgents.Collect(ch)
r.mConnects.Collect(ch)
@@ -551,8 +552,14 @@ func (r *Registry) Collect(ch chan<- prom.Metric) {
// KickAll sends a signal to all registered agents in the registry to perform a kick action.
func (r *Registry) KickAll(ctx context.Context) {
- for _, agentInfo := range r.agents {
- r.Kick(ctx, agentInfo.id)
+ ids := make([]string, 0, r.agentsCache.Size())
+ for _, agentInfo := range r.agentsCache.All() {
+ // NOTE: Can't call Kick() inside `for r.agentsCache.All()` loop.
+ ids = append(ids, agentInfo.id)
+ }
+
+ for _, id := range ids {
+ r.Kick(ctx, id)
}
}
diff --git a/managed/services/agents/registry_test.go b/managed/services/agents/registry_test.go
new file mode 100644
index 00000000000..06fea82a448
--- /dev/null
+++ b/managed/services/agents/registry_test.go
@@ -0,0 +1,246 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package agents
+
+import (
+ "context"
+ "net/url"
+ "testing"
+ "time"
+
+ sqlmock "github.com/DATA-DOG/go-sqlmock"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+ "gopkg.in/reform.v1"
+ "gopkg.in/reform.v1/dialects/postgresql"
+
+ "github.com/percona/pmm/managed/models"
+ "github.com/percona/pmm/utils/logger"
+)
+
+func TestRegistryIsConnectedUsesInMemoryStateWhenHAIsDisabled(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ r.agentsCache.Set("agent-connected", pmmAgentInfo{id: "agent-connected"})
+
+ assert.True(t, r.IsConnected("agent-connected"))
+ assert.False(t, r.IsConnected("agent-missing"))
+}
+
+func TestRegistryIsConnectedUsesFreshHACacheWithoutDatabaseLookup(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: true}})
+ r.connectionCache["agent-connected"] = struct{}{}
+ r.connectionCacheTTL = time.Now().Add(time.Minute)
+
+ assert.True(t, r.IsConnected("agent-connected"))
+}
+
+func TestRegistryIsConnectedRebuildsCacheFromDatabaseInHAMode(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ r := NewRegistry(db, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: true}})
+ r.connectionCacheTTL = time.Now().Add(-time.Second)
+
+ mock.ExpectBegin()
+ mock.ExpectQuery(`SELECT .+ FROM "agents"`).WillReturnRows(newAgentRows(
+ agentRow{id: "agent-connected", connected: true},
+ agentRow{id: "agent-disconnected", connected: false},
+ ))
+ mock.ExpectCommit()
+
+ assert.True(t, r.IsConnected("agent-connected"))
+ assert.False(t, r.IsConnected("agent-disconnected"))
+ require.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestRegistryIsConnectedReturnsFalseWhenHACacheRebuildFails(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ r := NewRegistry(db, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: true}})
+ r.connectionCache["agent-stale"] = struct{}{}
+ r.connectionCacheTTL = time.Now().Add(-time.Second)
+
+ mock.ExpectBegin()
+ mock.ExpectQuery(`SELECT .+ FROM "agents"`).WillReturnError(assert.AnError)
+ mock.ExpectRollback()
+
+ assert.False(t, r.IsConnected("agent-stale"))
+ require.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestRegistryGetReturnsConnectedAgentAndMissingAgentError(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ r.agentsCache.Set("agent-connected", pmmAgentInfo{id: "agent-connected"})
+
+ agent, err := r.get("agent-connected")
+ require.NoError(t, err)
+ assert.Equal(t, "agent-connected", agent.id)
+
+ _, err = r.get("agent-missing")
+ require.Error(t, err)
+ assert.Equal(t, codes.FailedPrecondition, status.Code(err))
+}
+
+func TestRegistryKickRemovesAgentAndClosesKickChannel(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ kickCh := make(chan struct{})
+ r.agentsCache.Set("agent-1", pmmAgentInfo{id: "agent-1", kickChan: kickCh})
+ ctx := logger.Set(context.Background(), "test-request")
+
+ r.Kick(ctx, "agent-1")
+
+ _, exists := r.agentsCache.Get("agent-1")
+ assert.False(t, exists)
+
+ select {
+ case <-kickCh:
+ default:
+ t.Fatal("kick channel should be closed")
+ }
+}
+
+func TestRegistryKickAllDisconnectsEveryRegisteredAgent(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ kickCh1 := make(chan struct{})
+ kickCh2 := make(chan struct{})
+ r.agentsCache.Set("agent-1", pmmAgentInfo{id: "agent-1", kickChan: kickCh1})
+ r.agentsCache.Set("agent-2", pmmAgentInfo{id: "agent-2", kickChan: kickCh2})
+ ctx := logger.Set(context.Background(), "test-request")
+
+ r.KickAll(ctx)
+
+ assert.EqualValues(t, 0, r.agentsCache.Size())
+
+ select {
+ case <-kickCh1:
+ default:
+ t.Fatal("first kick channel should be closed")
+ }
+
+ select {
+ case <-kickCh2:
+ default:
+ t.Fatal("second kick channel should be closed")
+ }
+}
+
+type fakeHAService struct {
+ params *models.HAParams
+}
+
+func (s *fakeHAService) Params() *models.HAParams {
+ return s.params
+}
+
+type fakeVictoriaMetricsParams struct{}
+
+func (fakeVictoriaMetricsParams) ExternalVM() bool {
+ return false
+}
+
+func (fakeVictoriaMetricsParams) URLFor(_ string) (*url.URL, error) {
+ return new(url.URL), nil
+}
+
+func (fakeVictoriaMetricsParams) URL() string {
+ return ""
+}
+
+func (fakeVictoriaMetricsParams) VMAgentArgs() []string {
+ return nil
+}
+
+type agentRow struct {
+ id string
+ connected bool
+}
+
+func newAgentRows(agents ...agentRow) *sqlmock.Rows {
+ rows := sqlmock.NewRows([]string{
+ "agent_id", "agent_type", "runs_on_node_id", "service_id", "node_id",
+ "pmm_agent_id", "custom_labels", "environment_variables", "created_at", "updated_at",
+ "disabled", "status", "listen_port", "version", "process_exec_path", "is_connected",
+ "username", "password", "agent_password", "tls", "tls_skip_verify",
+ "log_level", "exporter_options", "qan_options", "rta_options",
+ "aws_options", "azure_options", "mongo_options", "mysql_options", "postgresql_options", "valkey_options",
+ })
+
+ now := time.Now()
+ for _, a := range agents {
+ rows.AddRow(
+ a.id,
+ string(models.PMMAgentType),
+ "node-1",
+ nil,
+ nil,
+ nil,
+ nil,
+ nil,
+ now,
+ now,
+ false,
+ "",
+ nil,
+ nil,
+ nil,
+ a.connected,
+ nil,
+ nil,
+ nil,
+ false,
+ false,
+ nil,
+ `{}`,
+ `{}`,
+ `{}`,
+ `{}`,
+ `{}`,
+ `{}`,
+ `{}`,
+ `{}`,
+ `{}`,
+ )
+ }
+
+ return rows
+}
diff --git a/managed/services/agents/state.go b/managed/services/agents/state.go
index 8dad8d4dc8b..eaa7310ba8a 100644
--- a/managed/services/agents/state.go
+++ b/managed/services/agents/state.go
@@ -17,25 +17,32 @@ package agents
import (
"context"
+ "errors"
"fmt"
- "sync"
+ "runtime"
"time"
"github.com/AlekSi/pointer"
"github.com/sirupsen/logrus"
+ "golang.org/x/sync/errgroup"
+ "golang.org/x/sync/singleflight"
"google.golang.org/protobuf/encoding/prototext"
"gopkg.in/reform.v1"
+ "github.com/percona/pmm/agent/utils/backoff"
agentv1 "github.com/percona/pmm/api/agent/v1"
"github.com/percona/pmm/managed/models"
"github.com/percona/pmm/utils/logger"
- "github.com/percona/pmm/version"
)
const (
// Constants for delayed batch updates.
- updateBatchDelay = time.Second
- stateChangeTimeout = 5 * time.Second
+ updateBatchDelay = time.Second
+ // State update parameters.
+ stateChangeTimeout = 5 * time.Second
+ // Backoff delays to distribute re-try attempts to update agent's state in case of failure (e.g. DB timeout).
+ backoffMinDelay = 1 * time.Second
+ backoffMaxDelay = 10 * time.Second
loggerComponentNameStateUpdater = "state-updater"
)
@@ -46,6 +53,9 @@ type StateUpdater struct {
vmdb prometheusService
vmParams victoriaMetricsParams
nomad nomad
+ // dbGroup deduplicates concurrent requests to DB (for example models.GetSettings).
+ // In case of massive pmm-agents connection attempts no need to query DB for each of them, just one is enough.
+ dbGroup singleflight.Group
}
// NewStateUpdater creates new agent state updater.
@@ -62,7 +72,10 @@ func NewStateUpdater(db *reform.DB, r *Registry, vmdb prometheusService, vmParam
// RequestStateUpdate requests state update on pmm-agent with given ID. It sets
// the status to done if the agent is not connected.
func (u *StateUpdater) RequestStateUpdate(ctx context.Context, pmmAgentID string) {
- l := logger.Get(ctx).WithField("component", loggerComponentNameStateUpdater)
+ l := logger.Get(ctx).WithFields(logrus.Fields{
+ "component": loggerComponentNameStateUpdater,
+ "pmm_agent_id": pmmAgentID,
+ })
agent, err := u.r.get(pmmAgentID)
if err != nil {
@@ -78,30 +91,27 @@ func (u *StateUpdater) RequestStateUpdate(ctx context.Context, pmmAgentID string
// UpdateAgentsState sends SetStateRequest to all pmm-agents with push metrics agents.
func (u *StateUpdater) UpdateAgentsState(ctx context.Context) error {
- pmmAgents, err := models.FindAllPMMAgentsIDs(u.db.Querier)
+ pmmAgents, err := models.FindAllPMMAgentsIDs(u.db.WithContext(ctx))
if err != nil {
return fmt.Errorf("cannot find pmmAgentsIDs for AgentsState update: %w", err)
}
- var wg sync.WaitGroup
- limiter := make(chan struct{}, 10) //nolint:mnd
- for _, pmmAgentID := range pmmAgents {
- wg.Add(1)
- limiter <- struct{}{}
- go func(pmmAgentID string) {
- defer wg.Done()
- u.RequestStateUpdate(ctx, pmmAgentID)
- <-limiter
- }(pmmAgentID)
+ var wg errgroup.Group
+ wg.SetLimit(runtime.GOMAXPROCS(0))
+
+ for i := range pmmAgents {
+ wg.Go(func() error {
+ u.RequestStateUpdate(ctx, pmmAgents[i])
+ return nil
+ })
}
- wg.Wait()
- return nil
+ return wg.Wait()
}
// runStateChangeHandler runs pmm-agent state update loop for given pmm-agent until ctx is canceled or agent is kicked.
-func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgentInfo) {
- l := logger.Get(ctx).
- WithField("component", loggerComponentNameStateUpdater).
- WithField("agent_id", agent.id)
+func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent pmmAgentInfo) {
+ // NOTE: ctx here tied to gRPC stream from /agent.v1.AgentService/Connect handler
+ // and is alive while connection to pmm-agent is up.
+ l := logger.Get(ctx).WithField("component", loggerComponentNameStateUpdater)
l.Info("Starting runStateChangeHandler ...")
defer l.Info("Done runStateChangeHandler.")
@@ -113,6 +123,14 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgen
panic("stateChangeChan should have capacity 1")
}
+ // stateUpdateBackoff is used to avoid thundering herd problem when many
+ // pmm-agents are trying to update their state at the same time
+ // and fail due to (e.g. DB or context timeout). It is used to avoid system degradation
+ // (exhausted DB connections in particular).
+ stateUpdateBackoff := backoff.New(backoffMinDelay, backoffMaxDelay)
+ timer := time.NewTimer(updateBatchDelay)
+ defer timer.Stop()
+
for {
select {
case <-ctx.Done():
@@ -122,28 +140,49 @@ func (u *StateUpdater) runStateChangeHandler(ctx context.Context, agent *pmmAgen
return
case <-agent.stateChangeChan:
- // batch several update requests together by delaying the first one
- sleepCtx, sleepCancel := context.WithTimeout(ctx, updateBatchDelay)
- <-sleepCtx.Done()
- sleepCancel()
+ if !timer.Stop() {
+ select {
+ case <-timer.C:
+ default:
+ }
+ }
+ timer.Reset(updateBatchDelay)
- if ctx.Err() != nil {
+ select {
+ // batch several update requests together by delaying the first one
+ case <-timer.C:
+ case <-ctx.Done():
return
}
nCtx, cancel := context.WithTimeout(ctx, stateChangeTimeout)
err := u.sendSetStateRequest(nCtx, agent)
+ cancel()
if err != nil {
l.Error(err)
+ if errors.Is(err, context.DeadlineExceeded) {
+ // state update failed due to context timeout
+ // (most likely - waiting for free SQL connection in pool).
+ // Sleep for a while to avoid thundering herd problem when many
+ // pmm-agents are trying to update their state at the same time.
+ timer.Reset(stateUpdateBackoff.Delay())
+ select {
+ case <-timer.C:
+ case <-ctx.Done():
+ return
+ }
+ }
u.RequestStateUpdate(ctx, agent.id)
+ continue
}
- cancel()
+ // seems DB came back to normal - reset backoff.
+ stateUpdateBackoff.Reset()
}
}
}
// sendSetStateRequest sends SetStateRequest to given pmm-agent.
-func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentInfo) error { //nolint:gocognit,cyclop,maintidx
+func (u *StateUpdater) sendSetStateRequest(ctx context.Context, pmmAgentInfo pmmAgentInfo) error { //nolint:gocognit,cyclop,maintidx
l := logger.Get(ctx).WithField("component", loggerComponentNameStateUpdater)
start := time.Now()
defer func() {
@@ -151,33 +190,85 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI
l.Warnf("sendSetStateRequest took %s.", dur)
}
}()
- pmmAgent, err := models.FindAgentByID(u.db.Querier, agent.id)
- if err != nil {
- return fmt.Errorf("failed to get PMM Agent: %w", err)
- }
- pmmAgentVersion, err := version.Parse(*pmmAgent.Version)
- if err != nil {
- return fmt.Errorf("failed to parse PMM agent version %q: %w", *pmmAgent.Version, err)
- }
- settings, err := models.GetSettings(u.db.Querier)
+ // Use singleflight to avoid fetching settings for each pmm-agent separately.
+ fetchedSettings, err, _ := u.dbGroup.Do("settings", func() (any, error) { //nolint:contextcheck
+ // NOTE 1: leader's context is not used here directly in order to allow to finish
+ // the request to DB, so that even if leader's request is already terminated -
+ // the rest of waiters in singleflight group will receive the response from DB.
+ settingsCtx, cancel := context.WithTimeout(context.Background(), stateChangeTimeout)
+ defer cancel()
+
+ sett, fetchErr := models.GetSettings(u.db.WithContext(settingsCtx))
+ if fetchErr != nil {
+ // IMPORTANT: On error, we call Forget(hash) IMMEDIATELY.
+ // This prevents the error from getting stuck in the internal singleflight map
+ // and allows the next request to retry immediately (e.g. if the DB is being restored).
+ u.dbGroup.Forget("settings")
+ return nil, fetchErr
+ }
+ return sett, nil
+ })
if err != nil {
+ l.WithError(err).Error("failed to get settings")
return fmt.Errorf("failed to get settings: %w", err)
}
+ settings, ok := fetchedSettings.(*models.Settings)
+ if !ok {
+ l.Errorf("failed to cast settings: %T", fetchedSettings)
+ return errors.New("failed to get settings")
+ }
filters := models.AgentFilters{
- PMMAgentID: agent.id,
+ PMMAgentID: pmmAgentInfo.id,
IgnoreNomad: !settings.IsNomadEnabled(),
// fetch enabled only
Disabled: new(false),
}
- agents, err := models.FindAgents(u.db.Querier, filters)
+ // It is completely OK to re-use the same Querier for multiple queries, as it is safe for concurrent use
+ // and creates less preasure on GC.
+ q := u.db.WithContext(ctx)
+ agents, err := models.FindAgents(q, filters)
if err != nil {
+ l.WithError(err).Errorf("failed to collect agents")
return fmt.Errorf("failed to collect agents: %w", err)
}
+ // pre-fetch node info since it's common for all subagents of particluar pmm-agent.
+ // Use singleflight to avoid fetching settings for each pmm-agent separately in cases
+ // when several pmm-agents are running on the same node.
+ nodeKey := "node/" + pmmAgentInfo.runsOnNodeID
+ fetchedNode, err, _ := u.dbGroup.Do(nodeKey, func() (any, error) { //nolint:contextcheck
+ // NOTE 1: leader's context is not used here directly in order to allow to finish
+ // the request to DB, so that even if leader's request is already terminated -
+ // the rest of waiters in singleflight group will receive the response from DB.
+ nodeCtx, cancel := context.WithTimeout(context.Background(), stateChangeTimeout)
+ defer cancel()
+ node, fetchErr := models.FindNodeByID(u.db.WithContext(nodeCtx), pmmAgentInfo.runsOnNodeID)
+ if fetchErr != nil {
+ // IMPORTANT: On error, we call Forget(nodeKey) IMMEDIATELY.
+ // This prevents the error from getting stuck in the internal singleflight map
+ // and allows the next request to retry immediately (e.g. if the DB is being restored).
+ u.dbGroup.Forget(nodeKey)
+ return nil, fetchErr
+ }
+ return node, nil
+ })
+ if err != nil {
+ l.WithError(err).
+ WithField("node_id", pmmAgentInfo.runsOnNodeID).
+ Error("failed to fetch node info")
+ return fmt.Errorf("failed to fetch node info: %w", err)
+ }
+ node, ok := fetchedNode.(*models.Node)
+ if !ok {
+ l.WithField("node_id", pmmAgentInfo.runsOnNodeID).
+ Errorf("failed to cast Node: %T", fetchedNode)
+ return fmt.Errorf("failed to fetch node %s info", pmmAgentInfo.runsOnNodeID)
+ }
+
redactMode := redactSecrets
- if l.Logger.GetLevel() >= logrus.DebugLevel {
+ if l.Logger.IsLevelEnabled(logrus.DebugLevel) {
redactMode = exposeSecrets
}
@@ -189,16 +280,12 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI
case models.PMMAgentType:
continue
case models.VMAgentType:
- scrapeCfg, err := u.vmdb.BuildScrapeConfigForVMAgent(ctx, agent.id)
+ scrapeCfg, err := u.vmdb.BuildScrapeConfigForVMAgent(q, pmmAgentInfo.id)
if err != nil {
- return fmt.Errorf("cannot get agent scrape config for agent %s: %w", agent.id, err)
+ return fmt.Errorf("cannot get agent scrape config for agent %s: %w", pmmAgentInfo.id, err)
}
agentProcesses[row.AgentID] = vmAgentConfig(string(scrapeCfg), u.vmParams)
case models.NomadAgentType:
- node, err := models.FindNodeByID(u.db.Querier, pointer.GetString(row.NodeID))
- if err != nil {
- return err
- }
params, err := nomadClientConfig(u.nomad, node, row)
if err != nil {
return err
@@ -206,33 +293,28 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI
agentProcesses[row.AgentID] = params
case models.NodeExporterType:
- node, err := models.FindNodeByID(u.db.Querier, pointer.GetString(row.NodeID))
- if err != nil {
- return err
- }
-
- params, err := nodeExporterConfig(node, row, pmmAgentVersion)
+ params, err := nodeExporterConfig(node, row, pmmAgentInfo.version)
if err != nil {
return err
}
agentProcesses[row.AgentID] = params
case models.RDSExporterType:
- node, err := models.FindNodeByID(u.db.Querier, pointer.GetString(row.NodeID))
+ rdsNode, err := models.FindNodeByID(q, pointer.GetString(row.NodeID))
if err != nil {
return err
}
- rdsExporters[node] = row
+ rdsExporters[rdsNode] = row
case models.ExternalExporterType:
// ignore
case models.AzureDatabaseExporterType:
- service, err := models.FindServiceByID(u.db.Querier, pointer.GetString(row.ServiceID))
+ service, err := models.FindServiceByID(q, pointer.GetString(row.ServiceID))
if err != nil {
return err
}
- config, err := azureDatabaseExporterConfig(row, service, redactMode, pmmAgentVersion)
+ config, err := azureDatabaseExporterConfig(row, service, redactMode, pmmAgentInfo.version)
if err != nil {
return err
}
@@ -244,48 +326,47 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI
models.QANMongoDBProfilerAgentType, models.QANMongoDBMongologAgentType,
models.QANPostgreSQLPgStatementsAgentType, models.QANPostgreSQLPgStatMonitorAgentType,
models.RTAMongoDBAgentType:
- service, err := models.FindServiceByID(u.db.Querier, pointer.GetString(row.ServiceID))
+ service, err := models.FindServiceByID(q, pointer.GetString(row.ServiceID))
if err != nil {
return err
}
- node, _ := models.FindNodeByID(u.db.Querier, pointer.GetString(pmmAgent.RunsOnNodeID))
switch row.AgentType { //nolint:exhaustive
case models.MySQLdExporterType:
- cfg, err := mysqldExporterConfig(node, service, row, redactMode, pmmAgentVersion)
+ cfg, err := mysqldExporterConfig(node, service, row, redactMode, pmmAgentInfo.version)
if err != nil {
return err
}
agentProcesses[row.AgentID] = cfg
case models.MongoDBExporterType:
- cfg, err := mongodbExporterConfig(node, service, row, redactMode, pmmAgentVersion)
+ cfg, err := mongodbExporterConfig(node, service, row, redactMode, pmmAgentInfo.version)
if err != nil {
return err
}
agentProcesses[row.AgentID] = cfg
case models.PostgresExporterType:
- cfg, err := postgresExporterConfig(node, service, row, redactMode, pmmAgentVersion)
+ cfg, err := postgresExporterConfig(node, service, row, redactMode, pmmAgentInfo.version)
if err != nil {
return err
}
agentProcesses[row.AgentID] = cfg
case models.ProxySQLExporterType:
- agentProcesses[row.AgentID] = proxysqlExporterConfig(node, service, row, redactMode, pmmAgentVersion)
+ agentProcesses[row.AgentID] = proxysqlExporterConfig(node, service, row, redactMode, pmmAgentInfo.version)
case models.ValkeyExporterType:
- agentProcesses[row.AgentID] = valkeyExporterConfig(node, service, row, redactMode, pmmAgentVersion)
+ agentProcesses[row.AgentID] = valkeyExporterConfig(node, service, row, redactMode, pmmAgentInfo.version)
case models.QANMySQLPerfSchemaAgentType:
- builtinAgents[row.AgentID] = qanMySQLPerfSchemaAgentConfig(service, row, pmmAgentVersion)
+ builtinAgents[row.AgentID] = qanMySQLPerfSchemaAgentConfig(service, row, pmmAgentInfo.version)
case models.QANMySQLSlowlogAgentType:
- builtinAgents[row.AgentID] = qanMySQLSlowlogAgentConfig(service, row, pmmAgentVersion)
+ builtinAgents[row.AgentID] = qanMySQLSlowlogAgentConfig(service, row, pmmAgentInfo.version)
case models.QANMongoDBProfilerAgentType:
- builtinAgents[row.AgentID] = qanMongoDBProfilerAgentConfig(service, row, pmmAgentVersion)
+ builtinAgents[row.AgentID] = qanMongoDBProfilerAgentConfig(service, row, pmmAgentInfo.version)
case models.QANMongoDBMongologAgentType:
- builtinAgents[row.AgentID] = qanMongoDBMongologAgentConfig(service, row, pmmAgentVersion)
+ builtinAgents[row.AgentID] = qanMongoDBMongologAgentConfig(service, row, pmmAgentInfo.version)
case models.QANPostgreSQLPgStatementsAgentType:
- builtinAgents[row.AgentID] = qanPostgreSQLPgStatementsAgentConfig(service, row, pmmAgentVersion)
+ builtinAgents[row.AgentID] = qanPostgreSQLPgStatementsAgentConfig(service, row, pmmAgentInfo.version)
case models.QANPostgreSQLPgStatMonitorAgentType:
- builtinAgents[row.AgentID] = qanPostgreSQLPgStatMonitorAgentConfig(service, row, pmmAgentVersion)
+ builtinAgents[row.AgentID] = qanPostgreSQLPgStatMonitorAgentConfig(service, row, pmmAgentInfo.version)
case models.RTAMongoDBAgentType:
- builtinAgents[row.AgentID] = rtaMongoDBAgentConfig(service, row, pmmAgentVersion)
+ builtinAgents[row.AgentID] = rtaMongoDBAgentConfig(service, row, pmmAgentInfo.version)
}
default:
@@ -311,8 +392,8 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI
for awsAccessKey, exporters := range groupedRdsExporters {
// TODO: split by 50 exporters per group
- groupID := u.r.roster.add(agent.id, rdsPrefix+awsAccessKey, exporters)
- c, err := rdsExporterConfig(exporters, redactMode, pmmAgentVersion)
+ groupID := u.r.roster.add(pmmAgentInfo.id, rdsPrefix+awsAccessKey, exporters)
+ c, err := rdsExporterConfig(exporters, redactMode, pmmAgentInfo.version)
if err != nil {
return err
}
@@ -331,7 +412,7 @@ func (u *StateUpdater) sendSetStateRequest(ctx context.Context, agent *pmmAgentI
l.Debugf("sendSetStateRequest:\n%s\n", prototext.Format(logger.RedactMessage(state)))
}
- resp, err := agent.channel.SendAndWaitResponse(state)
+ resp, err := pmmAgentInfo.channel.SendAndWaitResponse(state)
if err != nil {
return err
}
diff --git a/managed/services/agents/state_test.go b/managed/services/agents/state_test.go
new file mode 100644
index 00000000000..1e482378105
--- /dev/null
+++ b/managed/services/agents/state_test.go
@@ -0,0 +1,152 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package agents
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ sqlmock "github.com/DATA-DOG/go-sqlmock"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gopkg.in/reform.v1"
+ "gopkg.in/reform.v1/dialects/postgresql"
+
+ "github.com/percona/pmm/managed/models"
+ "github.com/percona/pmm/utils/logger"
+)
+
+func TestRequestStateUpdateQueuesUpdateForConnectedAgent(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ r.agentsCache.Set("agent-1", pmmAgentInfo{id: "agent-1", stateChangeChan: make(chan struct{}, 1)})
+ u := NewStateUpdater(nil, r, nil, nil, nil)
+ ctx := logger.Set(context.Background(), "test-request")
+
+ u.RequestStateUpdate(ctx, "agent-1")
+
+ agent, ok := r.agentsCache.Get("agent-1")
+ require.True(t, ok)
+ select {
+ case <-agent.stateChangeChan:
+ default:
+ t.Fatal("expected state update signal")
+ }
+}
+
+func TestRequestStateUpdateDoesNothingForMissingAgent(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ u := NewStateUpdater(nil, r, nil, nil, nil)
+ ctx := logger.Set(context.Background(), "test-request")
+
+ assert.NotPanics(t, func() {
+ u.RequestStateUpdate(ctx, "missing-agent")
+ })
+}
+
+func TestRequestStateUpdateDoesNotBlockWhenUpdateIsAlreadyQueued(t *testing.T) {
+ t.Parallel()
+
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ agent := pmmAgentInfo{id: "agent-1", stateChangeChan: make(chan struct{}, 1)}
+ agent.stateChangeChan <- struct{}{}
+ r.agentsCache.Set("agent-1", agent)
+ u := NewStateUpdater(nil, r, nil, nil, nil)
+ ctx := logger.Set(context.Background(), "test-request")
+
+ done := make(chan struct{})
+ go func() {
+ u.RequestStateUpdate(ctx, "agent-1")
+ close(done)
+ }()
+
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("RequestStateUpdate should not block when update is already queued")
+ }
+
+ assert.Len(t, agent.stateChangeChan, 1)
+}
+
+func TestUpdateAgentsStateQueuesUpdatesForAllConnectedAgents(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ r.agentsCache.Set("agent-1", pmmAgentInfo{id: "agent-1", stateChangeChan: make(chan struct{}, 1)})
+ r.agentsCache.Set("agent-2", pmmAgentInfo{id: "agent-2", stateChangeChan: make(chan struct{}, 1)})
+ u := NewStateUpdater(db, r, nil, nil, nil)
+ ctx := logger.Set(context.Background(), "test-request")
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE agent_type = \$1 ORDER BY agent_id`).
+ WithArgs(string(models.PMMAgentType)).
+ WillReturnRows(newAgentRows(
+ agentRow{id: "agent-1", connected: true},
+ agentRow{id: "agent-2", connected: true},
+ ))
+
+ err = u.UpdateAgentsState(ctx)
+ require.NoError(t, err)
+
+ for _, agentID := range []string{"agent-1", "agent-2"} {
+ agent, ok := r.agentsCache.Get(agentID)
+ require.True(t, ok)
+ select {
+ case <-agent.stateChangeChan:
+ default:
+ t.Fatalf("expected state update signal for %s", agentID)
+ }
+ }
+
+ require.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestUpdateAgentsStateReturnsErrorWhenFetchingAgentsFails(t *testing.T) {
+ t.Parallel()
+
+ sqlDB, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _ = mock.ExpectClose()
+ assert.NoError(t, sqlDB.Close())
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ r := NewRegistry(nil, fakeVictoriaMetricsParams{}, &fakeHAService{params: &models.HAParams{Enabled: false}})
+ u := NewStateUpdater(db, r, nil, nil, nil)
+ ctx := logger.Set(context.Background(), "test-request")
+
+ mock.ExpectQuery(`SELECT .+ FROM "agents" WHERE agent_type = \$1 ORDER BY agent_id`).
+ WithArgs(string(models.PMMAgentType)).
+ WillReturnError(assert.AnError)
+
+ err = u.UpdateAgentsState(ctx)
+ require.Error(t, err)
+ require.ErrorContains(t, err, "cannot find pmmAgentsIDs for AgentsState update")
+ require.NoError(t, mock.ExpectationsWereMet())
+}
diff --git a/managed/services/grafana/access_control_cache.go b/managed/services/grafana/access_control_cache.go
index 3b678188500..9ed1d009099 100644
--- a/managed/services/grafana/access_control_cache.go
+++ b/managed/services/grafana/access_control_cache.go
@@ -28,7 +28,7 @@ import (
const accessControlCacheExpiration = 3 * time.Second
// accessControl provides caching for the access control configuration.
-type accessControl struct {
+type accessControlCache struct {
mu sync.RWMutex
db *reform.DB
@@ -36,7 +36,7 @@ type accessControl struct {
lastUpdated time.Time
}
-func (a *accessControl) isEnabled() bool {
+func (a *accessControlCache) isEnabled() bool {
a.mu.RLock()
if a.lastUpdated.Add(accessControlCacheExpiration).After(time.Now()) {
@@ -54,7 +54,7 @@ func (a *accessControl) isEnabled() bool {
return enabled
}
-func (a *accessControl) reload() (bool, error) {
+func (a *accessControlCache) reload() (bool, error) {
a.mu.Lock()
defer a.mu.Unlock()
diff --git a/managed/services/grafana/auth_server.go b/managed/services/grafana/auth_server.go
index cd435322b22..997782863fd 100644
--- a/managed/services/grafana/auth_server.go
+++ b/managed/services/grafana/auth_server.go
@@ -24,21 +24,18 @@ import (
"fmt"
"net/http"
"net/http/httputil"
- "net/url"
- "path"
- "strconv"
"strings"
- "sync"
"time"
- "unicode/utf8"
"github.com/lib/pq"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
+ "golang.org/x/sync/singleflight"
"google.golang.org/grpc/codes"
"gopkg.in/reform.v1"
"github.com/percona/pmm/managed/models"
+ "github.com/percona/pmm/utils/cache"
)
const (
@@ -94,9 +91,9 @@ var rules = map[string]role{
"/v1/users/current/orgs": none,
// must be available without authentication for health checking
+ // Handled in NGINX config.
"/v1/server/readyz": none,
"/v1/server/leaderHealthCheck": none,
- "/ping": none, // PMM 1.x variant
// must not be available without authentication as it can leak data
"/v1/server/version": viewer,
@@ -152,45 +149,37 @@ var lbacPrefixes = []string{
const lbacHeaderName = "X-Proxy-Filter"
-// nginx auth_request directive supports only 401 and 403 - every other code results in 500.
-// Our APIs can return codes.PermissionDenied which maps to 403 / http.StatusForbidden.
-// Our APIs MUST NOT return codes.Unauthenticated which maps to 401 / http.StatusUnauthorized
-// as this code is reserved for auth_request.
-const authenticationErrorCode = 401
+const (
+ // Nginx auth_request directive supports only 401 and 403 - every other code results in 500.
+ // Our APIs can return codes.PermissionDenied which maps to 403 / http.StatusForbidden.
+ // Our APIs MUST NOT return codes.Unauthenticated which maps to 401 / http.StatusUnauthorized
+ // as this code is reserved for auth_request.
+ authenticationErrorCode = 401
+ // HTTP headers used to pass auth error details back to NGINX.
+ // The same headers are parsed in NGINX configuration file.
+ authResponseCodeHeader = "X-Auth-Code"
+ authResponseErrorHeader = "X-Auth-Error"
+ authResponseMessageHeader = "X-Auth-Message"
+)
const (
- // Note: cacheInvalidationInterval is used to invalidate cache for grafana responses.
- cacheInvalidationInterval = 60 * time.Second
- authenticationTimeout = 15 * time.Second
- prometheusNamespace = "pmm_managed"
- prometheusSubsystem = "auth"
+ authenticationTimeout = 15 * time.Second
+ prometheusNamespace = "pmm_managed"
+ prometheusSubsystem = "auth"
)
-func statusCodeToString(code int) string {
- switch code {
- case http.StatusOK:
- return "200"
- case http.StatusBadRequest:
- return "400"
- case http.StatusUnauthorized:
- return "401"
- case http.StatusForbidden:
- return "403"
- case http.StatusNotFound:
- return "404"
- case http.StatusMethodNotAllowed:
- return "405"
- case http.StatusRequestTimeout:
- return "408"
- case http.StatusTooManyRequests:
- return "429"
- case http.StatusInternalServerError:
- return "500"
- case http.StatusServiceUnavailable:
- return "503"
- default:
- return strconv.Itoa(code)
- }
+const (
+ // Ttl for auth response validiness in auth cache.
+ cacheItemTTL = 60 * time.Second
+ // Auth response cache cleanup interval.
+ cacheInvalidationInterval = 2 * cacheItemTTL
+)
+
+// authResult contains authentication response details that is a result of all
+// authentication and authorization (including LBAC) checks.
+type authResult struct {
+ // encoded filers to be added as proxy headers.
+ vmProxyFilters string
}
// clientError contains authentication error response details.
@@ -199,22 +188,8 @@ type authError struct {
message string
}
-var (
- // ErrInvalidUserID is returned when user ID is not valid.
- ErrInvalidUserID = errors.New("InvalidUserID")
-
- // ErrCannotGetUserID is returned when we cannot retrieve user ID.
- ErrCannotGetUserID = errors.New("CannotGetUserID")
-)
-
-type cacheItem struct {
- u authUser
- created time.Time
-}
-
-// clientInterface exist only to make fuzzing simpler.
-type clientInterface interface {
- getAuthUser(ctx context.Context, authHeaders http.Header, l *logrus.Entry) (authUser, error)
+func (a *authError) Error() string {
+ return fmt.Sprintf("%s: %s", a.message, a.code)
}
type authMetrics struct {
@@ -230,10 +205,16 @@ type authMetrics struct {
mDurations *prom.HistogramVec
}
+type cachedAuthUser struct {
+ user authUser
+ authorization string
+ cookie string
+}
+
// AuthServer authenticates incoming requests via Grafana API.
type AuthServer struct {
// c is the client used to interact with the Grafana API.
- c clientInterface
+ c grafanaAuthUserGetter
// db is the PostgreSQL database handle using reform ORM.
db *reform.DB
// l is the structured logger for the auth component.
@@ -241,12 +222,13 @@ type AuthServer struct {
// cache stores authentication responses to reduce Grafana API calls.
// Stores positive responses only.
- cache map[string]cacheItem
- // rw protects the cache for concurrent access.
- rw sync.RWMutex
+ // TODO: cache negative response as well.
+ cache *cache.TTLCache[cachedAuthUser]
+ // authUserGroup deduplicates concurrent Grafana auth lookups for the same auth header set.
+ authUserGroup singleflight.Group
// accessControl manages RBAC and LBAC filtering logic.
- accessControl *accessControl
+ accessControl accessControl
// TODO server metrics should be provided by middleware https://jira.percona.com/browse/PMM-4326
// Prometheus metrics for the AuthServer.
@@ -254,15 +236,19 @@ type AuthServer struct {
}
// NewAuthServer creates new AuthServer.
-func NewAuthServer(c clientInterface, db *reform.DB) *AuthServer {
+func NewAuthServer(ctx context.Context, c grafanaAuthUserGetter, db *reform.DB) *AuthServer {
+ cache, err := cache.NewCacheTTL[cachedAuthUser](ctx, cacheItemTTL, cacheInvalidationInterval)
+ if err != nil {
+ panic(err)
+ }
s := &AuthServer{
- c: c,
- db: db,
- l: logrus.WithField("component", "grafana/auth"),
- cache: make(map[string]cacheItem),
- accessControl: &accessControl{
+ c: c,
+ db: db,
+ l: logrus.WithField("component", "grafana/auth"),
+ accessControl: &accessControlCache{
db: db,
},
+ cache: cache,
metrics: authMetrics{
mAuthRequests: prom.NewCounterVec(
prom.CounterOpts{
@@ -316,10 +302,7 @@ func (s *AuthServer) Collect(ch chan<- prom.Metric) {
s.metrics.mGrafanaAuthRequests.Collect(ch)
s.metrics.mCache.Collect(ch)
- s.rw.RLock()
- cacheSize := len(s.cache)
- s.rw.RUnlock()
- ch <- prom.MustNewConstMetric(s.metrics.mCacheSizeDesc, prom.GaugeValue, float64(cacheSize))
+ ch <- prom.MustNewConstMetric(s.metrics.mCacheSizeDesc, prom.GaugeValue, float64(s.cache.Size()))
s.metrics.mDurations.Collect(ch)
}
@@ -340,29 +323,6 @@ func (s *AuthServer) incCacheMiss() {
s.metrics.mCache.WithLabelValues("miss").Inc()
}
-// Run runs cache invalidator which removes expired cache items.
-func (s *AuthServer) Run(ctx context.Context) {
- t := time.NewTicker(cacheInvalidationInterval)
- defer t.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return
-
- case <-t.C:
- now := time.Now()
- s.rw.Lock()
- for key, item := range s.cache {
- if now.Add(-cacheInvalidationInterval).After(item.created) {
- delete(s.cache, key)
- }
- }
- s.rw.Unlock()
- }
- }
-}
-
// ServeHTTP serves internal location /auth_request for both authentication subrequests
// and subsequent normal requests.
func (s *AuthServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
@@ -381,151 +341,81 @@ func (s *AuthServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
err := extractOriginalRequest(req)
if err != nil {
- s.l.Warnf("Failed to parse request: %s.", err)
+ s.l.WithError(err).Warn("Failed to parse original request headers.")
rw.WriteHeader(http.StatusBadRequest)
- method := req.Header.Get("X-Original-Method")
- if method == "" {
- method = req.Method
- }
-
- route := req.Header.Get("X-Original-Uri")
- if route == "" {
- route = req.URL.Path
- } else if i := strings.IndexByte(route, '?'); i >= 0 {
- route = route[:i]
- }
- cleaned, cleanErr := cleanPath(route)
- if cleanErr == nil {
- route = cleaned
- }
-
- s.incAuthRequests(method, route, http.StatusBadRequest)
+ s.incAuthRequests(req.Method, req.URL.Path, http.StatusBadRequest)
return
}
- l := s.l.WithField("req", fmt.Sprintf("%s %s", req.Method, req.URL.Path))
+ // NOTE: now req.Method and req.URL.Path contain original request values
+ // that NGINX received from the client. The original request values are used for
+ // logging and authentication.
+
+ l := s.l.WithFields(logrus.Fields{"method": req.Method, "path": req.URL.Path})
// TODO l := logger.Get(ctx) once we have it after https://jira.percona.com/browse/PMM-4326
+ // Limit the total time spent on authentication to avoid long delays
+ // in case of Grafana being slow or unavailable.
ctx, cancel := context.WithTimeout(req.Context(), authenticationTimeout)
defer cancel()
- authUser, authErr := s.authenticate(ctx, req, l)
- if authErr != nil {
- // copy grpc-gateway behavior: set correct codes, set both "error" and "message"
- m := map[string]any{
- "code": int(authErr.code),
- "error": authErr.message,
- "message": authErr.message, //nolint:goconst
+ authRes, err := s.processRequest(ctx, req, l)
+ if err != nil {
+ authErr, ok := errors.AsType[*authError](err)
+ if ok {
+ status := convertAuthErrorToHTTPStatus(authErr.code)
+ s.incAuthRequests(req.Method, req.URL.Path, status)
+ writeResponseErrorStatus(rw, status, int(authErr.code), authErr.message, authErr.message)
+ return
}
-
- status := httpStatusForAuthError(authErr.code)
- s.incAuthRequests(req.Method, req.URL.Path, status)
- s.returnError(rw, status, m, l)
+ s.incAuthRequests(req.Method, req.URL.Path, http.StatusInternalServerError)
+ writeResponseErrorStatus(rw, http.StatusInternalServerError, http.StatusInternalServerError,
+ statusCodeToString(http.StatusInternalServerError), statusCodeToString(http.StatusInternalServerError))
return
}
- var userID int
- if authUser != nil {
- userID = authUser.userID
+ if authRes.vmProxyFilters != "" {
+ // Add HTTP headers to response based on filled fields in authResult.
+ rw.Header().Set(lbacHeaderName, authRes.vmProxyFilters)
}
-
- errF := s.maybeAddLBACFilters(ctx, rw, req, userID, l)
- if errF != nil {
- // copy grpc-gateway behavior: set correct codes, set both "error" and "message"
- m := map[string]any{
- "code": int(codes.Internal),
- "error": "Internal server error.",
- "message": "Internal server error.",
- }
- l.Errorf("Failed to add VMProxy filters: %s", errF)
-
- s.incAuthRequests(req.Method, req.URL.Path, authenticationErrorCode)
- s.returnError(rw, authenticationErrorCode, m, l)
- return
- }
-
s.incAuthRequests(req.Method, req.URL.Path, http.StatusOK)
}
-// httpStatusForAuthError maps an authError code to the HTTP status nginx receives.
-// PermissionDenied uses 403 so nginx denies outright; the 401 re-run is a GET and would
-// wrongly pass method-specific rules. Authentication and internal errors stay 401.
-func httpStatusForAuthError(code codes.Code) int {
- if code == codes.PermissionDenied {
- return http.StatusForbidden
- }
- return authenticationErrorCode
-}
-
-func (s *AuthServer) returnError(rw http.ResponseWriter, status int, msg map[string]any, l *logrus.Entry) {
- // nginx ignores the auth_request subrequest body: on 401 it re-runs the request to
- // /auth_request to fetch this body; on 403 it serves a static body via error_page 403.
- rw.Header().Set("Content-Type", "application/json")
-
- rw.WriteHeader(status)
- err := json.NewEncoder(rw).Encode(msg)
- if err != nil {
- l.Warnf("%s", err)
- }
-}
-
-// maybeAddLBACFilters adds extra filters to requests proxied through VMProxy.
-// In case the request is not proxied through VMProxy, this is a no-op.
-func (s *AuthServer) maybeAddLBACFilters(ctx context.Context, rw http.ResponseWriter, req *http.Request, userID int, l *logrus.Entry) error {
- if !s.shallAddLBACFilters(req) {
- l.Debugf("Skipping LBAC filters for non-proxied request.")
- return nil
- }
-
- if userID == 0 {
- l.Debugf("Getting authenticated user info")
- authUser, err := s.getAuthUser(ctx, req, l)
- if err != nil {
- return ErrCannotGetUserID
- }
-
- if authUser == nil {
- return fmt.Errorf("%w: user is empty", ErrCannotGetUserID)
- }
-
- userID = authUser.userID
- }
-
+// addLBACFilters adds extra filters to requests proxied through VMProxy.
+func (s *AuthServer) addLBACFilters(ctx context.Context, userID int, l *logrus.Entry) (string, error) {
if userID <= 0 {
// Anonymous users don't have a numeric user ID and cannot have LBAC roles.
// Skip adding filters and allow the request to proceed.
l.Debugf("Skipping LBAC filters for anonymous user.")
- return nil
+ return "", nil
}
filters, err := s.getLBACFilters(ctx, userID)
if err != nil {
- return err
+ return "", err
}
if len(filters) == 0 {
- return nil
+ return "", nil
}
jsonFilters, err := json.Marshal(filters)
if err != nil {
- return fmt.Errorf("failed to marshal LBAC filters: %w", err)
+ return "", fmt.Errorf("failed to marshal LBAC filters: %w", err)
}
- rw.Header().Set(lbacHeaderName, base64.StdEncoding.EncodeToString(jsonFilters))
-
- return nil
+ return base64.StdEncoding.EncodeToString(jsonFilters), nil
}
-// shallAddLBACFilters decides if LBAC filters must be added to the outgoing request.
-func (s *AuthServer) shallAddLBACFilters(req *http.Request) bool {
+// needAddLBACFilters decides if LBAC filters must be added to the outgoing request.
+func (s *AuthServer) needAddLBACFilters(urlPath string) bool {
if !s.accessControl.isEnabled() {
return false
}
for _, p := range lbacPrefixes {
- if strings.HasPrefix(req.URL.Path, p) {
+ if strings.HasPrefix(urlPath, p) {
return true
}
}
@@ -540,7 +430,7 @@ func (s *AuthServer) getLBACFilters(ctx context.Context, userID int) ([]string,
s.metrics.mDurations.WithLabelValues("db").Observe(time.Since(start).Seconds())
}()
- roles, err := models.GetUserRoles(s.db.Querier, userID)
+ roles, err := models.GetUserRoles(s.db.WithContext(ctx), userID)
if err != nil {
return nil, err
}
@@ -564,14 +454,15 @@ func (s *AuthServer) getLBACFilters(ctx context.Context, userID int) ([]string,
}
// Reload roles
- roles, err = models.GetUserRoles(s.db.Querier, userID)
+ roles, err = models.GetUserRoles(s.db.WithContext(ctx), userID)
if err != nil {
return nil, err
}
}
if len(roles) == 0 {
- logrus.Panicf("User %d has no roles", userID)
+ logrus.Errorf("User %d has no roles", userID)
+ return nil, fmt.Errorf("user %d has no roles", userID)
}
filters := make([]string, 0, len(roles))
@@ -588,215 +479,179 @@ func (s *AuthServer) getLBACFilters(ctx context.Context, userID int) ([]string,
return filters, nil
}
-// extractOriginalRequest replaces req.Method and req.URL.Path with values from original request.
-// Error is returned if original request information is missing or invalid.
-func extractOriginalRequest(req *http.Request) error {
- origMethod, origURI := req.Header.Get("X-Original-Method"), req.Header.Get("X-Original-Uri")
-
- if origMethod == "" {
- return errors.New("empty X-Original-Method")
- }
+// processRequest checks if user has access to a specific path.
+// It returns user information retrieved during authentication.
+// Paths which require no Grafana role return zero value for
+// some user fields such as authUser.userID.
+// This func expects that req.Method and req.URL.Path are already replaced
+// with original request values - extractOriginalRequest(req) has been called beforehand.
+func (s *AuthServer) processRequest(ctx context.Context, req *http.Request, l *logrus.Entry) (authResult, error) {
+ // Determine the minimal required role for the (already cleaned) original request path.
+ minRole, prefix := resolveRule(req.Method, req.URL.Path, l)
+ l = l.WithField("prefix", prefix)
- if origURI == "" {
- return errors.New("empty X-Original-Uri")
- }
- if origURI[0] != '/' {
- return fmt.Errorf("unexpected X-Original-Uri: %q", origURI)
- }
- if !utf8.ValidString(origURI) {
- return fmt.Errorf("invalid X-Original-Uri: %q", origURI)
+ needLbacFilter := s.needAddLBACFilters(req.URL.Path)
+ if minRole == none && !needLbacFilter {
+ l.WithField("path", req.URL.Path).Debugf("Minimum required role is %s, granting access without authentication.", minRole)
+ return authResult{}, nil
}
- cleanedOrigURI, err := cleanPath(origURI)
+ user, err := s.authenticateUser(req, l) //nolint:contextcheck
if err != nil {
- return fmt.Errorf("failed to unescape path %q: %w", origURI, err)
- }
-
- req.Method = origMethod
- req.URL.Path = cleanedOrigURI
- return nil
-}
-
-// nextPrefix returns path's prefix, stopping on slashes, dots, and colons, e.g.:
-// /inventory.Nodes/ListNodes -> /inventory.Nodes/ -> /inventory.Nodes -> /inventory. -> /inventory -> /
-// /v1/inventory/Nodes/List -> /v1/inventory/Nodes/ -> /v1/inventory/Nodes -> /v1/inventory/ -> /v1/inventory -> /v1/ -> /v1 -> /
-// That works for both gRPC and JSON URLs.
-// The chain ends with "/" no matter what.
-func nextPrefix(path string) string {
- if len(path) == 0 || path[0] != '/' || path == "/" {
- return "/"
- }
-
- if t := strings.TrimRight(path, "."); t != path {
- return t
+ l.WithError(err).Error("Failed to authenticate user.")
+ var zero authResult
+ return zero, err
}
- if t := strings.TrimRight(path, "/"); t != path {
- return t
- }
-
- if t := strings.TrimRight(path, ":"); t != path {
- return t
+ l = l.WithField("role", user.role.String())
+ err = authorizeUser(minRole, user, l)
+ if err != nil {
+ l.WithError(err).Error("Failed to authorize user.")
+ var zero authResult
+ return zero, err
}
- i := strings.LastIndexAny(path, "/.:")
- return path[:i+1]
-}
-
-// resolveRule returns the minimal role for the given method and path, plus the matched
-// prefix. It walks prefixes longest-to-shortest; a method-specific rule ("METHOD prefix")
-// beats a path-only rule at the same prefix, so read and write on a shared path can differ.
-// With no match it logs a warning and falls back to grafanaAdmin.
-func resolveRule(method, cleanedPath string, l *logrus.Entry) (role, string) {
- prefix := cleanedPath
- for {
- if r, ok := methodRules[method+" "+prefix]; ok {
- return r, prefix
- }
- if r, ok := rules[prefix]; ok {
- return r, prefix
- }
- if prefix == "/" {
- l.Warn("No explicit rule, falling back to Grafana admin.")
- return grafanaAdmin, prefix
+ var lbacFilters string
+ if needLbacFilter {
+ lbacFilters, err = s.addLBACFilters(ctx, user.userID, l)
+ if err != nil {
+ l.WithError(err).Error("Failed to add VMProxy LBAC filters.")
+ var zero authResult
+ return zero, errStaticAuthErrorInternalError
}
- prefix = nextPrefix(prefix)
- }
-}
-
-// isLocalAgentConnection reports whether the request is a local PMM agent
-// connection for endpoints that are allowed from localhost.
-// This func expects that req.Method and req.URL.Path are already replaced
-// with original request values - extractOriginalRequest(req) has been called beforehand.
-func isLocalAgentConnection(req *http.Request) bool {
- ip := strings.Split(req.RemoteAddr, ":")[0]
- // pmmAgent := req.Header.Get("Pmm-Agent-Id")
- path := req.URL.Path
- if ip == "127.0.0.1" &&
- (path == connectionEndpoint || path == rtaCollectEndpoint) {
- return true
}
- return false
+ return authResult{vmProxyFilters: lbacFilters}, nil
}
-// authenticate checks if user has access to a specific path.
-// It returns user information retrieved during authentication.
-// Paths which require no Grafana role return zero value for
-// some user fields such as authUser.userID.
-// This func expects that req.Method and req.URL.Path are already replaced
-// with original request values - extractOriginalRequest(req) has been called beforehand.
-func (s *AuthServer) authenticate(ctx context.Context, req *http.Request, l *logrus.Entry) (*authUser, *authError) {
- // Determine the minimal required role for the (already cleaned) original request path.
- minRole, prefix := resolveRule(req.Method, req.URL.Path, l)
- l = l.WithField("prefix", prefix)
-
- if minRole == none {
- l.Debugf("Minimal required role is %s, granting access without checking Grafana.", minRole)
- return nil, nil
- }
+var (
+ // Holds mapping of local agent connect endpoints to *authUser.
+ staticAuthUsers = map[string]authUser{
+ connectionEndpoint: {role: rules[connectionEndpoint], userID: 0},
+ connectionEndpointV2: {role: rules[connectionEndpointV2], userID: 0},
+ rtaCollectEndpoint: {role: rules[rtaCollectEndpoint], userID: 0},
+ }
+ errStaticAuthErrorPermissionDenied = &authError{code: codes.PermissionDenied, message: "Access denied."}
+ errStaticAuthErrorInternalError = &authError{code: codes.Internal, message: "Internal server error."}
+)
- var user *authUser
+// authenticateUser performs identity/authentication only.
+func (s *AuthServer) authenticateUser(req *http.Request, l *logrus.Entry) (authUser, error) {
if isLocalAgentConnection(req) {
- if req.URL.Path == connectionEndpoint {
- user = &authUser{
- role: rules[connectionEndpoint],
- userID: 0,
- }
- } else {
- user = &authUser{
- role: rules[rtaCollectEndpoint],
- userID: 0,
- }
- }
- } else {
- var authErr *authError
- // Get authenticated user from Grafana
- user, authErr = s.getAuthUser(ctx, req, l)
- if authErr != nil {
- return nil, authErr
+ user, ok := staticAuthUsers[req.URL.Path]
+ if ok {
+ return user, nil
}
+ var zero authUser
+ return zero, errStaticAuthErrorPermissionDenied
}
- l = l.WithField("role", user.role.String())
+ // Non-local requests require user info retrieval from Grafana.
+ return s.getAuthUser(req, l)
+}
+// authorizeUser performs role check only.
+func authorizeUser(minRole role, user authUser, l *logrus.Entry) error {
if user.role == grafanaAdmin {
l.Debugf("Grafana admin, granting access.")
- return user, nil
+ return nil
}
if minRole <= user.role {
l.Debugf("Minimal required role is %s, granting access.", minRole)
- return user, nil
+ return nil
}
l.Warnf("Minimal required role is %s, denying access.", minRole)
- return nil, &authError{code: codes.PermissionDenied, message: "Access denied"}
+ return errStaticAuthErrorPermissionDenied
}
-func cleanPath(p string) (string, error) {
- unescaped, err := url.PathUnescape(p)
- if err != nil {
- return "", err
+// getAuthUser retrieves user information from cache (if exists) based on the request's authentication headers,
+// otherwise from Grafana.
+func (s *AuthServer) getAuthUser(req *http.Request, l *logrus.Entry) (authUser, error) {
+ // Marginally faster than req.Header.Get("...")
+ var authorization, cookie string
+ if vals := req.Header["Authorization"]; len(vals) > 0 {
+ authorization = vals[0]
+ }
+ if vals := req.Header["Cookie"]; len(vals) > 0 {
+ cookie = vals[0]
}
- cleanedPath := path.Clean(unescaped)
-
- cleanedPath = strings.ReplaceAll(cleanedPath, "\n", " ")
+ authCacheKey := getAuthCacheKey(req)
- u, err := url.Parse(cleanedPath)
- if err != nil {
- return "", err
+ // Hot-path: lookup user in cache first.
+ if cached, ok := s.cache.Get(authCacheKey); ok {
+ // Verify auth headers for this hash to prevent serving wrong user on rare hash collisions.
+ if cached.authorization == authorization && cached.cookie == cookie {
+ s.incCacheHit()
+ return cached.user, nil
+ }
}
- u.RawQuery = ""
- return u.String(), nil
-}
+ s.incCacheMiss()
-func (s *AuthServer) getAuthUser(ctx context.Context, req *http.Request, l *logrus.Entry) (*authUser, *authError) {
- // check Grafana with some headers from request
- authHeaders := s.authHeaders(req)
- j, err := json.Marshal(authHeaders)
- if err != nil {
- l.Warnf("%s", err)
- return nil, &authError{code: codes.Internal, message: "Internal server error."}
- }
- hash := base64.StdEncoding.EncodeToString(j)
- s.rw.RLock()
- item, ok := s.cache[hash]
- s.rw.RUnlock()
- // Check the item's age on read: the background invalidator runs only once per
- // cacheInvalidationInterval, so without this an entry could be served for almost
- // twice that long. Re-fetch once an entry is older than the interval.
- if ok && time.Since(item.created) < cacheInvalidationInterval {
- s.incCacheHit()
- return &item.u, nil
- }
+ // Cold-path: Cache miss / Stale data.
+ // Use single-flight to avoid calling Grafana for the same user's authHeaders.
+ // It appears when after restart the same vm-agent starts to send buffered metrics
+ // to server in parallel and all such requests have to be authenticated.
+ res, err, _ := s.authUserGroup.Do(authCacheKey, func() (any, error) {
+ // Recheck inside singleflight to avoid duplicate upstream calls when
+ // another goroutine already populated cache while we were waiting.
+ if cached, ok := s.cache.Get(authCacheKey); ok {
+ if cached.authorization == authorization && cached.cookie == cookie {
+ return cached.user, nil
+ }
+ }
- s.incCacheMiss()
- return s.retrieveRole(ctx, hash, authHeaders, l)
-}
+ // NOTE 1: leader's context is not used here directly in order to allow to finish
+ // the request to Grafana, so that even if leader's request is already terminated -
+ // the rest of waiters in singleflight group will receive the response from Grafana.
+ grafanaCtx, cancel := context.WithTimeout(context.Background(), authenticationTimeout)
+ defer cancel()
-func (s *AuthServer) authHeaders(req *http.Request) http.Header {
- authHeaders := make(http.Header)
- for _, k := range []string{
- "Authorization",
- "Cookie",
- } {
- if v := req.Header.Get(k); v != "" {
- authHeaders.Set(k, v)
+ userAuthInfo, authErr := s.getGrafanaAuthUser(grafanaCtx, extractAuthHeaders(req), l)
+ if authErr != nil {
+ // IMPORTANT: On error, we call Forget(hash) IMMEDIATELY.
+ // This prevents the error from getting stuck in the internal singleflight map
+ // and allows the next request to retry immediately (e.g. if the Grafana is being restored).
+ s.authUserGroup.Forget(authCacheKey)
+ return nil, authErr
}
+
+ // Store the retrieved user info in cache for future requests.
+ s.cache.Set(authCacheKey, cachedAuthUser{
+ user: userAuthInfo,
+ authorization: authorization,
+ cookie: cookie,
+ })
+ return userAuthInfo, nil
+ })
+ if err != nil {
+ l.WithError(err).Error("Grafana user lookup failed.")
+ var zero authUser
+ return zero, err
}
- return authHeaders
+
+ user, ok := res.(authUser)
+ if !ok {
+ l.WithField("type", fmt.Sprintf("%T", res)).Error("Unexpected Grafana user result type.")
+ var zero authUser
+ return zero, errStaticAuthErrorInternalError
+ }
+
+ return user, nil
}
-func (s *AuthServer) retrieveRole(ctx context.Context, hash string, authHeaders http.Header, l *logrus.Entry) (*authUser, *authError) {
+// getGrafanaAuthUser calls Grafana to retrieve user's info. Passed authHeaders are used for authentication.
+func (s *AuthServer) getGrafanaAuthUser(ctx context.Context, authHeaders http.Header, l *logrus.Entry) (authUser, error) {
start := time.Now()
defer func() {
s.metrics.mDurations.WithLabelValues("grafana").Observe(time.Since(start).Seconds())
}()
- authUser, err := s.c.getAuthUser(ctx, authHeaders, l)
+ authUserInfo, err := s.c.getAuthUser(ctx, authHeaders, l)
if err != nil {
- l.Warnf("%s", err)
+ var zero authUser
+ l.WithError(err).Error("Failed to retrieve user info from Grafana.")
cErr, ok := errors.AsType[*clientError](err)
if ok {
s.incGrafanaAuthRequests(cErr.Code)
@@ -805,20 +660,14 @@ func (s *AuthServer) retrieveRole(ctx context.Context, hash string, authHeaders
if cErr.Code == http.StatusUnauthorized || cErr.Code == http.StatusForbidden {
code = codes.Unauthenticated
}
- return nil, &authError{code: code, message: cErr.ErrorMessage}
+ return zero, &authError{code: code, message: cErr.ErrorMessage}
}
s.incGrafanaAuthRequests(http.StatusInternalServerError)
- return nil, &authError{code: codes.Internal, message: "Internal server error."}
- }
- s.rw.Lock()
- s.cache[hash] = cacheItem{
- u: authUser,
- created: time.Now(),
+ return zero, errStaticAuthErrorInternalError
}
- s.rw.Unlock()
s.incGrafanaAuthRequests(http.StatusOK)
- return &authUser, nil
+ return authUserInfo, nil
}
var _ prom.Collector = (*AuthServer)(nil)
diff --git a/managed/services/grafana/auth_server_bench_test.go b/managed/services/grafana/auth_server_bench_test.go
new file mode 100644
index 00000000000..8ce4f2e7e1f
--- /dev/null
+++ b/managed/services/grafana/auth_server_bench_test.go
@@ -0,0 +1,148 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package grafana
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "testing"
+
+ "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/mock"
+)
+
+func BenchmarkAuthServerAuthenticateUser(b *testing.B) {
+ l := logrus.WithField("benchmark", b.Name())
+
+ b.Run("localhost static endpoint", func(b *testing.B) {
+ s := NewAuthServer(b.Context(), newMockGrafanaAuthUserGetter(b), nil)
+ req := httptest.NewRequestWithContext(b.Context(), http.MethodPost, connectionEndpoint, nil)
+ req.RemoteAddr = "127.0.0.1:12345"
+
+ b.ReportAllocs()
+ for b.Loop() {
+ got, authErr := s.authenticateUser(req, l)
+ if authErr != nil {
+ b.Fatalf("authenticateUser returned error: %v", authErr)
+ }
+ if got == (authUser{}) {
+ b.Fatal("authenticateUser returned zero user")
+ }
+ }
+ })
+
+ b.Run("remote cache hit", func(b *testing.B) {
+ grafanaMock := newMockGrafanaAuthUserGetter(b)
+ s := NewAuthServer(b.Context(), grafanaMock, nil)
+ req := httptest.NewRequestWithContext(b.Context(), http.MethodGet, "/v1/server/settings", nil)
+ req.RemoteAddr = "10.0.0.1:443"
+ req.Header.Set("Authorization", "Bearer hit")
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: admin, userID: 42}, nil)
+
+ _, authErr := s.authenticateUser(req, l)
+ if authErr != nil {
+ b.Fatalf("warmup authenticateUser returned error: %v", authErr)
+ }
+
+ b.ReportAllocs()
+ for b.Loop() {
+ got, err := s.authenticateUser(req, l)
+ if err != nil {
+ b.Fatalf("authenticateUser returned error: %v", err)
+ }
+ if got == (authUser{}) {
+ b.Fatal("authenticateUser returned zero user")
+ }
+ }
+ })
+
+ b.Run("remote cache miss", func(b *testing.B) {
+ grafanaMock := newMockGrafanaAuthUserGetter(b)
+ s := NewAuthServer(b.Context(), grafanaMock, nil)
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: admin, userID: 42}, nil)
+
+ req := httptest.NewRequestWithContext(b.Context(), http.MethodGet, "/v1/server/settings", nil)
+ req.RemoteAddr = "10.0.0.1:4423"
+ req.Header.Set("Authorization", "Bearer ")
+
+ seq := 0
+ b.ReportAllocs()
+ for b.Loop() {
+ req.Header.Set("Authorization", "Bearer "+strconv.Itoa(seq))
+ seq++
+
+ got, err := s.authenticateUser(req, l)
+ if err != nil {
+ b.Fatalf("authenticateUser returned error: %v", err)
+ }
+ if got == (authUser{}) {
+ b.Fatal("authenticateUser returned zero user")
+ }
+ }
+ })
+}
+
+func BenchmarkAuthServerServeHTTP(b *testing.B) {
+ grafanaMock := newMockGrafanaAuthUserGetter(b)
+ accessControlMock := newMockAccessControl(b)
+ accessControlMock.On("isEnabled").Return(false).Maybe()
+ b.Cleanup(func() {
+ grafanaMock.AssertExpectations(b)
+ accessControlMock.AssertExpectations(b)
+ })
+
+ s := NewAuthServer(b.Context(), grafanaMock, nil)
+ s.accessControl = accessControlMock
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: admin, userID: 1001}, nil)
+
+ b.ReportAllocs()
+
+ for _, tc := range []struct {
+ name string
+ method string
+ path string
+ }{
+ {name: "method specific alerting write", method: http.MethodPut, path: "/v1/alerting/templates/template-id"},
+ {name: "metrics write path", method: http.MethodGet, path: "/victoriametrics/api/v1/write"},
+ {name: "query metrics path", method: http.MethodGet, path: "/graph/api/ds/query"},
+ {name: "server readyz path", method: http.MethodGet, path: "/v1/server/readyz"},
+ {name: "pmm agent connect path", method: http.MethodGet, path: "/agent.v1.AgentService/Connect"},
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ tokenSeq := 0
+ for b.Loop() {
+ req := httptest.NewRequestWithContext(b.Context(), http.MethodGet, "/auth_request", nil)
+ req.Header.Set("X-Original-Method", tc.method)
+ req.Header.Set("X-Original-Uri", tc.path)
+ req.Header.Set("Authorization", "Bearer "+strconv.Itoa(tokenSeq))
+ tokenSeq++
+
+ rw := httptest.NewRecorder()
+ s.ServeHTTP(rw, req)
+ if rw.Code != http.StatusOK {
+ b.Fatalf("unexpected status code: got %d, want %d", rw.Code, http.StatusOK)
+ }
+ }
+ })
+ }
+}
diff --git a/managed/services/grafana/auth_server_fuzz.go b/managed/services/grafana/auth_server_fuzz.go
index d72f17a776a..b69e3e58f25 100644
--- a/managed/services/grafana/auth_server_fuzz.go
+++ b/managed/services/grafana/auth_server_fuzz.go
@@ -45,7 +45,7 @@ func Fuzz(data []byte) int {
return 0
}
- _ = s.authenticate(context.Background(), req, logrus.NewEntry(logrus.StandardLogger()))
+ _ = s.processRequest(context.Background(), req, logrus.NewEntry(logrus.StandardLogger()))
return 1
}
diff --git a/managed/services/grafana/auth_server_test.go b/managed/services/grafana/auth_server_test.go
index da17a93088a..a97072ae3e8 100644
--- a/managed/services/grafana/auth_server_test.go
+++ b/managed/services/grafana/auth_server_test.go
@@ -18,402 +18,1000 @@ package grafana
import (
"encoding/base64"
"encoding/json"
+ "errors"
"fmt"
"net/http"
"net/http/httptest"
+ "strconv"
"testing"
"time"
- "github.com/google/uuid"
- "github.com/prometheus/client_golang/prometheus/testutil"
+ sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
- "google.golang.org/grpc/metadata"
"gopkg.in/reform.v1"
"gopkg.in/reform.v1/dialects/postgresql"
- "github.com/percona/pmm/managed/models"
- "github.com/percona/pmm/managed/utils/testdb"
- "github.com/percona/pmm/managed/utils/tests"
- "github.com/percona/pmm/utils/logger"
+ ucache "github.com/percona/pmm/utils/cache"
)
-func TestNextPrefix(t *testing.T) {
- for _, paths := range [][]string{
- {"/inventory.Nodes/ListNodes", "/inventory.Nodes/", "/inventory.Nodes", "/inventory.", "/inventory", "/", "/"},
- {"/v1/inventory/Nodes/List", "/v1/inventory/Nodes/", "/v1/inventory/Nodes", "/v1/inventory/", "/v1/inventory", "/v1/", "/v1", "/", "/"},
- {"/.x", "/.", "/", "/"},
- {".", "/", "/"},
- {"./", "/", "/"},
- {"hax0r", "/", "/"},
- {"", "/"},
- {"/v1/server/AWSInstanceCheck/..%2f..%2finventory/Services/List'"},
+// newTestAuthServer creates an AuthServer with access-control cache disabled,
+// so tests never trigger DB reload unless they explicitly need it.
+func newTestAuthServer(t *testing.T) (*AuthServer, *mockGrafanaAuthUserGetter, sqlmock.Sqlmock) {
+ t.Helper()
+
+ grafanaMock := newMockGrafanaAuthUserGetter(t)
+ accessControlMock := newMockAccessControl(t)
+ accessControlMock.On("isEnabled").Return(false).Maybe()
+
+ sqlDB, sqlMock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ require.NoError(t, sqlMock.ExpectationsWereMet())
+ _ = sqlDB.Close()
+ grafanaMock.AssertExpectations(t)
+ accessControlMock.AssertExpectations(t)
+ })
+
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+ s := NewAuthServer(t.Context(), grafanaMock, db)
+ s.accessControl = accessControlMock
+
+ return s, grafanaMock, sqlMock
+}
+
+func newOriginalReq(t *testing.T, method, path string) *http.Request {
+ t.Helper()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/auth_request", nil)
+ req.Header.Set("X-Original-Method", method)
+ req.Header.Set("X-Original-Uri", path)
+ return req
+}
+
+func setupLBACServer(t *testing.T) (*AuthServer, *mockGrafanaAuthUserGetter, sqlmock.Sqlmock) {
+ t.Helper()
+ s, grafanaMock, sqlMock := newTestAuthServer(t)
+ accessControlMock := newMockAccessControl(t)
+ s.accessControl = accessControlMock
+ accessControlMock.On("isEnabled").Return(true).Maybe()
+ t.Cleanup(func() {
+ accessControlMock.AssertExpectations(t)
+ })
+ return s, grafanaMock, sqlMock
+}
+
+func cacheSize(s *AuthServer) int64 {
+ return s.cache.Size()
+}
+
+func requireAuthErrorCode(t *testing.T, err error, want codes.Code) {
+ t.Helper()
+
+ if authErr, ok := errors.AsType[*authError](err); ok {
+ assert.Equal(t, want, authErr.code)
+ return
+ }
+
+ require.Failf(t, "expected authError", "unexpected error type: %T", err)
+}
+
+func roleRows(rows ...struct {
+ id uint32
+ title string
+ filter string
+},
+) *sqlmock.Rows {
+ r := sqlmock.NewRows([]string{"id", "title", "description", "filter", "created_at", "updated_at"})
+ now := time.Now().UTC()
+ for _, row := range rows {
+ r = r.AddRow(row.id, row.title, "", row.filter, now, now)
+ }
+ return r
+}
+
+func TestStatusCodeToString(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ name string
+ code int
+ want string
+ }{
+ {name: "ok", code: http.StatusOK, want: "200"},
+ {name: "bad request", code: http.StatusBadRequest, want: "400"},
+ {name: "unauthorized", code: http.StatusUnauthorized, want: "401"},
+ {name: "forbidden", code: http.StatusForbidden, want: "403"},
+ {name: "not found", code: http.StatusNotFound, want: "404"},
+ {name: "method not allowed", code: http.StatusMethodNotAllowed, want: "405"},
+ {name: "request timeout", code: http.StatusRequestTimeout, want: "408"},
+ {name: "too many requests", code: http.StatusTooManyRequests, want: "429"},
+ {name: "internal", code: http.StatusInternalServerError, want: "500"},
+ {name: "service unavailable", code: http.StatusServiceUnavailable, want: "503"},
+ {name: "unknown", code: http.StatusTeapot, want: strconv.Itoa(http.StatusTeapot)},
} {
- t.Run(paths[0], func(t *testing.T) {
- for i, path := range paths[:len(paths)-1] {
- tests.AddToFuzzCorpus(t, "", []byte(path))
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, tc.want, statusCodeToString(tc.code))
+ })
+ }
+}
- expected := paths[i+1]
- actual := nextPrefix(path)
- assert.Equal(t, expected, actual, "path = %q", path)
- }
+func TestAuthErrorError(t *testing.T) {
+ t.Parallel()
+
+ err := authError{code: codes.PermissionDenied, message: errStaticAuthErrorPermissionDenied.message}
+ assert.Equal(t, fmt.Sprintf("%s: %s", errStaticAuthErrorPermissionDenied.message, codes.PermissionDenied), err.Error())
+}
+
+func TestHTTPStatusForAuthError(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ name string
+ code codes.Code
+ want int
+ }{
+ {name: "permission denied", code: codes.PermissionDenied, want: http.StatusForbidden},
+ {name: "unauthenticated", code: codes.Unauthenticated, want: authenticationErrorCode},
+ {name: "internal", code: codes.Internal, want: authenticationErrorCode},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+ assert.Equal(t, tc.want, convertAuthErrorToHTTPStatus(tc.code))
})
}
}
-func TestResolveRule(t *testing.T) {
+func TestAuthServerNeedAddLBACFilters(t *testing.T) {
+ t.Parallel()
+
+ t.Run("disabled LBAC - lbacPrefixes", func(t *testing.T) {
+ t.Parallel()
+
+ s, _, _ := newTestAuthServer(t)
+ for _, prefix := range lbacPrefixes {
+ t.Run(prefix, func(t *testing.T) {
+ t.Parallel()
+ assert.False(t, s.needAddLBACFilters(prefix))
+ })
+ }
+ })
+
+ t.Run("disabled LBAC - other prefixes", func(t *testing.T) {
+ t.Parallel()
+
+ s, _, _ := newTestAuthServer(t)
+ for _, prefix := range []string{
+ "/v1/server/settings",
+ "/inventory.",
+ "/v1/advisors/checks:",
+ "/v1/alerting",
+ "/v1/users/current",
+ "/v1/realtimeanalytics/sessions:start",
+ } {
+ t.Run(prefix, func(t *testing.T) {
+ t.Parallel()
+ assert.False(t, s.needAddLBACFilters(prefix))
+ })
+ }
+ })
+
+ t.Run("enabled LBAC - lbacPrefixes", func(t *testing.T) {
+ t.Parallel()
+ c := newMockGrafanaAuthUserGetter(t)
+ ac := newMockAccessControl(t)
+ ac.On("isEnabled").Return(true).Maybe()
+ t.Cleanup(func() {
+ c.AssertExpectations(t)
+ ac.AssertExpectations(t)
+ })
+
+ s, _, _ := setupLBACServer(t)
+ for _, prefix := range lbacPrefixes {
+ t.Run(prefix, func(t *testing.T) {
+ t.Parallel()
+ assert.True(t, s.needAddLBACFilters(prefix))
+ })
+ }
+ })
+
+ t.Run("enabled LBAC - other prefixes", func(t *testing.T) {
+ t.Parallel()
+
+ s, _, _ := setupLBACServer(t)
+ for _, prefix := range []string{
+ "/v1/server/settings",
+ "/inventory.",
+ "/v1/advisors/checks:",
+ "/v1/alerting",
+ "/v1/users/current",
+ "/v1/realtimeanalytics/sessions:start",
+ } {
+ t.Run(prefix, func(t *testing.T) {
+ t.Parallel()
+ assert.False(t, s.needAddLBACFilters(prefix))
+ })
+ }
+ })
+}
+
+func TestAuthServerGetLBACFilters(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
- method string
- path string
- wantRole role
+ name string
+ userID int
+ setupMock func(sqlmock.Sqlmock, int)
+ want []string
+ wantErr error
}{
- // Alerting: only listing templates is viewable; writes need editor.
- {http.MethodGet, "/v1/alerting/templates", viewer}, // ListTemplates
- {http.MethodPost, "/v1/alerting/templates", editor}, // CreateTemplate
- {http.MethodPut, "/v1/alerting/templates/foo", editor}, // UpdateTemplate
- {http.MethodDelete, "/v1/alerting/templates/foo", editor}, // DeleteTemplate
- {http.MethodPost, "/v1/alerting/rules", editor}, // CreateRule
- // No matching rule falls back to grafanaAdmin.
- {http.MethodGet, "/v1/unknown", grafanaAdmin},
+ {
+ name: "returns all filters when user has restricted roles",
+ userID: 1001,
+ setupMock: func(m sqlmock.Sqlmock, userID int) {
+ m.ExpectQuery("SELECT").WithArgs(userID).WillReturnRows(roleRows(
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 1, title: "Role A", filter: "filter-a"},
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 2, title: "Role B", filter: "filter-b"},
+ ))
+ },
+ want: []string{"filter-a", "filter-b"},
+ },
+ {
+ name: "returns empty slice when any role has empty filter",
+ userID: 1002,
+ setupMock: func(m sqlmock.Sqlmock, userID int) {
+ m.ExpectQuery("SELECT").WithArgs(userID).WillReturnRows(roleRows(
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 3, title: "Role Full Access", filter: ""},
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 4, title: "Role Ignored", filter: "filter-b"},
+ ))
+ },
+ want: []string{},
+ },
+ {
+ name: "returns error when roles query fails",
+ userID: 1003,
+ setupMock: func(m sqlmock.Sqlmock, userID int) {
+ m.ExpectQuery("SELECT").WithArgs(userID).WillReturnError(assert.AnError)
+ },
+ wantErr: assert.AnError,
+ },
} {
- t.Run(fmt.Sprintf("%s %s", tc.method, tc.path), func(t *testing.T) {
+ t.Run(tc.name, func(t *testing.T) {
t.Parallel()
+ s, _, sqlMock := newTestAuthServer(t)
+ tc.setupMock(sqlMock, tc.userID)
- got, _ := resolveRule(tc.method, tc.path, logrus.WithField("test", t.Name()))
- assert.Equal(t, tc.wantRole, got)
+ got, err := s.getLBACFilters(t.Context(), tc.userID)
+ if tc.wantErr != nil {
+ require.ErrorIs(t, err, tc.wantErr)
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, tc.want, got)
})
}
}
-func TestAuthServerAuthenticate(t *testing.T) {
+func TestAuthServerAddLBACFilters(t *testing.T) {
t.Parallel()
- ctx := t.Context()
- c := NewClient("127.0.0.1:3000")
- s := NewAuthServer(c, nil)
+ log := logrus.WithField("test", t.Name())
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/dummy", nil)
- require.NoError(t, err)
- req.SetBasicAuth("admin", "admin")
- authHeaders := req.Header
-
- t.Run("GrafanaAdminFallback", func(t *testing.T) {
+ t.Run("anonymous user is allowed without filters", func(t *testing.T) {
t.Parallel()
+ s, _, _ := setupLBACServer(t)
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/foo", nil)
+ filters, err := s.addLBACFilters(t.Context(), 0, log)
require.NoError(t, err)
- req.SetBasicAuth("admin", "admin")
+ assert.Empty(t, filters)
+ })
- _, res := s.authenticate(ctx, req, logrus.WithField("test", t.Name()))
- assert.Nil(t, res)
+ t.Run("returns error when user has no assigned roles and default role assignment fails", func(t *testing.T) {
+ t.Parallel()
+ s, _, sqlMock := setupLBACServer(t)
+ sqlMock.ExpectQuery("SELECT").WithArgs(1003).WillReturnRows(roleRows())
+ sqlMock.ExpectBegin()
+ sqlMock.ExpectQuery("SELECT").WillReturnError(assert.AnError)
+ sqlMock.ExpectRollback()
+
+ encoded, err := s.addLBACFilters(t.Context(), 1003, log)
+ require.ErrorIs(t, err, assert.AnError)
+ require.Empty(t, encoded)
})
- t.Run("NoAnonymousAccess", func(t *testing.T) {
+ t.Run("encodes filters for request when user has restricted roles", func(t *testing.T) {
t.Parallel()
+ s, _, sqlMock := setupLBACServer(t)
+ sqlMock.ExpectQuery("SELECT").WithArgs(1001).WillReturnRows(roleRows(
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 1, title: "Role A", filter: "filter-a"},
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 2, title: "Role B", filter: "filter-b"},
+ ))
+
+ encoded, err := s.addLBACFilters(t.Context(), 1001, log)
+ require.NoError(t, err)
+ require.NotEmpty(t, encoded)
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/foo", nil)
+ decoded, err := base64.StdEncoding.DecodeString(encoded)
require.NoError(t, err)
- _, res := s.authenticate(ctx, req, logrus.WithField("test", t.Name()))
- assert.Equal(t, &authError{code: codes.Unauthenticated, message: "Unauthorized"}, res)
+ var parsed []string
+ require.NoError(t, json.Unmarshal(decoded, &parsed))
+ assert.Equal(t, []string{"filter-a", "filter-b"}, parsed)
})
- for uri, minRole := range rules {
- for _, role := range []role{viewer, editor, admin} {
- t.Run(fmt.Sprintf("uri=%s,minRole=%s,role=%s", uri, minRole, role), func(t *testing.T) {
- t.Parallel()
+ t.Run("does not encode filters when at least one role has full access", func(t *testing.T) {
+ t.Parallel()
+ s, _, sqlMock := setupLBACServer(t)
+ sqlMock.ExpectQuery("SELECT").WithArgs(1002).WillReturnRows(roleRows(
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 1, title: "Role A", filter: "filter-a"},
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 2, title: "Role B", filter: ""},
+ ))
+
+ encoded, err := s.addLBACFilters(t.Context(), 1002, log)
+ require.NoError(t, err)
+ require.Empty(t, encoded)
+ })
+}
- login := fmt.Sprintf("%s-%s-%d", minRole, role, time.Now().Nanosecond())
- userID, err := c.testCreateUser(ctx, login, role, authHeaders)
- require.NoError(t, err)
- require.NotZero(t, userID)
- if err != nil {
- defer func() {
- err = c.testDeleteUser(ctx, userID, authHeaders)
- require.NoError(t, err)
- }()
- }
+func TestAuthorizeUserAuthServer(t *testing.T) {
+ t.Parallel()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
- require.NoError(t, err)
- req.SetBasicAuth(login, login)
+ l := logrus.WithField("test", t.Name())
- _, res := s.authenticate(ctx, req, logrus.WithField("test", t.Name()))
- if minRole <= role {
- assert.Nil(t, res)
- } else {
- assert.Equal(t, &authError{code: codes.PermissionDenied, message: "Access denied"}, res)
- }
- })
- }
+ for _, tc := range []struct {
+ name string
+ minRole role
+ user authUser
+ wantErr *authError
+ }{
+ {name: "grafana admin", minRole: admin, user: authUser{role: grafanaAdmin}, wantErr: nil},
+ {name: "role allowed", minRole: viewer, user: authUser{role: editor}, wantErr: nil},
+ {name: "none role allowed on none route", minRole: none, user: authUser{role: none}, wantErr: nil},
+ {name: "role denied", minRole: admin, user: authUser{role: viewer}, wantErr: errStaticAuthErrorPermissionDenied},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ got := authorizeUser(tc.minRole, tc.user, l)
+ if tc.wantErr == nil {
+ require.NoError(t, got)
+ return
+ }
+ assert.Equal(t, tc.wantErr, got)
+ })
}
}
-func TestServerClientConnection(t *testing.T) {
+func TestAuthServerAuthenticateUser(t *testing.T) {
t.Parallel()
- ctx := t.Context()
- c := NewClient("127.0.0.1:3000")
- s := NewAuthServer(c, nil)
+ l := logrus.WithField("test", t.Name())
- t.Run("Basic auth - success", func(t *testing.T) {
+ t.Run("localhost static endpoints return configured static users", func(t *testing.T) {
t.Parallel()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, connectionEndpoint, nil)
- require.NoError(t, err)
- req.SetBasicAuth("admin", "admin")
+ s, _, _ := newTestAuthServer(t)
+ for _, path := range []string{connectionEndpoint, connectionEndpointV2, rtaCollectEndpoint} {
+ t.Run(path, func(t *testing.T) {
+ t.Parallel()
- _, authError := s.authenticate(ctx, req, logrus.WithField("test", t.Name()))
- assert.Nil(t, authError)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, path, nil)
+ req.RemoteAddr = "127.0.0.1:12345"
+
+ got, err := s.authenticateUser(req, l)
+ require.NoError(t, err)
+ assert.Equal(t, staticAuthUsers[path], got)
+ })
+ }
})
- // Beware: Five or more wrong tries will lock user with error message: "Invalid user or password".
- t.Run("Basic auth - fail", func(t *testing.T) {
+ t.Run("remote request to static endpoint uses grafana authentication", func(t *testing.T) {
t.Parallel()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, connectionEndpoint, nil)
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, connectionEndpoint, nil)
+ req.RemoteAddr = "10.10.10.10:443"
+ req.Header.Set("Authorization", "Bearer remote")
+
+ want := authUser{role: admin, userID: 77}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).Return(want, nil).Once()
+
+ got, err := s.authenticateUser(req, l)
require.NoError(t, err)
- req.SetBasicAuth("admin", "wrong")
+ assert.Equal(t, want, got)
+ })
- _, authError := s.authenticate(ctx, req, logrus.WithField("test", t.Name()))
- assert.Equal(t, codes.Unauthenticated, authError.code)
+ t.Run("localhost non-whitelisted path falls back to grafana auth", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/qan/", nil)
+ req.RemoteAddr = "127.0.0.1:12345"
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{}, &clientError{Code: http.StatusUnauthorized, ErrorMessage: http.StatusText(http.StatusUnauthorized)}).
+ Once()
+
+ got, err := s.authenticateUser(req, l)
+ assert.Equal(t, authUser{}, got)
+ require.Error(t, err)
+ requireAuthErrorCode(t, err, codes.Unauthenticated)
})
- t.Run("Token auth - success", func(t *testing.T) {
+ t.Run("remote request goes through grafana auth", func(t *testing.T) {
t.Parallel()
- nodeName := fmt.Sprintf("N1-%d", time.Now().UnixNano())
- headersMD := metadata.New(map[string]string{
- "Authorization": "Basic YWRtaW46YWRtaW4=",
- })
- ctx := metadata.NewIncomingContext(t.Context(), headersMD)
- _, serviceToken, err := c.CreateServiceAccount(ctx, nodeName, true)
- require.NoError(t, err)
- defer func() {
- warning, err := c.DeleteServiceAccount(ctx, nodeName, true)
- require.NoError(t, err)
- require.Empty(t, warning)
- }()
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/server/settings", nil)
+ req.RemoteAddr = "10.0.0.1:443"
+ req.Header.Set("Authorization", "Bearer ok")
+
+ want := authUser{role: admin, userID: 42}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).Return(want, nil).Once()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, connectionEndpoint, nil)
+ got, err := s.authenticateUser(req, l)
require.NoError(t, err)
- req.Header.Set("Authorization", "Bearer "+serviceToken)
+ assert.Equal(t, want, got)
+ })
- _, authError := s.authenticate(ctx, req, logrus.WithField("test", t.Name()))
- assert.Nil(t, authError)
+ t.Run("remote request with grafana failed auth returns unauthenticated", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/server/settings", nil)
+ req.RemoteAddr = "10.0.0.1:443"
+ req.Header.Set("Authorization", "Bearer broken")
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{}, &clientError{Code: http.StatusUnauthorized, ErrorMessage: http.StatusText(http.StatusUnauthorized)}).
+ Once()
+
+ got, err := s.authenticateUser(req, l)
+ assert.Equal(t, authUser{}, got)
+ require.Error(t, err)
+ requireAuthErrorCode(t, err, codes.Unauthenticated)
})
- t.Run("Token auth - fail", func(t *testing.T) {
+ t.Run("get empty user info for anonymous user", func(t *testing.T) {
t.Parallel()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, connectionEndpoint, nil)
- require.NoError(t, err)
- req.Header.Set("Authorization", "Bearer wrong")
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/qan/query", nil)
- _, authError := s.authenticate(ctx, req, logrus.WithField("test", t.Name()))
- assert.Equal(t, codes.Internal, authError.code)
+ userInfo := authUser{role: none, userID: 0}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(userInfo, nil).
+ Once()
+
+ got, err := s.authenticateUser(req, l)
+ require.NoError(t, err)
+ assert.Equal(t, userInfo, got)
+ // assert.True(t, len(s.cache) == 0, "cache should be empty on anonymous user")
})
}
-func TestAuthServerAddVMGatewayToken(t *testing.T) {
- ctx := logger.Set(t.Context(), t.Name())
- uuid.SetRand(&tests.IDReader{})
-
- sqlDB := testdb.Open(t, models.SetupFixtures, nil)
- db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+func TestAuthServerGetGrafanaAuthUser(t *testing.T) {
+ t.Parallel()
- defer func(t *testing.T) {
- t.Helper()
+ l := logrus.WithField("test", t.Name())
+ headers := http.Header{"Authorization": []string{"Bearer token"}}
- uuid.SetRand(nil)
+ for _, tc := range []struct {
+ name string
+ retUser authUser
+ retErr error
+ wantUser authUser
+ wantErr error
+ wantErrCode codes.Code
+ mustBeError func(*testing.T, error)
+ }{
+ {
+ name: "success",
+ retUser: authUser{role: editor, userID: 7},
+ wantUser: authUser{role: editor, userID: 7},
+ },
+ {
+ name: "unauthorized from grafana",
+ retErr: &clientError{Code: http.StatusUnauthorized, ErrorMessage: http.StatusText(http.StatusUnauthorized)},
+ wantErr: &authError{code: codes.Unauthenticated, message: http.StatusText(http.StatusUnauthorized)},
+ wantErrCode: codes.Unauthenticated,
+ },
+ {
+ name: "forbidden from grafana",
+ retErr: &clientError{Code: http.StatusForbidden, ErrorMessage: http.StatusText(http.StatusForbidden)},
+ wantErr: &authError{code: codes.Unauthenticated, message: http.StatusText(http.StatusForbidden)},
+ wantErrCode: codes.Unauthenticated,
+ },
+ {
+ name: "upstream internal keeps internal code",
+ retErr: &clientError{Code: http.StatusInternalServerError, ErrorMessage: http.StatusText(http.StatusInternalServerError)},
+ wantErr: &authError{code: codes.Internal, message: http.StatusText(http.StatusInternalServerError)},
+ wantErrCode: codes.Internal,
+ },
+ {
+ name: "generic error maps to static internal",
+ retErr: errors.New("boom"),
+ wantErr: errStaticAuthErrorInternalError,
+ mustBeError: func(t *testing.T, got error) {
+ t.Helper()
+ assert.Equal(t, errStaticAuthErrorInternalError, got)
+ },
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
- require.NoError(t, sqlDB.Close())
- }(t)
+ s, grafanaMock, _ := newTestAuthServer(t)
+ grafanaMock.On("getAuthUser", mock.Anything, headers, mock.Anything).Return(tc.retUser, tc.retErr).Once()
- c := NewClient("127.0.0.1:3000")
- s := NewAuthServer(c, db)
+ got, err := s.getGrafanaAuthUser(t.Context(), headers, l)
- roleA := models.Role{
- Title: "Role A",
- Filter: "filter A",
+ if tc.wantErr == nil {
+ require.NoError(t, err)
+ assert.Equal(t, tc.wantUser, got)
+ } else {
+ assert.Equal(t, authUser{}, got)
+ require.Error(t, err)
+ assert.Equal(t, tc.wantErr, err)
+ if tc.wantErrCode != 0 {
+ requireAuthErrorCode(t, err, tc.wantErrCode)
+ }
+ if tc.mustBeError != nil {
+ tc.mustBeError(t, err)
+ }
+ }
+ })
}
- err := models.CreateRole(db.Querier, &roleA)
- require.NoError(t, err)
+}
- roleB := models.Role{
- Title: "Role B",
- Filter: "filter B",
- }
- err = models.CreateRole(db.Querier, &roleB)
- require.NoError(t, err)
+func TestAuthServerGetAuthUser(t *testing.T) {
+ t.Parallel()
+
+ l := logrus.WithField("test", t.Name())
- roleC := models.Role{
- Title: "Role C",
- Filter: "",
+ mkReq := func(t *testing.T, auth string) *http.Request {
+ t.Helper()
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/server/settings", nil)
+ req.Header.Set("Authorization", auth)
+ return req
}
- err = models.CreateRole(db.Querier, &roleC)
- require.NoError(t, err)
- // Enable access control
- _, err = models.UpdateSettings(db.Querier, &models.ChangeSettingsParams{
- EnableAccessControl: new(true),
+ t.Run("cache hit uses cached user", func(t *testing.T) {
+ t.Parallel()
+
+ s, _, _ := newTestAuthServer(t)
+ req := mkReq(t, "Bearer cached")
+
+ hash := getAuthCacheKey(req)
+ s.cache.Set(hash, cachedAuthUser{user: authUser{role: viewer, userID: 11}, authorization: "Bearer cached"})
+
+ got, authErr := s.getAuthUser(req, l)
+ require.NoError(t, authErr)
+ assert.Equal(t, authUser{role: viewer, userID: 11}, got)
})
- require.NoError(t, err)
- for userID, roleIDs := range map[int][]int{
- 1337: {int(roleA.ID)},
- 1338: {int(roleA.ID), int(roleB.ID)},
- 1339: {int(roleA.ID), int(roleC.ID)},
- 1: {int(roleA.ID)},
- } {
- err := db.InTransaction(func(tx *reform.TX) error {
- return models.AssignRoles(tx, userID, roleIDs)
- })
+ t.Run("stale cache refreshes via grafana", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ shortTTLCache, err := ucache.NewCacheTTL[cachedAuthUser](t.Context(), time.Millisecond, time.Second)
require.NoError(t, err)
- }
+ s.cache = shortTTLCache
+ req := mkReq(t, "Bearer stale")
+
+ headers := extractAuthHeaders(req)
+ hash := getAuthCacheKey(req)
+ s.cache.Set(hash, cachedAuthUser{user: authUser{role: viewer, userID: 1}, authorization: "Bearer stale"})
+ time.Sleep(2 * time.Millisecond)
+
+ want := authUser{role: admin, userID: 99}
+ grafanaMock.On("getAuthUser", mock.Anything, headers, mock.Anything).Return(want, nil).Once()
+
+ got, authErr := s.getAuthUser(req, l)
+ require.NoError(t, authErr)
+ assert.Equal(t, want, got)
+ item, ok := s.cache.Get(hash)
+ require.True(t, ok)
+ assert.Equal(t, want, item.user)
+ })
- t.Run("shall properly evaluate adding filters", func(t *testing.T) {
- for uri, shallAdd := range map[string]bool{
- "/": false,
- "/dummy": false,
- "/prometheus/api/": false,
- "/prometheus/api/v1/": true,
- "/prometheus/api/v1/query": true,
- "/graph/api/datasources/uid": true,
- "/graph/api/ds/query": true,
- "/v1/qan/metrics:getFilters": true,
- "/v1/qan/query:exists": true,
- } {
- for _, userID := range []int{0, 1337, 1338} {
- t.Run(fmt.Sprintf("uri=%s userID=%d", uri, userID), func(t *testing.T) {
+ t.Run("cache miss calls grafana and caches response", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ token := "miss"
+ req := mkReq(t, "Bearer "+token)
+ headers := extractAuthHeaders(req)
+
+ want := authUser{role: editor, userID: 8}
+ grafanaMock.On("getAuthUser", mock.Anything, headers, mock.Anything).Return(want, nil).Once()
+
+ got, authErr := s.getAuthUser(req, l)
+ require.NoError(t, authErr)
+ assert.Equal(t, want, got)
+
+ hash := getAuthCacheKey(req)
+ item, ok := s.cache.Get(hash)
+ require.True(t, ok)
+ assert.Equal(t, want, item.user)
+ })
+
+ t.Run("grafana auth failure is returned", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := mkReq(t, "Bearer fail")
+ headers := extractAuthHeaders(req)
+
+ grafanaMock.On("getAuthUser", mock.Anything, headers, mock.Anything).
+ Return(authUser{}, &clientError{Code: http.StatusUnauthorized, ErrorMessage: http.StatusText(http.StatusUnauthorized)}).
+ Once()
+
+ got, authErr := s.getAuthUser(req, l)
+ assert.Equal(t, authUser{}, got)
+ require.Error(t, authErr)
+ requireAuthErrorCode(t, authErr, codes.Unauthenticated)
+ assert.Zero(t, cacheSize(s), "cache should be empty on auth failure")
+ })
+}
+
+func TestAuthServerProcessRequest(t *testing.T) {
+ t.Parallel()
+
+ l := logrus.WithField("test", t.Name())
+
+ t.Run("none role path bypasses auth", func(t *testing.T) {
+ t.Parallel()
+
+ s, _, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/server/readyz", nil)
+
+ res, authErr := s.processRequest(t.Context(), req, l)
+ require.NoError(t, authErr)
+ assert.Equal(t, authResult{}, res)
+ assert.Zero(t, cacheSize(s), "cache should be empty on none role path")
+ })
+
+ t.Run("authentication failure is propagated", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/server/settings", nil)
+ req.Header.Set("Authorization", "Bearer bad")
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{}, &clientError{Code: http.StatusUnauthorized, ErrorMessage: http.StatusText(http.StatusUnauthorized)}).
+ Once()
+
+ res, authErr := s.processRequest(t.Context(), req, l)
+ assert.Equal(t, authResult{}, res)
+ require.Error(t, authErr)
+ requireAuthErrorCode(t, authErr, codes.Unauthenticated)
+ assert.Zero(t, cacheSize(s), "cache should be empty on auth failure")
+ })
+
+ t.Run("authorization failure is propagated", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/server/settings", nil)
+ token := "viewer"
+ req.Header.Set("Authorization", "Bearer "+token)
+ userInfo := authUser{role: viewer, userID: 11}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(userInfo, nil).
+ Once()
+
+ res, authErr := s.processRequest(t.Context(), req, l)
+ assert.Equal(t, authResult{}, res)
+
+ hash := getAuthCacheKey(req)
+ item, ok := s.cache.Get(hash)
+ require.True(t, ok)
+ assert.Equal(t, userInfo, item.user)
+
+ assert.Equal(t, errStaticAuthErrorPermissionDenied, authErr)
+ })
+
+ t.Run("success returns auth result", func(t *testing.T) {
+ t.Parallel()
+
+ for uri, minRole := range rules {
+ for _, role := range []role{viewer, editor, admin} {
+ t.Run(fmt.Sprintf("uri=%s,minRole=%s,role=%s", uri, minRole, role), func(t *testing.T) {
t.Parallel()
- rw := httptest.NewRecorder()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
- require.NoError(t, err)
- if userID == 0 {
- req.SetBasicAuth("admin", "admin")
- }
- err = s.maybeAddLBACFilters(ctx, rw, req, userID, logrus.WithField("test", t.Name()))
- require.NoError(t, err)
+ s, grafanaMock, _ := newTestAuthServer(t)
+
+ token := fmt.Sprintf("%s-%s-%d", minRole, role, time.Now().Nanosecond())
- headerString := rw.Header().Get(lbacHeaderName)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, uri, nil)
+ req.Header.Set("Authorization", "Bearer "+token)
- if shallAdd {
- require.NotEmpty(t, headerString)
+ if minRole > none {
+ userInfo := authUser{role: role, userID: 99}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(userInfo, nil).
+ Once()
+ }
+
+ res, authErr := s.processRequest(t.Context(), req, l)
+ if minRole <= role {
+ require.NoError(t, authErr)
+ assert.Equal(t, authResult{}, res)
} else {
- require.Empty(t, headerString)
+ assert.Equal(t, errStaticAuthErrorPermissionDenied, authErr)
+ assert.Equal(t, authResult{}, res)
}
})
}
}
})
- //nolint:paralleltest
- t.Run("shall be a valid JSON array", func(t *testing.T) {
+ t.Run("access forbidden for anonymous user", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/qan/query", nil)
+
+ userInfo := authUser{role: none, userID: 0}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(userInfo, nil).
+ Once()
+
+ res, authErr := s.processRequest(t.Context(), req, l)
+ assert.Equal(t, errStaticAuthErrorPermissionDenied, authErr)
+ assert.Equal(t, authResult{}, res)
+ // assert.True(t, len(s.cache) == 0, "cache should be empty on anonymous user")
+ })
+
+ t.Run("access granted for anonymous user", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/qan", nil)
+
+ userInfo := authUser{role: viewer, userID: 0}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(userInfo, nil).
+ Once()
+
+ res, authErr := s.processRequest(t.Context(), req, l)
+ require.NoError(t, authErr)
+ assert.Empty(t, res.vmProxyFilters)
+ // assert.True(t, len(s.cache) == 0, "cache should be empty on anonymous user")
+ })
+
+ t.Run("access granted for anonymous user with LBAC enabled", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := setupLBACServer(t)
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/v1/qan", nil)
+
+ userInfo := authUser{role: viewer, userID: 0}
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(userInfo, nil).
+ Once()
+
+ res, authErr := s.processRequest(t.Context(), req, l)
+ require.NoError(t, authErr)
+ assert.Empty(t, res.vmProxyFilters)
+ // assert.True(t, len(s.cache) == 0, "cache should be empty on anonymous user")
+ })
+}
+
+func TestAuthServerServeHTTP(t *testing.T) {
+ t.Parallel()
+
+ t.Run("bad original request headers returns 400", func(t *testing.T) {
+ t.Parallel()
+
+ s, _, _ := newTestAuthServer(t)
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/auth_request", nil)
+
rw := httptest.NewRecorder()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/prometheus/api/v1/", nil)
- require.NoError(t, err)
+ s.ServeHTTP(rw, req)
+ assert.Equal(t, http.StatusBadRequest, rw.Code)
+ })
- err = s.maybeAddLBACFilters(ctx, rw, req, 1338, logrus.WithField("test", t.Name()))
- require.NoError(t, err)
+ t.Run("permission denied returns 403 status code", func(t *testing.T) {
+ t.Parallel()
- headerString := rw.Header().Get(lbacHeaderName)
- require.NotEmpty(t, headerString)
+ s, grafanaMock, _ := newTestAuthServer(t)
- filters, err := base64.StdEncoding.DecodeString(headerString)
- require.NoError(t, err)
- var parsed []string
- err = json.Unmarshal(filters, &parsed)
- require.NoError(t, err)
+ req := newOriginalReq(t, http.MethodGet, "/v1/server/settings")
+ req.Header.Set("Authorization", "Bearer viewer")
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: viewer, userID: 17}, nil).
+ Once()
+
+ rw := httptest.NewRecorder()
+ s.ServeHTTP(rw, req)
- require.Len(t, parsed, 2)
- require.Equal(t, "filter A", parsed[0])
- require.Equal(t, "filter B", parsed[1])
+ assert.Equal(t, http.StatusForbidden, rw.Code)
+ assert.Empty(t, rw.Body.String())
+ assert.Equal(t, strconv.Itoa(int(codes.PermissionDenied)), rw.Header().Get(authResponseCodeHeader))
+ assert.Equal(t, "Access denied.", rw.Header().Get(authResponseErrorHeader))
+ assert.Equal(t, "Access denied.", rw.Header().Get(authResponseMessageHeader))
})
- //nolint:paralleltest
- t.Run("shall not add any filters if at least one role has full access", func(t *testing.T) {
+ t.Run("success with disabled LBAC", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
+
+ req := newOriginalReq(t, http.MethodGet, "/prometheus/api/v1/query")
+ req.Header.Set("Authorization", "Bearer admin")
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: admin, userID: 1001}, nil).
+ Once()
+
rw := httptest.NewRecorder()
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/prometheus/api/v1/", nil)
- require.NoError(t, err)
+ s.ServeHTTP(rw, req)
+
+ assert.Equal(t, http.StatusOK, rw.Code)
+ header := rw.Header().Get(lbacHeaderName)
+ require.Empty(t, header)
+ })
+
+ t.Run("success with enabled LBAC", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, sqlMock := setupLBACServer(t)
+ sqlMock.ExpectQuery("SELECT").WithArgs(1001).WillReturnRows(roleRows(
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 1, title: "Role A", filter: "filter-a"},
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 2, title: "Role B", filter: "filter-b"},
+ ))
+
+ req := newOriginalReq(t, http.MethodGet, "/prometheus/api/v1/query")
+ req.Header.Set("Authorization", "Bearer admin")
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: admin, userID: 1001}, nil).
+ Once()
+
+ rw := httptest.NewRecorder()
+ s.ServeHTTP(rw, req)
- err = s.maybeAddLBACFilters(ctx, rw, req, 1339, logrus.WithField("test", t.Name()))
+ assert.Equal(t, http.StatusOK, rw.Code)
+ header := rw.Header().Get(lbacHeaderName)
+ require.NotEmpty(t, header)
+
+ decoded, err := base64.StdEncoding.DecodeString(header)
require.NoError(t, err)
+ var filters []string
+ require.NoError(t, json.Unmarshal(decoded, &filters))
+ assert.Equal(t, []string{"filter-a", "filter-b"}, filters)
+ })
+
+ t.Run("uses original method for method specific authorization", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, _ := newTestAuthServer(t)
- headerString := rw.Header().Get(lbacHeaderName)
- require.Empty(t, headerString)
+ req := newOriginalReq(t, http.MethodPost, "/v1/alerting/templates")
+ req.Header.Set("Authorization", "Bearer viewer")
+
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: viewer, userID: 17}, nil).
+ Once()
+
+ rw := httptest.NewRecorder()
+ s.ServeHTTP(rw, req)
+
+ assert.Equal(t, http.StatusForbidden, rw.Code)
+ assert.Empty(t, rw.Body.String())
+ assert.Equal(t, strconv.Itoa(int(codes.PermissionDenied)), rw.Header().Get(authResponseCodeHeader))
+ assert.Equal(t, "Access denied.", rw.Header().Get(authResponseErrorHeader))
+ assert.Equal(t, "Access denied.", rw.Header().Get(authResponseMessageHeader))
})
-}
-func TestCleanPath(t *testing.T) {
- t.Parallel()
- tests := []struct {
- path string
- expected string
- }{
- {
- "/v1/server/AWSInstanceCheck/..%2f..%2finventory/Services/List",
- "/v1/inventory/Services/List",
- }, {
- "/v1/server/AWSInstanceCheck/..%2f..%2f..%2fmanaged/logs.zip",
- "/managed/logs.zip",
- }, {
- "/v1/server/AWSInstanceCheck/..%2f..%2f..%2f/logs.zip",
- "/logs.zip",
- }, {
- "/graph/api/datasources/proxy/8/?query=WITH%20(%0A%20%20%20%20CASE%20%0A%20%20%20%20%20%20%20%20WHEN%20(3000%20%25%2060)%20%3D%200%20THEN%203000%0A%20%20%20%20ELSE%2060%20END%0A)%20AS%20scale%0ASELECT%0A%20%20%20%20(intDiv(toUInt32(timestamp)%2C%203000)%20*%203000)%20*%201000%20as%20t%2C%0A%20%20%20%20hostname%20h%2C%0A%20%20%20%20status%20s%2C%0A%20%20%20%20SUM(req_count)%20as%20req_count%0AFROM%20pinba.report_by_all%0AWHERE%0A%20%20%20%20timestamp%20%3E%3D%20toDateTime(1707139680)%20AND%20timestamp%20%3C%3D%20toDateTime(1707312480)%0A%20%20%20%20AND%20status%20%3E%3D%20400%0A%20%20%20%20AND%20CASE%20WHEN%20%27all%27%20%3C%3E%20%27all%27%20THEN%20schema%20%3D%20%27all%27%20ELSE%201%20END%0A%20%20%20%20AND%20CASE%20WHEN%20%27all%27%20%3C%3E%20%27all%27%20THEN%20hostname%20%3D%20%27all%27%20ELSE%201%20END%0A%20%20%20%20AND%20CASE%20WHEN%20%27all%27%20%3C%3E%20%27all%27%20THEN%20server_name%20%3D%20%27all%27%20ELSE%201%20END%0AGROUP%20BY%20t%2C%20h%2C%20s%0AORDER%20BY%20t%20FORMAT%20JSON",
- "/graph/api/datasources/proxy/8/",
- },
- }
- for _, tt := range tests {
- t.Run(tt.path, func(t *testing.T) {
- t.Parallel()
- cleanedPath, err := cleanPath(tt.path)
- require.NoError(t, err)
- assert.Equalf(t, tt.expected, cleanedPath, "cleanPath(%v)", tt.path)
- })
- }
-}
+ t.Run("none role route bypasses grafana authentication", func(t *testing.T) {
+ t.Parallel()
-func TestAuthServerServeHTTPBadRequestMetricsUsesCleanedRoute(t *testing.T) {
- t.Parallel()
+ s, _, _ := newTestAuthServer(t)
- s := NewAuthServer(nil, nil)
- rr := httptest.NewRecorder()
- req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/auth_request", nil)
+ req := newOriginalReq(t, http.MethodGet, "/v1/server/readyz")
- // Trigger extractOriginalRequest error (missing X-Original-Method),
- // but keep X-Original-Uri so ServeHTTP records a metric for it.
- req.Header.Set("X-Original-Uri", "/v1/server/AWSInstanceCheck/..%2f..%2f..%2f/logs.zip?foo=bar")
+ rw := httptest.NewRecorder()
+ s.ServeHTTP(rw, req)
- s.ServeHTTP(rr, req)
+ assert.Equal(t, http.StatusOK, rw.Code)
+ })
- require.Equal(t, http.StatusBadRequest, rr.Code)
+ t.Run("unauthenticated grafana response returns 401 payload", func(t *testing.T) {
+ t.Parallel()
- value := testutil.ToFloat64(s.metrics.mAuthRequests.WithLabelValues(http.MethodGet, "/logs.zip", "400"))
- require.InDelta(t, 1.0, value, 0.0, "expected auth request metric with cleaned route")
-}
+ s, grafanaMock, _ := newTestAuthServer(t)
-func TestAuthServerServeHTTPBadRequestMetricsFallbackToRawRouteOnCleanError(t *testing.T) {
- t.Parallel()
+ req := newOriginalReq(t, http.MethodGet, "/v1/server/settings")
+ req.Header.Set("Authorization", "Bearer broken")
- s := NewAuthServer(nil, nil)
- rr := httptest.NewRecorder()
- req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/auth_request", nil)
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{}, &clientError{Code: http.StatusUnauthorized, ErrorMessage: http.StatusText(http.StatusUnauthorized)}).
+ Once()
- // Invalid escape sequence keeps cleanPath from normalizing the path,
- // so ServeHTTP should use route value after query trimming.
- req.Header.Set("X-Original-Uri", "/bad%2?foo=bar")
+ rw := httptest.NewRecorder()
+ s.ServeHTTP(rw, req)
- s.ServeHTTP(rr, req)
+ assert.Equal(t, http.StatusUnauthorized, rw.Code)
+ assert.Empty(t, rw.Body.String())
+ assert.Equal(t, strconv.Itoa(int(codes.Unauthenticated)), rw.Header().Get(authResponseCodeHeader))
+ assert.Equal(t, http.StatusText(http.StatusUnauthorized), rw.Header().Get(authResponseErrorHeader))
+ assert.Equal(t, http.StatusText(http.StatusUnauthorized), rw.Header().Get(authResponseMessageHeader))
+ })
+
+ t.Run("enabled LBAC with full access role does not set proxy filter header", func(t *testing.T) {
+ t.Parallel()
+
+ s, grafanaMock, sqlMock := setupLBACServer(t)
+
+ sqlMock.ExpectQuery("SELECT").WithArgs(1002).WillReturnRows(roleRows(
+ struct {
+ id uint32
+ title string
+ filter string
+ }{id: 1, title: "Role A", filter: ""},
+ ))
- require.Equal(t, http.StatusBadRequest, rr.Code)
+ req := newOriginalReq(t, http.MethodGet, "/prometheus/api/v1/query")
+ req.Header.Set("Authorization", "Bearer admin")
- value := testutil.ToFloat64(s.metrics.mAuthRequests.WithLabelValues(http.MethodPost, "/bad%2", "400"))
- require.InDelta(t, 1.0, value, 0.0, "expected auth request metric with original route when cleaning fails")
+ grafanaMock.On("getAuthUser", mock.Anything, mock.Anything, mock.Anything).
+ Return(authUser{role: admin, userID: 1002}, nil).
+ Once()
+
+ rw := httptest.NewRecorder()
+ s.ServeHTTP(rw, req)
+
+ assert.Equal(t, http.StatusOK, rw.Code)
+ assert.Empty(t, rw.Header().Get(lbacHeaderName))
+ })
}
diff --git a/managed/services/grafana/deps.go b/managed/services/grafana/deps.go
index 39862b3980b..df67b7155f6 100644
--- a/managed/services/grafana/deps.go
+++ b/managed/services/grafana/deps.go
@@ -14,3 +14,18 @@
// along with this program. If not, see .
package grafana
+
+import (
+ "context"
+ "net/http"
+
+ "github.com/sirupsen/logrus"
+)
+
+// grafanaAuthUserGetter exist only to make fuzzing simpler.
+type grafanaAuthUserGetter interface {
+ getAuthUser(ctx context.Context, authHeaders http.Header, l *logrus.Entry) (authUser, error)
+}
+type accessControl interface {
+ isEnabled() bool
+}
diff --git a/managed/services/grafana/helpers.go b/managed/services/grafana/helpers.go
new file mode 100644
index 00000000000..bb71fa5e48a
--- /dev/null
+++ b/managed/services/grafana/helpers.go
@@ -0,0 +1,282 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package grafana
+
+import (
+ "errors"
+ "fmt"
+ "net/http"
+ "net/netip"
+ "net/url"
+ "path"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+
+ "github.com/sirupsen/logrus"
+ "google.golang.org/grpc/codes"
+)
+
+// statusCodeToString returns HTTP status code string presentation.
+func statusCodeToString(code int) string {
+ switch code {
+ case http.StatusOK:
+ return "200"
+ case http.StatusBadRequest:
+ return "400"
+ case http.StatusUnauthorized:
+ return "401"
+ case http.StatusForbidden:
+ return "403"
+ case http.StatusNotFound:
+ return "404"
+ case http.StatusMethodNotAllowed:
+ return "405"
+ case http.StatusRequestTimeout:
+ return "408"
+ case http.StatusTooManyRequests:
+ return "429"
+ case http.StatusInternalServerError:
+ return "500"
+ case http.StatusServiceUnavailable:
+ return "503"
+ default:
+ return strconv.Itoa(code)
+ }
+}
+
+// convertAuthErrorToHTTPStatus maps an authError code to the HTTP status nginx receives.
+// PermissionDenied uses 403 so nginx denies outright; the 401 re-run is a GET and would
+// wrongly pass method-specific rules. Authentication and internal errors stay 401.
+func convertAuthErrorToHTTPStatus(code codes.Code) int {
+ if code == codes.PermissionDenied {
+ return http.StatusForbidden
+ }
+ return authenticationErrorCode
+}
+
+// writeResponseErrorStatus sends an HTTP response header with the provided
+// status code and writes custom HTTP headers with auth error details.
+func writeResponseErrorStatus(rw http.ResponseWriter, status, authCode int, authError, authMessage string) {
+ // nginx ignores the auth_request subrequest body: we use custom HTTP headers
+ // to pass auth response error details to nginx.
+ rw.Header().Set(authResponseCodeHeader, strconv.Itoa(authCode))
+ rw.Header().Set(authResponseErrorHeader, authError)
+ rw.Header().Set(authResponseMessageHeader, authMessage)
+ rw.WriteHeader(status)
+}
+
+// extractOriginalRequest replaces req.Method and req.URL.Path with values from original request.
+// Error is returned if original request information is missing or invalid.
+func extractOriginalRequest(req *http.Request) error {
+ origMethod, origURI := req.Header.Get("X-Original-Method"), req.Header.Get("X-Original-Uri")
+
+ if origMethod == "" {
+ return errors.New("empty X-Original-Method")
+ }
+
+ if origURI == "" {
+ return errors.New("empty X-Original-Uri")
+ }
+
+ if origURI[0] != '/' {
+ return fmt.Errorf("unexpected X-Original-Uri: %q", origURI)
+ }
+
+ if !utf8.ValidString(origURI) {
+ return fmt.Errorf("invalid X-Original-Uri: %q", origURI)
+ }
+
+ cleanedOrigURI, err := cleanPath(origURI)
+ if err != nil {
+ return fmt.Errorf("failed to unescape path %q: %w", origURI, err)
+ }
+
+ req.Method = origMethod
+ req.URL.Path = cleanedOrigURI
+ return nil
+}
+
+// nextPrefix returns path's prefix, stopping on slashes, dots, and colons, e.g.:
+// /inventory.Nodes/ListNodes -> /inventory.Nodes/ -> /inventory.Nodes -> /inventory. -> /inventory -> /
+// /v1/inventory/Nodes/List -> /v1/inventory/Nodes/ -> /v1/inventory/Nodes -> /v1/inventory/ -> /v1/inventory -> /v1/ -> /v1 -> /
+// That works for both gRPC and JSON URLs.
+// The chain ends with "/" no matter what.
+func nextPrefix(path string) string {
+ if len(path) == 0 || path[0] != '/' || path == "/" {
+ return "/"
+ }
+
+ if t := strings.TrimRight(path, "."); t != path {
+ return t
+ }
+
+ if t := strings.TrimRight(path, "/"); t != path {
+ return t
+ }
+
+ if t := strings.TrimRight(path, ":"); t != path {
+ return t
+ }
+
+ i := strings.LastIndexAny(path, "/.:")
+ return path[:i+1]
+}
+
+// resolveRule returns the minimal role for the given method and path, plus the matched
+// prefix. It walks prefixes longest-to-shortest; a method-specific rule ("METHOD prefix")
+// beats a path-only rule at the same prefix, so read and write on a shared path can differ.
+// With no match it logs a warning and falls back to grafanaAdmin.
+func resolveRule(method, cleanedPath string, l *logrus.Entry) (role, string) {
+ prefix := cleanedPath
+ for {
+ if r, ok := methodRules[method+" "+prefix]; ok {
+ return r, prefix
+ }
+ if r, ok := rules[prefix]; ok {
+ return r, prefix
+ }
+ if prefix == "/" {
+ l.Warn("No explicit rule, falling back to Grafana admin.")
+ return grafanaAdmin, prefix
+ }
+ prefix = nextPrefix(prefix)
+ }
+}
+
+// isLocalAgentConnection reports whether the request is a local PMM agent
+// connection for endpoints that are allowed from localhost.
+// This func expects that req.Method and req.URL.Path are already replaced
+// with original request values - extractOriginalRequest(req) has been called beforehand.
+func isLocalAgentConnection(req *http.Request) bool {
+ path := req.URL.Path
+ if isLocalhostRemoteAddr(req.RemoteAddr) &&
+ (path == connectionEndpoint ||
+ path == connectionEndpointV2 ||
+ path == rtaCollectEndpoint) {
+ return true
+ }
+
+ return false
+}
+
+// isLocalhostRemoteAddr validates if an HTTP req.RemoteAddr originates from loopback.
+// Execution time: ~1-2ns (fast path) / ~15ns (fallback). Heap Allocations: 0.
+func isLocalhostRemoteAddr(remoteAddr string) bool {
+ // 1. Optimistic Fast Path (Branch Predictor friendly)
+ // Go's HTTP server canonically formats IPv4/IPv6 loopbacks as exactly these strings.
+ // The trailing colon (':') is critical to prevent matching IPs like "127.0.0.10:80".
+ // strings.HasPrefix is highly optimized, bypassing parsing overhead entirely.
+ if strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:") {
+ return true
+ }
+
+ // 2. Strict Semantic Parsing Fallback
+ // Catches edge cases like 127.0.0.2:port or IPv4-mapped IPv6 (::ffff:127.0.0.1:port).
+ // netip.ParseAddrPort is zero-allocation. It returns a stack-allocated struct (netip.AddrPort)
+ // without pointers, completely bypassing the Green Tea GC mark/sweep phases.
+ // Loopback traffic can originate from 127.0.0.2 (common in Kubernetes/mesh proxies),
+ // IPv6 ::1, or IPv4-mapped IPv6 addresses like ::ffff:127.0.0.1
+ ap, err := netip.ParseAddrPort(remoteAddr)
+ if err == nil {
+ return ap.Addr().IsLoopback()
+ }
+
+ return false
+}
+
+// cleanPath returns a clean, unescaped path from a raw URI.
+// It achieves 0 allocations if the path requires no modifications.
+func cleanPath(uri string) (string, error) {
+ // 1. Strip query parameters
+ if i := strings.IndexByte(uri, '?'); i >= 0 {
+ uri = uri[:i]
+ }
+
+ // 2. Fast-path check: scan for characters that require processing
+ needsWork := false
+ for i := range len(uri) {
+ c := uri[i]
+ // Check for URL encoding (%), dot\-segments (/\. or /\.\.), double slashes (//), or CR/LF.
+ if c == '%' || c == '\n' || c == '\r' || (c == '/' && i > 0 && uri[i-1] == '/') {
+ needsWork = true
+ break
+ }
+
+ if c == '.' && i > 0 && uri[i-1] == '/' {
+ // Match only dot\-segments, not dots inside normal segments (e.g. logs.zip).
+ if i+1 == len(uri) || uri[i+1] == '/' ||
+ (uri[i+1] == '.' && (i+2 == len(uri) || uri[i+2] == '/')) {
+ needsWork = true
+ break
+ }
+ }
+ }
+
+ // 3. Return zero-allocation slice if clean
+ if !needsWork {
+ return uri, nil
+ }
+
+ // 4. Slow-path: Allocate and process
+ unescaped, err := url.PathUnescape(uri)
+ if err != nil {
+ return "", err
+ }
+ unescaped = strings.ReplaceAll(unescaped, "\n", " ")
+ unescaped = strings.ReplaceAll(unescaped, "\r", " ")
+ return path.Clean(unescaped), nil
+}
+
+// extractAuthHeaders extracts auth info from request.
+func extractAuthHeaders(req *http.Request) http.Header {
+ // Marginally faster than req.Header.Get("...")
+ var authorization, cookie string
+ if vals := req.Header["Authorization"]; len(vals) > 0 {
+ authorization = vals[0]
+ }
+ if vals := req.Header["Cookie"]; len(vals) > 0 {
+ cookie = vals[0]
+ }
+
+ // Fast path: no auth headers -> no map allocation.
+ if authorization == "" && cookie == "" {
+ return nil
+ }
+
+ h := make(http.Header, 2) //nolint:mnd
+ if authorization != "" {
+ h.Set("Authorization", authorization)
+ }
+ if cookie != "" {
+ h.Set("Cookie", cookie)
+ }
+ return h
+}
+
+// getAuthCacheKey returns cache key directly from request auth headers.
+func getAuthCacheKey(req *http.Request) string {
+ // Marginally faster than req.Header.Get("...")
+ var authorization, cookie string
+ if vals := req.Header["Authorization"]; len(vals) > 0 {
+ authorization = vals[0]
+ }
+ if vals := req.Header["Cookie"]; len(vals) > 0 {
+ cookie = vals[0]
+ }
+
+ return authorization + ":" + cookie
+}
diff --git a/managed/services/grafana/helpers_bench_test.go b/managed/services/grafana/helpers_bench_test.go
new file mode 100644
index 00000000000..7357ff1ffc7
--- /dev/null
+++ b/managed/services/grafana/helpers_bench_test.go
@@ -0,0 +1,155 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package grafana
+
+import (
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/sirupsen/logrus"
+)
+
+func BenchmarkCleanPath(b *testing.B) {
+ const unescapedURI = "/v1/server/AWSInstanceCheck/..%2f..%2f..%2f..%2fgraph/api/datasources/proxy/8%2f%2f.%2f..%2f8%2f%2f?query=WITH%20CASE%20WHEN%203000%20x%2060%20THEN%203000%20ELSE%2060%20END%20SELECT%20hostname%2Cstatus%20FROM%20pinba.report_by_all%20WHERE%20timestamp%3E%3D1707139680%20AND%20timestamp%3C%3D1707312480%20ORDER%20BY%20t"
+ const expectedCleanPath = "/graph/api/datasources/proxy/8"
+
+ b.ReportAllocs()
+
+ // cleanedPath, err := cleanPath(unescapedURI)
+ // require.NoError(b, err)
+ // require.Equal(b, expectedCleanPath, cleanedPath)
+
+ b.ResetTimer()
+ for b.Loop() {
+ cleanedPath, err := cleanPath(unescapedURI)
+ if err != nil {
+ b.Fatalf("cleanPath returned error: %v", err)
+ }
+ if cleanedPath != expectedCleanPath {
+ b.Fatalf("unexpected cleaned path: got %q, want %q", cleanedPath, expectedCleanPath)
+ }
+ }
+}
+
+func BenchmarkAuthCacheKey(b *testing.B) {
+ b.ReportAllocs()
+
+ for _, tc := range []struct {
+ name string
+ set func(*http.Request)
+ }{
+ {
+ name: "authorization-only",
+ set: func(r *http.Request) {
+ r.Header.Set("Authorization", "Bearer token")
+ },
+ },
+ {
+ name: "cookie-only",
+ set: func(r *http.Request) {
+ r.Header.Set("Cookie", "grafana_session=abc")
+ },
+ },
+ {
+ name: "authorization-and-cookie",
+ set: func(r *http.Request) {
+ r.Header.Set("Authorization", "Bearer token")
+ r.Header.Set("Cookie", "grafana_session=abc")
+ },
+ },
+ {
+ name: "fallback-with-extra-header",
+ set: func(r *http.Request) {
+ r.Header.Set("Authorization", "Bearer token")
+ r.Header.Set("Cookie", "grafana_session=abc")
+ r.Header.Set("X-Extra", "1")
+ },
+ },
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ req := httptest.NewRequestWithContext(b.Context(), http.MethodGet, "/", nil)
+ tc.set(req)
+ for b.Loop() {
+ key := getAuthCacheKey(req)
+ if key == ":" {
+ b.Fatalf("authCacheKey returned empty key")
+ }
+ }
+ })
+ }
+}
+
+func BenchmarkResolveRule(b *testing.B) {
+ b.ReportAllocs()
+
+ logrus.SetOutput(io.Discard)
+ l := logrus.NewEntry(logrus.StandardLogger())
+ for _, tc := range []struct {
+ name string
+ method string
+ path string
+ }{
+ {name: "method specific alerting write", method: http.MethodPut, path: "/v1/alerting/templates/template-id"},
+ {name: "unknown path fallback", method: http.MethodGet, path: "/v1/not-found-endpoint"},
+ {name: "metrics write path", method: http.MethodPost, path: "/victoriametrics/api/v1/write"},
+ {name: "query metrics path", method: http.MethodGet, path: "/graph/api/ds/query"},
+ {name: "server readyz path", method: http.MethodGet, path: "/v1/server/readyz"},
+ {name: "pmm agent connect path", method: http.MethodPost, path: "/agent.v1.AgentService/Connect"},
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ for b.Loop() {
+ _, _ = resolveRule(tc.method, tc.path, l)
+ }
+ })
+ }
+}
+
+func BenchmarkIsLocalAgentConnection(b *testing.B) {
+ for _, tc := range []struct {
+ name string
+ remoteAddr string
+ path string
+ }{
+ // local IPv4
+ {name: "local connect endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: connectionEndpoint},
+ {name: "local connectV2 endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: connectionEndpointV2},
+ {name: "local rta endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: rtaCollectEndpoint},
+ {name: "local unknown endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: "/v1/server/version"},
+ // local IPv6
+ {name: "local connect endpoint IPv6", remoteAddr: "[::1]:12345", path: connectionEndpoint},
+ {name: "local connectV2 endpoint IPv6", remoteAddr: "[::1]:12345", path: connectionEndpointV2},
+ {name: "local rta endpoint IPv6", remoteAddr: "[::1]:12345", path: rtaCollectEndpoint},
+ {name: "local unknown endpoint IPv6", remoteAddr: "[::1]:12345", path: "/v1/server/version"},
+ // remote
+ {name: "remote connect endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: connectionEndpoint},
+ {name: "remote connectV2 endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: connectionEndpointV2},
+ {name: "remote rta endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: rtaCollectEndpoint},
+ {name: "remote unknown endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: "/v1/server/version"},
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ req := httptest.NewRequestWithContext(b.Context(), http.MethodGet, tc.path, nil)
+ req.RemoteAddr = tc.remoteAddr
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ _ = isLocalAgentConnection(req)
+ }
+ })
+ }
+}
diff --git a/managed/services/grafana/helpers_test.go b/managed/services/grafana/helpers_test.go
new file mode 100644
index 00000000000..25966a41efc
--- /dev/null
+++ b/managed/services/grafana/helpers_test.go
@@ -0,0 +1,493 @@
+// Copyright (C) 2023 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package grafana
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/percona/pmm/managed/utils/tests"
+)
+
+func TestExtractOriginalRequest(t *testing.T) {
+ t.Parallel()
+
+ invalidUTF8URI := string([]byte{'/', 'b', 'a', 'd', 0xff})
+
+ for _, tc := range []struct {
+ name string
+ initialMethod string
+ origMethod *string
+ origURI *string
+ wantMethod string
+ wantPath string
+ wantErr string
+ }{
+ {
+ name: "normalizes traversal and strips query",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodPost),
+ origURI: new("/v1/server/AWSInstanceCheck/..%2f..%2fmanaged/logs.zip?foo=bar"),
+ wantMethod: http.MethodPost,
+ wantPath: "/v1/managed/logs.zip",
+ },
+ {
+ name: "keeps already clean path",
+ initialMethod: http.MethodPost,
+ origMethod: new(http.MethodGet),
+ origURI: new("/v1/server/version"),
+ wantMethod: http.MethodGet,
+ wantPath: "/v1/server/version",
+ },
+ {
+ name: "collapses duplicate slashes",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new("/v1//server///logs.zip"),
+ wantMethod: http.MethodGet,
+ wantPath: "/v1/server/logs.zip",
+ },
+ {
+ name: "cleans plain dot segments",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodDelete),
+ origURI: new("/v1/server/../inventory/./Services/List"),
+ wantMethod: http.MethodDelete,
+ wantPath: "/v1/inventory/Services/List",
+ },
+ {
+ name: "cleans encoded slashes in traversal",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new("/v1/server/AWSInstanceCheck/..%2F..%2Finventory/Services/List"),
+ wantMethod: http.MethodGet,
+ wantPath: "/v1/inventory/Services/List",
+ },
+ {
+ name: "sanitizes encoded newline and carriage return",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new("/v1/server/logs%0A%0D.zip"),
+ wantMethod: http.MethodGet,
+ wantPath: "/v1/server/logs .zip",
+ },
+ {
+ name: "sanitizes raw newline and carriage return",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new("/v1/server/logs\n\r.zip"),
+ wantMethod: http.MethodGet,
+ wantPath: "/v1/server/logs .zip",
+ },
+ {
+ name: "accepts custom method",
+ initialMethod: http.MethodGet,
+ origMethod: new("CUSTOM"),
+ origURI: new("/v1/management/Jobs"),
+ wantMethod: "CUSTOM",
+ wantPath: "/v1/management/Jobs",
+ },
+ {
+ name: "fails on missing original method",
+ initialMethod: http.MethodGet,
+ origURI: new("/v1/server/version"),
+ wantErr: "empty X-Original-Method",
+ },
+ {
+ name: "fails on missing original uri",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ wantErr: "empty X-Original-Uri",
+ },
+ {
+ name: "fails on uri without leading slash",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new("v1/server/version"),
+ wantErr: "unexpected X-Original-Uri",
+ },
+ {
+ name: "fails on invalid utf8 uri",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new(invalidUTF8URI),
+ wantErr: "invalid X-Original-Uri",
+ },
+ {
+ name: "fails on invalid escape sequence",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new("/v1/server/%zz/logs.zip"),
+ wantErr: "failed to unescape path",
+ },
+ {
+ name: "fails on incomplete escape",
+ initialMethod: http.MethodGet,
+ origMethod: new(http.MethodGet),
+ origURI: new("/v1/server/%"),
+ wantErr: "failed to unescape path",
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ req := httptest.NewRequestWithContext(t.Context(), tc.initialMethod, "/auth_request", nil)
+ if tc.origMethod != nil {
+ req.Header.Set("X-Original-Method", *tc.origMethod)
+ }
+ if tc.origURI != nil {
+ req.Header.Set("X-Original-Uri", *tc.origURI)
+ }
+
+ err := extractOriginalRequest(req)
+ if tc.wantErr != "" {
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), tc.wantErr)
+ assert.Equal(t, tc.initialMethod, req.Method)
+ assert.Equal(t, "/auth_request", req.URL.Path)
+ return
+ }
+
+ require.NoError(t, err)
+ assert.Equal(t, tc.wantMethod, req.Method)
+ assert.Equal(t, tc.wantPath, req.URL.Path)
+ })
+ }
+}
+
+func TestNextPrefix(t *testing.T) {
+ t.Parallel()
+ for _, paths := range [][]string{
+ {"/inventory.Nodes/ListNodes", "/inventory.Nodes/", "/inventory.Nodes", "/inventory.", "/inventory", "/", "/"},
+ {"/v1/inventory/Nodes/List", "/v1/inventory/Nodes/", "/v1/inventory/Nodes", "/v1/inventory/", "/v1/inventory", "/v1/", "/v1", "/", "/"},
+ {"/.x", "/.", "/", "/"},
+ {".", "/", "/"},
+ {"./", "/", "/"},
+ {"hax0r", "/", "/"},
+ {"", "/"},
+ {"/v1/server/AWSInstanceCheck/..%2f..%2finventory/Services/List'"},
+ } {
+ t.Run(paths[0], func(t *testing.T) {
+ t.Parallel()
+ for i, path := range paths[:len(paths)-1] {
+ tests.AddToFuzzCorpus(t, "", []byte(path))
+
+ expected := paths[i+1]
+ actual := nextPrefix(path)
+ assert.Equal(t, expected, actual, "path = %q", path)
+ }
+ })
+ }
+}
+
+func TestResolveRule(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ name string
+ method string
+ path string
+ wantRole role
+ wantPrefix string
+ }{
+ {name: "returns viewer role for template listing", method: http.MethodGet, path: "/v1/alerting/templates", wantRole: viewer, wantPrefix: "/v1/alerting"},
+ {name: "uses method specific rule for template creation", method: http.MethodPost, path: "/v1/alerting/templates", wantRole: editor, wantPrefix: "/v1/alerting/templates"},
+ {name: "uses method specific rule for template update path", method: http.MethodPut, path: "/v1/alerting/templates/foo", wantRole: editor, wantPrefix: "/v1/alerting/templates/"},
+ {name: "uses method specific rule for template delete path", method: http.MethodDelete, path: "/v1/alerting/templates/foo", wantRole: editor, wantPrefix: "/v1/alerting/templates/"},
+ {name: "returns editor role for alerting rules creation", method: http.MethodPost, path: "/v1/alerting/rules", wantRole: editor, wantPrefix: "/v1/alerting/rules"},
+ {name: "returns most specific matching path rule", method: http.MethodGet, path: "/v1/server/settings/readonly/details", wantRole: viewer, wantPrefix: "/v1/server/settings/readonly"},
+ {name: "falls back to grafana admin when no explicit rule matches", method: http.MethodGet, path: "/v1/unknown", wantRole: grafanaAdmin, wantPrefix: "/"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ got, gotPrefix := resolveRule(tc.method, tc.path, logrus.WithField("test", t.Name()))
+ assert.Equal(t, tc.wantRole, got)
+ assert.Equal(t, tc.wantPrefix, gotPrefix)
+ })
+ }
+}
+
+func TestIsLocalAgentConnection(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ name string
+ remoteAddr string
+ path string
+ want bool
+ }{
+ // local IPv4
+ {name: "local connect endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: connectionEndpoint, want: true},
+ {name: "local connectV2 endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: connectionEndpointV2, want: true},
+ {name: "local rta endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: rtaCollectEndpoint, want: true},
+ {name: "local unknown endpoint IPv4", remoteAddr: "127.0.0.1:12345", path: "/v1/server/version", want: false},
+ // local IPv6
+ {name: "local connect endpoint IPv6", remoteAddr: "[::1]:12345", path: connectionEndpoint, want: true},
+ {name: "local connectV2 endpoint IPv6", remoteAddr: "[::1]:12345", path: connectionEndpointV2, want: true},
+ {name: "local rta endpoint IPv6", remoteAddr: "[::1]:12345", path: rtaCollectEndpoint, want: true},
+ {name: "local unknown endpoint IPv6", remoteAddr: "[::1]:12345", path: "/v1/server/version", want: false},
+ // remote
+ {name: "remote connect endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: connectionEndpoint, want: false},
+ {name: "remote connectV2 endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: connectionEndpointV2, want: false},
+ {name: "remote rta endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: rtaCollectEndpoint, want: false},
+ {name: "remote unknown endpoint IPv4", remoteAddr: "10.0.0.2:12345", path: "/v1/server/version", want: false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, tc.path, nil)
+ req.RemoteAddr = tc.remoteAddr
+ assert.Equal(t, tc.want, isLocalAgentConnection(req))
+ })
+ }
+}
+
+func TestCleanPath(t *testing.T) {
+ t.Parallel()
+ tests := []struct {
+ path string
+ expected string
+ wantErr bool
+ }{
+ {
+ connectionEndpointV2,
+ connectionEndpointV2,
+ false,
+ }, {
+ connectionEndpoint,
+ connectionEndpoint,
+ false,
+ }, {
+ rtaCollectEndpoint,
+ rtaCollectEndpoint,
+ false,
+ }, {
+ "/v1/server/AWSInstanceCheck/..%2f..%2finventory/Services/List",
+ "/v1/inventory/Services/List",
+ false,
+ }, {
+ "/v1/server/AWSInstanceCheck/..%2f..%2f..%2fmanaged/logs.zip",
+ "/managed/logs.zip",
+ false,
+ }, {
+ "/v1/server/AWSInstanceCheck/..%2f..%2f..%2f/logs.zip",
+ "/logs.zip",
+ false,
+ }, {
+ "/managed/logs.zip?download=1",
+ "/managed/logs.zip",
+ false,
+ }, {
+ "/v1/server/./logs.zip",
+ "/v1/server/logs.zip",
+ false,
+ }, {
+ "/v1/server/../logs.zip",
+ "/v1/logs.zip",
+ false,
+ }, {
+ "/graph/api/datasources/proxy/8/?query=WITH%20(%0A%20%20%20%20CASE%20%0A%20%20%20%20%20%20%20%20WHEN%20(3000%20%25%2060)%20%3D%200%20THEN%203000%0A%20%20%20%20ELSE%2060%20END%0A)%20AS%20scale%0ASELECT%0A%20%20%20%20(intDiv(toUInt32(timestamp)%2C%203000)%20*%203000)%20*%201000%20as%20t%2C%0A%20%20%20%20hostname%20h%2C%0A%20%20%20%20status%20s%2C%0A%20%20%20%20SUM(req_count)%20as%20req_count%0AFROM%20pinba.report_by_all%0AWHERE%0A%20%20%20%20timestamp%20%3E%3D%20toDateTime(1707139680)%20AND%20timestamp%20%3C%3D%20toDateTime(1707312480)%0A%20%20%20%20AND%20status%20%3E%3D%20400%0A%20%20%20%20AND%20CASE%20WHEN%20%27all%27%20%3C%3E%20%27all%27%20THEN%20schema%20%3D%20%27all%27%20ELSE%201%20END%0A%20%20%20%20AND%20CASE%20WHEN%20%27all%27%20%3C%3E%20%27all%27%20THEN%20hostname%20%3D%20%27all%27%20ELSE%201%20END%0A%20%20%20%20AND%20CASE%20WHEN%20%27all%27%20%3C%3E%20%27all%27%20THEN%20server_name%20%3D%20%27all%27%20ELSE%201%20END%0AGROUP%20BY%20t%2C%20h%2C%20s%0AORDER%20BY%20t%20FORMAT%20JSON",
+ "/graph/api/datasources/proxy/8/",
+ false,
+ }, {
+ "/v1/server/%zz/logs.zip",
+ "",
+ true,
+ }, {
+ "/v1/server/%",
+ "",
+ true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.path, func(t *testing.T) {
+ t.Parallel()
+ got, err := cleanPath(tt.path)
+ if tt.wantErr {
+ require.Error(t, err)
+ return
+ }
+
+ require.NoError(t, err)
+ assert.Equalf(t, tt.expected, got, "cleanPath(%v)", tt.path)
+ })
+ }
+}
+
+func TestExtractAuthHeaders(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ name string
+ set func(req *http.Request)
+ want http.Header
+ }{
+ {
+ name: "returns authorization and cookie headers when present",
+ set: func(req *http.Request) {
+ req.Header.Set("Authorization", "Bearer token")
+ req.Header.Set("Cookie", "session=abc")
+ },
+ want: http.Header{
+ "Authorization": []string{"Bearer token"},
+ "Cookie": []string{"session=abc"},
+ },
+ },
+ {
+ name: "returns empty header when auth headers are missing",
+ set: func(_ *http.Request) {},
+ want: nil,
+ },
+ {
+ name: "ignores unrelated headers",
+ set: func(req *http.Request) {
+ req.Header.Set("X-Request-ID", "req-1")
+ req.Header.Set("Accept", "application/json")
+ },
+ want: nil,
+ },
+ {
+ name: "skips empty authorization and cookie values",
+ set: func(req *http.Request) {
+ req.Header["Authorization"] = []string{""}
+ req.Header["Cookie"] = []string{""}
+ },
+ want: nil,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ tc.set(req)
+
+ got := extractAuthHeaders(req)
+ assert.Equal(t, tc.want, got)
+ })
+ }
+}
+
+func TestGetAuthCacheKey(t *testing.T) {
+ t.Parallel()
+
+ t.Run("returns separator only for missing auth headers", func(t *testing.T) {
+ t.Parallel()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ key := getAuthCacheKey(req)
+ assert.Equal(t, ":", key)
+ })
+
+ t.Run("returns key for authorization header", func(t *testing.T) {
+ t.Parallel()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.Header.Set("Authorization", "Bearer token")
+ key := getAuthCacheKey(req)
+ assert.Equal(t, "Bearer token:", key)
+ })
+
+ t.Run("returns key for cookie header", func(t *testing.T) {
+ t.Parallel()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.Header.Set("Cookie", "grafana_session=abc")
+ key := getAuthCacheKey(req)
+ assert.Equal(t, ":grafana_session=abc", key)
+ })
+
+ t.Run("returns key for combined authorization and cookie headers", func(t *testing.T) {
+ t.Parallel()
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.Header.Set("Authorization", "Bearer token")
+ req.Header.Set("Cookie", "grafana_session=abc")
+ key := getAuthCacheKey(req)
+ assert.Equal(t, "Bearer token:grafana_session=abc", key)
+ })
+
+ t.Run("produces deterministic key for the same auth headers", func(t *testing.T) {
+ t.Parallel()
+
+ req1 := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req1.Header.Set("Authorization", "Bearer token")
+ req1.Header.Set("Cookie", "grafana_session=abc")
+ key1 := getAuthCacheKey(req1)
+
+ req2 := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req2.Header.Set("Authorization", "Bearer token")
+ req2.Header.Set("Cookie", "grafana_session=abc")
+ key2 := getAuthCacheKey(req2)
+
+ assert.Equal(t, key1, key2)
+ })
+
+ t.Run("ignores unrelated headers", func(t *testing.T) {
+ t.Parallel()
+
+ req1 := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req1.Header.Set("Authorization", "Bearer token")
+ req1.Header.Set("Cookie", "grafana_session=abc")
+ key1 := getAuthCacheKey(req1)
+
+ req2 := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req2.Header.Set("Authorization", "Bearer token")
+ req2.Header.Set("Cookie", "grafana_session=abc")
+ req2.Header.Set("X-Extra", "ignored")
+ key2 := getAuthCacheKey(req2)
+
+ assert.Equal(t, key1, key2)
+ })
+
+ t.Run("changes key when auth header value changes", func(t *testing.T) {
+ t.Parallel()
+
+ req1 := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req1.Header.Set("Authorization", "Bearer token-a")
+
+ req2 := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req2.Header.Set("Authorization", "Bearer token-b")
+
+ assert.NotEqual(t, getAuthCacheKey(req1), getAuthCacheKey(req2))
+ })
+
+ t.Run("uses both headers when both are present", func(t *testing.T) {
+ t.Parallel()
+
+ reqAuthOnly := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ reqAuthOnly.Header.Set("Authorization", "Bearer token")
+
+ reqCookieOnly := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ reqCookieOnly.Header.Set("Cookie", "grafana_session=abc")
+
+ reqBoth := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ reqBoth.Header.Set("Authorization", "Bearer token")
+ reqBoth.Header.Set("Cookie", "grafana_session=abc")
+
+ keyAuthOnly := getAuthCacheKey(reqAuthOnly)
+ keyCookieOnly := getAuthCacheKey(reqCookieOnly)
+ keyBoth := getAuthCacheKey(reqBoth)
+
+ assert.NotEqual(t, keyAuthOnly, keyBoth)
+ assert.NotEqual(t, keyCookieOnly, keyBoth)
+ })
+}
diff --git a/managed/services/grafana/mock_access_control_test.go b/managed/services/grafana/mock_access_control_test.go
new file mode 100644
index 00000000000..b304d84b334
--- /dev/null
+++ b/managed/services/grafana/mock_access_control_test.go
@@ -0,0 +1,43 @@
+// Code generated by mockery. DO NOT EDIT.
+
+package grafana
+
+import mock "github.com/stretchr/testify/mock"
+
+// mockAccessControl is an autogenerated mock type for the accessControl type
+type mockAccessControl struct {
+ mock.Mock
+}
+
+// isEnabled provides a mock function with no fields
+func (_m *mockAccessControl) isEnabled() bool {
+ ret := _m.Called()
+
+ if len(ret) == 0 {
+ panic("no return value specified for isEnabled")
+ }
+
+ var r0 bool
+ if rf, ok := ret.Get(0).(func() bool); ok {
+ r0 = rf()
+ } else {
+ r0 = ret.Get(0).(bool)
+ }
+
+ return r0
+}
+
+// newMockAccessControl creates a new instance of mockAccessControl. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+// The first argument is typically a *testing.T value.
+func newMockAccessControl(t interface {
+ mock.TestingT
+ Cleanup(func())
+},
+) *mockAccessControl {
+ mock := &mockAccessControl{}
+ mock.Mock.Test(t)
+
+ t.Cleanup(func() { mock.AssertExpectations(t) })
+
+ return mock
+}
diff --git a/managed/services/grafana/mock_grafana_auth_user_getter_test.go b/managed/services/grafana/mock_grafana_auth_user_getter_test.go
new file mode 100644
index 00000000000..15462355b22
--- /dev/null
+++ b/managed/services/grafana/mock_grafana_auth_user_getter_test.go
@@ -0,0 +1,59 @@
+// Code generated by mockery. DO NOT EDIT.
+
+package grafana
+
+import (
+ context "context"
+ http "net/http"
+
+ logrus "github.com/sirupsen/logrus"
+ mock "github.com/stretchr/testify/mock"
+)
+
+// mockGrafanaAuthUserGetter is an autogenerated mock type for the grafanaAuthUserGetter type
+type mockGrafanaAuthUserGetter struct {
+ mock.Mock
+}
+
+// getAuthUser provides a mock function with given fields: ctx, authHeaders, l
+func (_m *mockGrafanaAuthUserGetter) getAuthUser(ctx context.Context, authHeaders http.Header, l *logrus.Entry) (authUser, error) {
+ ret := _m.Called(ctx, authHeaders, l)
+
+ if len(ret) == 0 {
+ panic("no return value specified for getAuthUser")
+ }
+
+ var r0 authUser
+ var r1 error
+ if rf, ok := ret.Get(0).(func(context.Context, http.Header, *logrus.Entry) (authUser, error)); ok {
+ return rf(ctx, authHeaders, l)
+ }
+ if rf, ok := ret.Get(0).(func(context.Context, http.Header, *logrus.Entry) authUser); ok {
+ r0 = rf(ctx, authHeaders, l)
+ } else {
+ r0 = ret.Get(0).(authUser)
+ }
+
+ if rf, ok := ret.Get(1).(func(context.Context, http.Header, *logrus.Entry) error); ok {
+ r1 = rf(ctx, authHeaders, l)
+ } else {
+ r1 = ret.Error(1)
+ }
+
+ return r0, r1
+}
+
+// newMockGrafanaAuthUserGetter creates a new instance of mockGrafanaAuthUserGetter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+// The first argument is typically a *testing.T value.
+func newMockGrafanaAuthUserGetter(t interface {
+ mock.TestingT
+ Cleanup(func())
+},
+) *mockGrafanaAuthUserGetter {
+ mock := &mockGrafanaAuthUserGetter{}
+ mock.Mock.Test(t)
+
+ t.Cleanup(func() { mock.AssertExpectations(t) })
+
+ return mock
+}
diff --git a/managed/services/qan/client.go b/managed/services/qan/client.go
index 7efc5db27f7..f433f818995 100644
--- a/managed/services/qan/client.go
+++ b/managed/services/qan/client.go
@@ -179,7 +179,10 @@ func (c *Client) Collect(ctx context.Context, metricsBuckets []*agentv1.MetricsB
}
}()
- agents, err := collectAgents(c.db.Querier, metricsBuckets)
+ // It is completely OK to re-use the same Querier for multiple queries, as it is safe for concurrent use
+ // and creates less preasure on GC.
+ q := c.db.WithContext(ctx)
+ agents, err := collectAgents(q, metricsBuckets)
if err != nil {
return err
}
@@ -187,7 +190,7 @@ func (c *Client) Collect(ctx context.Context, metricsBuckets []*agentv1.MetricsB
if err != nil {
return err
}
- nodes, err := collectNodes(c.db.Querier, services)
+ nodes, err := collectNodes(q, services)
if err != nil {
return err
}
diff --git a/managed/services/realtimeanalytics/deps.go b/managed/services/realtimeanalytics/deps.go
index 829e78195eb..a83036b2430 100644
--- a/managed/services/realtimeanalytics/deps.go
+++ b/managed/services/realtimeanalytics/deps.go
@@ -27,3 +27,13 @@ type agentsRegistry interface {
type agentsStateUpdater interface {
RequestStateUpdate(ctx context.Context, pmmAgentID string)
}
+
+// Limiter defines the interface to perform request rate limiting.
+// If TryAcquire function return false, the request will be rejected.
+// Otherwise, the request will pass.
+type Limiter interface {
+ // Try to acquire a free slot to handle incoming request.
+ TryAcquire() bool
+ // Release the used slot.
+ Release()
+}
diff --git a/managed/services/realtimeanalytics/mock_limiter_test.go b/managed/services/realtimeanalytics/mock_limiter_test.go
new file mode 100644
index 00000000000..0bc254cb949
--- /dev/null
+++ b/managed/services/realtimeanalytics/mock_limiter_test.go
@@ -0,0 +1,48 @@
+// Code generated by mockery. DO NOT EDIT.
+
+package realtimeanalytics
+
+import mock "github.com/stretchr/testify/mock"
+
+// mockLimiter is an autogenerated mock type for the Limiter type
+type mockLimiter struct {
+ mock.Mock
+}
+
+// Release provides a mock function with no fields
+func (_m *mockLimiter) Release() {
+ _m.Called()
+}
+
+// TryAcquire provides a mock function with no fields
+func (_m *mockLimiter) TryAcquire() bool {
+ ret := _m.Called()
+
+ if len(ret) == 0 {
+ panic("no return value specified for TryAcquire")
+ }
+
+ var r0 bool
+ if rf, ok := ret.Get(0).(func() bool); ok {
+ r0 = rf()
+ } else {
+ r0 = ret.Get(0).(bool)
+ }
+
+ return r0
+}
+
+// newMockLimiter creates a new instance of mockLimiter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+// The first argument is typically a *testing.T value.
+func newMockLimiter(t interface {
+ mock.TestingT
+ Cleanup(func())
+},
+) *mockLimiter {
+ mock := &mockLimiter{}
+ mock.Mock.Test(t)
+
+ t.Cleanup(func() { mock.AssertExpectations(t) })
+
+ return mock
+}
diff --git a/managed/services/realtimeanalytics/service.go b/managed/services/realtimeanalytics/service.go
index 0414f9b6558..5560bd1cca6 100644
--- a/managed/services/realtimeanalytics/service.go
+++ b/managed/services/realtimeanalytics/service.go
@@ -55,15 +55,20 @@ type Service struct {
registry agentsRegistry
stateUpdater agentsStateUpdater
store *Store
+ // PMM Agents connection attempts rate limiter.
+ // Used to prevent the system degradation (exhausted db connections in particular)
+ // during massive agents connections (thundering herd).
+ rateLimiter Limiter
}
// NewService creates a new Real-Time Analytics service.
-func NewService(db *reform.DB, registry agentsRegistry, stateUpdater agentsStateUpdater, store *Store) *Service {
+func NewService(db *reform.DB, registry agentsRegistry, stateUpdater agentsStateUpdater, store *Store, rateLimiter Limiter) *Service {
return &Service{
db: db,
registry: registry,
stateUpdater: stateUpdater,
store: store,
+ rateLimiter: rateLimiter,
}
}
@@ -521,15 +526,27 @@ func (s *Service) Collect(stream grpc.ClientStreamingServer[rtav1.CollectRequest
agentMD, err := agentv1.ReceiveAgentConnectMetadata(stream)
if err != nil {
- l.Warnf("Disconnecting client: authentication failed: %v", err)
+ l.WithError(err).Warn("Disconnecting client: authentication failed")
return status.Error(codes.Unauthenticated, "Failed to receive agent metadata")
}
- // Validate that the pmm-agent exists
- agent, err := models.FindAgentByID(s.db.Querier, agentMD.ID)
+ agent, err := func() (*models.Agent, error) {
+ if !s.rateLimiter.TryAcquire() {
+ return nil, status.Error(codes.ResourceExhausted, "is rejected by ratelimit, please retry later.")
+ }
+ defer s.rateLimiter.Release()
+
+ // Validate that the pmm-agent exists
+ agent, err := models.FindAgentByID(s.db.WithContext(streamCtx), agentMD.ID)
+ if err != nil {
+ l.Warnf("Disconnecting client: agent validation failed: %v", err)
+ return nil, status.Error(codes.InvalidArgument, "Invalid Agent ID: "+agentMD.ID)
+ }
+ return agent, nil
+ }()
if err != nil {
- l.Warnf("Disconnecting client: agent validation failed: %v", err)
- return status.Error(codes.InvalidArgument, "Invalid Agent ID: "+agentMD.ID)
+ l.WithError(err).Warn("Disconnecting client")
+ return err
}
if agent.AgentType != models.PMMAgentType {
diff --git a/managed/services/realtimeanalytics/service_test.go b/managed/services/realtimeanalytics/service_test.go
index be12458c710..7a03ef7777a 100644
--- a/managed/services/realtimeanalytics/service_test.go
+++ b/managed/services/realtimeanalytics/service_test.go
@@ -129,7 +129,8 @@ func TestListServices(t *testing.T) {
registry := newMockAgentsRegistry(t)
stateUpdater := newMockAgentsStateUpdater(t)
store := NewStore()
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ svc := NewService(db, registry, stateUpdater, store, limiter)
t.Run("list all supported services", func(t *testing.T) {
resp, err := svc.ListServices(t.Context(), &rtav1.ListServicesRequest{})
@@ -235,7 +236,8 @@ func TestListSessions(t *testing.T) {
t.Run("list running sessions", func(t *testing.T) {
registry := newMockAgentsRegistry(t)
registry.On("IsConnected", pmmAgent.AgentID).Return(true)
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ svc := NewService(db, registry, stateUpdater, store, limiter)
rtaAgent.Status = inventoryv1.AgentStatus_name[int32(inventoryv1.AgentStatus_AGENT_STATUS_RUNNING)]
err = db.Update(rtaAgent)
@@ -254,7 +256,8 @@ func TestListSessions(t *testing.T) {
t.Run("filter sessions by cluster", func(t *testing.T) {
registry := newMockAgentsRegistry(t)
registry.On("IsConnected", pmmAgent.AgentID).Return(true)
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ svc := NewService(db, registry, stateUpdater, store, limiter)
resp, err := svc.ListSessions(t.Context(), &rtav1.ListSessionsRequest{ClusterName: "test-cluster"})
require.NoError(t, err)
@@ -268,7 +271,8 @@ func TestListSessions(t *testing.T) {
t.Run("show disconnected agents with unknown status", func(t *testing.T) {
registry := newMockAgentsRegistry(t)
registry.On("IsConnected", pmmAgent.AgentID).Return(false)
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ svc := NewService(db, registry, stateUpdater, store, limiter)
resp, err := svc.ListSessions(t.Context(), &rtav1.ListSessionsRequest{})
require.NoError(t, err)
@@ -320,7 +324,8 @@ func TestStartSession(t *testing.T) {
stateUpdater.On("RequestStateUpdate", mock.Anything, pmmAgent.AgentID).Return()
store := NewStore()
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ svc := NewService(db, registry, stateUpdater, store, limiter)
t.Run("start session for single service", func(t *testing.T) {
resp, err := svc.StartSession(t.Context(), &rtav1.StartSessionRequest{
@@ -548,7 +553,8 @@ func TestStopSession(t *testing.T) {
stateUpdater.On("RequestStateUpdate", mock.Anything, pmmAgent.AgentID).Return()
store := NewStore()
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ svc := NewService(db, registry, stateUpdater, store, limiter)
t.Run("stop session for single service", func(t *testing.T) {
resp, err := svc.StopSession(t.Context(), &rtav1.StopSessionRequest{
@@ -686,7 +692,8 @@ func TestSearchQueries(t *testing.T) {
registry := newMockAgentsRegistry(t)
stateUpdater := newMockAgentsStateUpdater(t)
store := NewStore()
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ svc := NewService(db, registry, stateUpdater, store, limiter)
// Populate store with static query data for service1
store.Set(service1.ServiceID, getServiceQueries(service1.ServiceID, service1.ServiceName, 2))
@@ -840,7 +847,10 @@ func TestService_Collect(t *testing.T) {
registry := newMockAgentsRegistry(t)
stateUpdater := newMockAgentsStateUpdater(t)
store := NewStore()
- svc := NewService(db, registry, stateUpdater, store)
+ limiter := newMockLimiter(t)
+ limiter.On("TryAcquire").Return(true).Once()
+ limiter.On("Release").Return().Once()
+ svc := NewService(db, registry, stateUpdater, store, limiter)
// // Create in-memory listener for testing
const bufSize = 1024 * 1024
@@ -906,3 +916,128 @@ func TestService_Collect(t *testing.T) {
assert.Equal(t, "mongodb-1", storeqQs[i].ServiceName)
}
}
+
+func TestService_CollectRejectsWhenRateLimitIsExceeded(t *testing.T) {
+ sqlDB := testdb.Open(t, models.SkipFixtures, nil)
+ t.Cleanup(func() {
+ require.NoError(t, sqlDB.Close())
+ })
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+
+ node, err := models.CreateNode(db.Querier, models.GenericNodeType, &models.CreateNodeParams{
+ NodeName: "test-node",
+ })
+ require.NoError(t, err)
+
+ pmmAgent, err := models.CreatePMMAgent(db.Querier, node.NodeID, nil)
+ require.NoError(t, err)
+
+ registry := newMockAgentsRegistry(t)
+ stateUpdater := newMockAgentsStateUpdater(t)
+ store := NewStore()
+ limiter := newMockLimiter(t)
+ limiter.On("TryAcquire").Return(false).Once()
+ svc := NewService(db, registry, stateUpdater, store, limiter)
+
+ const bufSize = 1024 * 1024
+ lis = bufconn.Listen(bufSize)
+
+ grpcMetrics := interceptors.NewServerMetricsWithExtension(&interceptors.GRPCMetricsExtension{})
+ s := grpc.NewServer(
+ grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
+ interceptors.Stream(grpcMetrics.StreamServerInterceptor()),
+ interceptors.StreamServiceEnabledInterceptor(),
+ grpc_validator.StreamServerInterceptor(),
+ )),
+ )
+ rtav1.RegisterCollectorServiceServer(s, svc)
+
+ serveError := make(chan error)
+ go func() {
+ serveError <- s.Serve(lis)
+ }()
+ t.Cleanup(func() {
+ s.GracefulStop()
+ require.NoError(t, <-serveError)
+ })
+
+ client := getTestClient(t)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ t.Cleanup(cancel)
+
+ streamCtx := agentv1.AddAgentConnectMetadata(ctx, &agentv1.AgentConnectMetadata{
+ ID: pmmAgent.AgentID,
+ Version: "1.0.0",
+ })
+
+ stream, err := client.Collect(streamCtx)
+ require.NoError(t, err)
+
+ err = stream.Send(&rtav1.CollectRequest{Queries: getServiceQueries("service-1", "mongodb-1", 1)})
+ require.NoError(t, err)
+
+ _, err = stream.CloseAndRecv()
+ require.Error(t, err)
+ assert.Equal(t, codes.ResourceExhausted, status.Code(err))
+ assert.Empty(t, store.Get("service-1"))
+}
+
+func TestService_CollectReturnsInvalidArgumentForUnknownAgentID(t *testing.T) {
+ sqlDB := testdb.Open(t, models.SkipFixtures, nil)
+ t.Cleanup(func() {
+ require.NoError(t, sqlDB.Close())
+ })
+ db := reform.NewDB(sqlDB, postgresql.Dialect, reform.NewPrintfLogger(t.Logf))
+
+ registry := newMockAgentsRegistry(t)
+ stateUpdater := newMockAgentsStateUpdater(t)
+ store := NewStore()
+ limiter := newMockLimiter(t)
+ limiter.On("TryAcquire").Return(true).Once()
+ limiter.On("Release").Return().Once()
+ svc := NewService(db, registry, stateUpdater, store, limiter)
+
+ const bufSize = 1024 * 1024
+ lis = bufconn.Listen(bufSize)
+
+ grpcMetrics := interceptors.NewServerMetricsWithExtension(&interceptors.GRPCMetricsExtension{})
+ s := grpc.NewServer(
+ grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
+ interceptors.Stream(grpcMetrics.StreamServerInterceptor()),
+ interceptors.StreamServiceEnabledInterceptor(),
+ grpc_validator.StreamServerInterceptor(),
+ )),
+ )
+ rtav1.RegisterCollectorServiceServer(s, svc)
+
+ serveError := make(chan error)
+ go func() {
+ serveError <- s.Serve(lis)
+ }()
+ t.Cleanup(func() {
+ s.GracefulStop()
+ require.NoError(t, <-serveError)
+ })
+
+ client := getTestClient(t)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ t.Cleanup(cancel)
+
+ streamCtx := agentv1.AddAgentConnectMetadata(ctx, &agentv1.AgentConnectMetadata{
+ ID: "missing-agent",
+ Version: "1.0.0",
+ })
+
+ stream, err := client.Collect(streamCtx)
+ require.NoError(t, err)
+
+ err = stream.Send(&rtav1.CollectRequest{Queries: getServiceQueries("service-1", "mongodb-1", 1)})
+ require.NoError(t, err)
+
+ _, err = stream.CloseAndRecv()
+ require.Error(t, err)
+ assert.Equal(t, codes.InvalidArgument, status.Code(err))
+ assert.Empty(t, store.Get("service-1"))
+}
diff --git a/managed/services/victoriametrics/victoriametrics.go b/managed/services/victoriametrics/victoriametrics.go
index 9703768c106..b2fc0267b5f 100644
--- a/managed/services/victoriametrics/victoriametrics.go
+++ b/managed/services/victoriametrics/victoriametrics.go
@@ -474,21 +474,19 @@ func scrapeConfigForVMAlert(interval time.Duration, pmmServerNodeName string) *c
}
// BuildScrapeConfigForVMAgent builds scrape configuration for given pmm-agent.
-func (svc *Service) BuildScrapeConfigForVMAgent(ctx context.Context, pmmAgentID string) ([]byte, error) {
+func (svc *Service) BuildScrapeConfigForVMAgent(q *reform.Querier, pmmAgentID string) ([]byte, error) {
if pmmAgentID == models.PMMServerAgentID {
return svc.buildVMConfig()
}
var cfg config.Config
- e := svc.db.InTransactionContext(ctx, nil, func(tx *reform.TX) error {
- settings, err := models.GetSettings(tx)
- if err != nil {
- return err
- } // In HA mode, skip ExternalExporter agents if this node is not the leader
- skipExternalExporter := !svc.haService.IsLeader()
- return AddScrapeConfigs(svc.l, &cfg, tx.Querier, new(settings.MetricsResolutions), new(pmmAgentID), true, skipExternalExporter)
- })
- if e != nil {
- return nil, e
+ settings, err := models.GetSettings(q)
+ if err != nil {
+ return nil, err
+ } // In HA mode, skip ExternalExporter agents if this node is not the leader
+ skipExternalExporter := !svc.haService.IsLeader()
+ err = AddScrapeConfigs(svc.l, &cfg, q, new(settings.MetricsResolutions), new(pmmAgentID), true, skipExternalExporter)
+ if err != nil {
+ return nil, err
}
return yaml.Marshal(cfg)
diff --git a/managed/utils/interceptors/interceptors.go b/managed/utils/interceptors/interceptors.go
index 9da9f704231..54c6522c610 100644
--- a/managed/utils/interceptors/interceptors.go
+++ b/managed/utils/interceptors/interceptors.go
@@ -145,7 +145,7 @@ func Stream(interceptor grpc.StreamServerInterceptor) func(srv any, ss grpc.Serv
info.FullMethod == "/realtimeanalytics.v1.CollectorService/Collect" {
md, _ := agentv1.ReceiveAgentConnectMetadata(ss)
if md != nil && md.ID != "" {
- l = l.WithField("agent_id", md.ID)
+ l = l.WithField("pmm_agent_id", md.ID)
}
}
ctx = logger.SetEntry(ctx, l)
diff --git a/utils/cache/cache.go b/utils/cache/cache.go
new file mode 100644
index 00000000000..bfc80750174
--- /dev/null
+++ b/utils/cache/cache.go
@@ -0,0 +1,129 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cache
+
+import (
+ "hash/maphash"
+ "iter"
+)
+
+// Cache is a high-performance, sharded, generic cache.
+type Cache[V any] struct {
+ shards [shardCount]*shard[uint64, V]
+ seed maphash.Seed
+}
+
+// NewCache initializes the cache.
+func NewCache[V any]() *Cache[V] {
+ c := &Cache[V]{
+ seed: maphash.MakeSeed(),
+ }
+
+ for i := range shardCount {
+ c.shards[i] = &shard[uint64, V]{
+ items: make(map[uint64]item[V]),
+ }
+ }
+
+ return c
+}
+
+// calculateKeyHash builds a deterministic cache key directly from passed key.
+func (c *Cache[V]) calculateKeyHash(key string) uint64 {
+ return maphash.String(c.seed, key)
+}
+
+// Get retrieves an item. Zero-allocation on the hot path.
+func (c *Cache[V]) Get(key string) (V, bool) {
+ keyHash := c.calculateKeyHash(key)
+ // Zero-allocation hash via maphash
+ shardKey := maphash.Comparable(c.seed, keyHash)
+ shard := c.shards[shardKey&shardMask]
+
+ shard.mu.RLock()
+ itm, found := shard.items[keyHash]
+ shard.mu.RUnlock()
+
+ if !found {
+ var zero V
+ return zero, false
+ }
+
+ return itm.value, true
+}
+
+// All returns an iterator over all items in the cache.
+// Deadlock Risk: Because the yield function executes while the shard's read lock is held,
+// you must not write to the cache inside the loop.
+// Calling a method that acquires a Lock() on the same shard will cause a deadlock.
+func (c *Cache[V]) All() iter.Seq2[uint64, V] {
+ return func(yield func(uint64, V) bool) {
+ for i := range shardCount {
+ s := c.shards[i]
+
+ s.mu.RLock()
+ for k, item := range s.items {
+ if !yield(k, item.value) {
+ s.mu.RUnlock()
+ return
+ }
+ }
+ s.mu.RUnlock()
+ }
+ }
+}
+
+// Set inserts or updates an item with a specific TTL.
+func (c *Cache[V]) Set(key string, value V) {
+ keyHash := c.calculateKeyHash(key)
+ shardKey := maphash.Comparable(c.seed, keyHash)
+ shard := c.shards[shardKey&shardMask]
+
+ shard.mu.Lock()
+ if _, exists := shard.items[keyHash]; !exists {
+ shard.size++
+ }
+ shard.items[keyHash] = item[V]{
+ value: value,
+ }
+ shard.mu.Unlock()
+}
+
+// Delete removes an item explicitly.
+func (c *Cache[V]) Delete(key string) {
+ keyHash := c.calculateKeyHash(key)
+ shardKey := maphash.Comparable(c.seed, keyHash)
+ shard := c.shards[shardKey&shardMask]
+
+ shard.mu.Lock()
+ if _, exists := shard.items[keyHash]; exists {
+ delete(shard.items, keyHash)
+ shard.size--
+ }
+ shard.mu.Unlock()
+}
+
+// Size returns the total number of items across all cache shards.
+func (c *Cache[V]) Size() int64 {
+ var total int64
+ for i := range shardCount {
+ shard := c.shards[i]
+ shard.mu.RLock()
+ total += shard.size
+ shard.mu.RUnlock()
+ }
+
+ return total
+}
diff --git a/utils/cache/cache_bench_test.go b/utils/cache/cache_bench_test.go
new file mode 100644
index 00000000000..27bc0ed18e6
--- /dev/null
+++ b/utils/cache/cache_bench_test.go
@@ -0,0 +1,101 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cache
+
+import (
+ "strconv"
+ "testing"
+)
+
+func BenchmarkCache_Get(b *testing.B) {
+ c := NewCache[int]()
+ c.Set("hit", 42)
+
+ b.Run("returns value for existing key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ v, ok := c.Get("hit")
+ if !ok || v != 42 {
+ b.Fatalf("unexpected get result: ok=%v value=%d", ok, v)
+ }
+ }
+ })
+
+ b.Run("returns miss for unknown key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ if _, ok := c.Get("missing"); ok {
+ b.Fatal("expected cache miss")
+ }
+ }
+ })
+}
+
+func BenchmarkCache_Set(b *testing.B) {
+ c := NewCache[int]()
+
+ b.Run("updates same key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ c.Set("stable", 1)
+ }
+ })
+
+ b.Run("inserts unique keys", func(b *testing.B) {
+ b.ReportAllocs()
+ n := 0
+ for b.Loop() {
+ c.Set(strconv.Itoa(n), n)
+ n++
+ }
+ })
+}
+
+func BenchmarkCache_Delete(b *testing.B) {
+ c := NewCache[int]()
+
+ b.Run("deletes existing key", func(b *testing.B) {
+ b.ReportAllocs()
+ n := 0
+ for b.Loop() {
+ k := strconv.Itoa(n)
+ c.Set(k, n)
+ c.Delete(k)
+ n++
+ }
+ })
+
+ b.Run("deletes missing key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ c.Delete("missing")
+ }
+ })
+}
+
+func BenchmarkCache_Size(b *testing.B) {
+ c := NewCache[int]()
+
+ for i := range 10_000 {
+ c.Set(strconv.Itoa(i), i)
+ }
+
+ b.ReportAllocs()
+ for b.Loop() {
+ if got := c.Size(); got != 10_000 {
+ b.Fatalf("unexpected size: got %d, want %d", got, 10_000)
+ }
+ }
+}
diff --git a/utils/cache/cache_test.go b/utils/cache/cache_test.go
new file mode 100644
index 00000000000..b912e642413
--- /dev/null
+++ b/utils/cache/cache_test.go
@@ -0,0 +1,143 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cache
+
+import (
+ "testing"
+)
+
+func TestNewCache_ReturnsCacheForValidInputs(t *testing.T) {
+ t.Parallel()
+
+ c := NewCache[int]()
+ if c == nil {
+ t.Fatal("expected cache instance")
+ }
+}
+
+func TestCache_CalculateCacheKey_ReturnsSameValueForSameInput(t *testing.T) {
+ t.Parallel()
+
+ c := NewCache[int]()
+
+ key := "Authorization:Bearer token"
+ first := c.calculateKeyHash(key)
+ second := c.calculateKeyHash(key)
+
+ if first != second {
+ t.Fatalf("expected stable key hash: first=%d second=%d", first, second)
+ }
+}
+
+func TestCache_CalculateCacheKey_ReturnsDifferentValuesForDifferentInputs(t *testing.T) {
+ t.Parallel()
+
+ c := NewCache[int]()
+
+ first := c.calculateKeyHash("Authorization:Bearer token-a")
+ second := c.calculateKeyHash("Authorization:Bearer token-b")
+
+ if first == second {
+ t.Fatal("expected different key hashes for different inputs")
+ }
+}
+
+func TestCache_Set_Get_Delete_StoresReadsAndRemovesValue(t *testing.T) {
+ t.Parallel()
+
+ c := NewCache[int]()
+
+ c.Set("k", 42)
+
+ got, ok := c.Get("k")
+ if !ok {
+ t.Fatal("expected key to exist")
+ }
+ if got != 42 {
+ t.Fatalf("unexpected value: got %d, want %d", got, 42)
+ }
+
+ c.Delete("k")
+
+ _, ok = c.Get("k")
+ if ok {
+ t.Fatal("expected key to be deleted")
+ }
+}
+
+func TestCache_Get_ReturnsMissForUnknownKey(t *testing.T) {
+ t.Parallel()
+
+ c := NewCache[int]()
+
+ got, ok := c.Get("missing")
+ if ok {
+ t.Fatal("expected missing key")
+ }
+ if got != 0 {
+ t.Fatalf("unexpected zero value: got %d", got)
+ }
+}
+
+func TestCache_Get_ReturnsStoredZeroValue(t *testing.T) {
+ t.Parallel()
+
+ c := NewCache[int]()
+
+ c.Set("k", 0)
+
+ got, ok := c.Get("k")
+ if !ok {
+ t.Fatal("expected key to exist")
+ }
+ if got != 0 {
+ t.Fatalf("unexpected value: got %d, want %d", got, 0)
+ }
+}
+
+func TestCache_Size_TracksInsertUpdateDeleteAndMissingDelete(t *testing.T) {
+ t.Parallel()
+
+ c := NewCache[int]()
+
+ if got := c.Size(); got != 0 {
+ t.Fatalf("unexpected size: got %d, want %d", got, 0)
+ }
+
+ c.Set("a", 1)
+ if got := c.Size(); got != 1 {
+ t.Fatalf("unexpected size after first insert: got %d, want %d", got, 1)
+ }
+
+ c.Set("a", 2)
+ if got := c.Size(); got != 1 {
+ t.Fatalf("unexpected size after update: got %d, want %d", got, 1)
+ }
+
+ c.Set("b", 3)
+ if got := c.Size(); got != 2 {
+ t.Fatalf("unexpected size after second insert: got %d, want %d", got, 2)
+ }
+
+ c.Delete("missing")
+ if got := c.Size(); got != 2 {
+ t.Fatalf("unexpected size after deleting missing key: got %d, want %d", got, 2)
+ }
+
+ c.Delete("a")
+ if got := c.Size(); got != 1 {
+ t.Fatalf("unexpected size after deleting existing key: got %d, want %d", got, 1)
+ }
+}
diff --git a/utils/cache/cache_ttl.go b/utils/cache/cache_ttl.go
new file mode 100644
index 00000000000..5a417eb8dc1
--- /dev/null
+++ b/utils/cache/cache_ttl.go
@@ -0,0 +1,176 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cache provides a high-performance cache implementation with sharding
+// and TTL support. It is designed to minimize lock contention and maximize
+// throughput in concurrent environments.
+package cache
+
+import (
+ "context"
+ "hash/maphash"
+ "time"
+)
+
+// TTLCache is a high-performance, sharded, generic TTL cache.
+type TTLCache[V any] struct {
+ shards [shardCount]*shard[uint64, V]
+ seed maphash.Seed
+ ttl time.Duration
+}
+
+// NewCacheTTL initializes the cache with TTL support and starts the background eviction worker.
+func NewCacheTTL[V any](ctx context.Context, ttl time.Duration, cleanupInterval time.Duration) (*TTLCache[V], error) {
+ if ctx == nil {
+ return nil, errInvalidContext
+ }
+
+ if ttl <= 0 {
+ return nil, errInvalidTTLInterval
+ }
+
+ if cleanupInterval <= 0 {
+ return nil, errInvalidCleanupInterval
+ }
+
+ c := &TTLCache[V]{
+ seed: maphash.MakeSeed(),
+ ttl: ttl,
+ }
+
+ for i := range shardCount {
+ c.shards[i] = &shard[uint64, V]{
+ items: make(map[uint64]item[V]),
+ }
+ }
+
+ go c.evictionWorker(ctx, cleanupInterval)
+ return c, nil
+}
+
+// calculateKeyHash builds a deterministic cache key directly from passed key.
+func (c *TTLCache[V]) calculateKeyHash(key string) uint64 {
+ return maphash.String(c.seed, key)
+}
+
+// Get retrieves an item. Zero-allocation on the hot path.
+func (c *TTLCache[V]) Get(key string) (V, bool) {
+ keyHash := c.calculateKeyHash(key)
+ // Zero-allocation hash via maphash
+ shardKey := maphash.Comparable(c.seed, keyHash)
+ shard := c.shards[shardKey&shardMask]
+
+ shard.mu.RLock()
+ itm, found := shard.items[keyHash]
+ shard.mu.RUnlock()
+
+ if !found {
+ var zero V
+ return zero, false
+ }
+
+ // Lazy eviction check.
+ // We read time.Now() outside the lock to minimize critical section time.
+ now := time.Now()
+ if now.After(itm.expires) {
+ // 3. Cold Path: Upgrade to Write Lock to physically remove an element from the map,
+ // so that the GC can instantly remove V from memory.
+ shard.mu.Lock()
+ // Double-checking: check if another goroutine has overwritten the key while we were switching locks.
+ if currentItm, stillExists := shard.items[keyHash]; stillExists && now.After(currentItm.expires) {
+ delete(shard.items, keyHash)
+ shard.size--
+ }
+ shard.mu.Unlock()
+
+ var zero V
+ return zero, false
+ }
+
+ return itm.value, true
+}
+
+// Set inserts or updates an item with a specific TTL.
+func (c *TTLCache[V]) Set(key string, value V) {
+ keyHash := c.calculateKeyHash(key)
+ shardKey := maphash.Comparable(c.seed, keyHash)
+ shard := c.shards[shardKey&shardMask]
+
+ expires := time.Now().Add(c.ttl)
+
+ shard.mu.Lock()
+ if _, exists := shard.items[keyHash]; !exists {
+ shard.size++
+ }
+ shard.items[keyHash] = item[V]{
+ value: value,
+ expires: expires,
+ }
+ shard.mu.Unlock()
+}
+
+// Delete removes an item explicitly.
+func (c *TTLCache[V]) Delete(key string) {
+ keyHash := c.calculateKeyHash(key)
+ shardKey := maphash.Comparable(c.seed, keyHash)
+ shard := c.shards[shardKey&shardMask]
+
+ shard.mu.Lock()
+ if _, exists := shard.items[keyHash]; exists {
+ delete(shard.items, keyHash)
+ shard.size--
+ }
+ shard.mu.Unlock()
+}
+
+// Size returns the total number of items across all cache shards.
+func (c *TTLCache[V]) Size() int64 {
+ var total int64
+ for i := range shardCount {
+ shard := c.shards[i]
+ shard.mu.RLock()
+ total += shard.size
+ shard.mu.RUnlock()
+ }
+
+ return total
+}
+
+// evictionWorker periodically sweeps shards to remove expired items.
+func (c *TTLCache[V]) evictionWorker(ctx context.Context, interval time.Duration) {
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case now := <-ticker.C:
+ // Iterate through shards. We lock one shard at a time to ensure
+ // we don't stall the entire cache during the sweep.
+ for i := range shardCount {
+ shard := c.shards[i]
+
+ shard.mu.Lock()
+ for key, itm := range shard.items {
+ if now.After(itm.expires) {
+ delete(shard.items, key)
+ shard.size--
+ }
+ }
+ shard.mu.Unlock()
+ }
+ }
+ }
+}
diff --git a/utils/cache/cache_ttl_bench_test.go b/utils/cache/cache_ttl_bench_test.go
new file mode 100644
index 00000000000..192d3b18d9b
--- /dev/null
+++ b/utils/cache/cache_ttl_bench_test.go
@@ -0,0 +1,155 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cache
+
+import (
+ "context"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func BenchmarkCacheTTL_Get(b *testing.B) {
+ ctx, cancel := context.WithCancel(b.Context())
+ defer cancel()
+
+ c, err := NewCacheTTL[int](ctx, time.Minute, time.Minute)
+ require.NoError(b, err)
+
+ c.Set("hit", 42)
+
+ b.Run("returns value for existing key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ v, ok := c.Get("hit")
+ if !ok || v != 42 {
+ b.Fatalf("unexpected get result: ok=%v value=%d", ok, v)
+ }
+ }
+ })
+
+ b.Run("returns miss for unknown key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ if _, ok := c.Get("missing"); ok {
+ b.Fatal("expected cache miss")
+ }
+ }
+ })
+}
+
+func BenchmarkCacheTTL_Set(b *testing.B) {
+ ctx, cancel := context.WithCancel(b.Context())
+ defer cancel()
+
+ c, err := NewCacheTTL[int](ctx, time.Minute, time.Minute)
+ require.NoError(b, err)
+
+ b.Run("updates same key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ c.Set("stable", 1)
+ }
+ })
+
+ b.Run("inserts unique keys", func(b *testing.B) {
+ b.ReportAllocs()
+ n := 0
+ for b.Loop() {
+ c.Set(strconv.Itoa(n), n)
+ n++
+ }
+ })
+}
+
+func BenchmarkCacheTTL_Delete(b *testing.B) {
+ ctx, cancel := context.WithCancel(b.Context())
+ defer cancel()
+
+ c, err := NewCacheTTL[int](ctx, time.Minute, time.Minute)
+ require.NoError(b, err)
+
+ b.Run("deletes existing key", func(b *testing.B) {
+ b.ReportAllocs()
+ n := 0
+ for b.Loop() {
+ k := strconv.Itoa(n)
+ c.Set(k, n)
+ c.Delete(k)
+ n++
+ }
+ })
+
+ b.Run("deletes missing key", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ c.Delete("missing")
+ }
+ })
+}
+
+func BenchmarkCacheTTL_Size(b *testing.B) {
+ ctx, cancel := context.WithCancel(b.Context())
+ defer cancel()
+
+ c, err := NewCacheTTL[int](ctx, time.Minute, time.Minute)
+ require.NoError(b, err)
+
+ for i := range 10_000 {
+ c.Set(strconv.Itoa(i), i)
+ }
+
+ b.ReportAllocs()
+ for b.Loop() {
+ if got := c.Size(); got != 10_000 {
+ b.Fatalf("unexpected size: got %d, want %d", got, 10_000)
+ }
+ }
+}
+
+func BenchmarkCacheTTL_EvictionEffectOnSize(b *testing.B) {
+ b.Run("size reflects evicted entries", func(b *testing.B) {
+ ctx, cancel := context.WithCancel(b.Context())
+ defer cancel()
+
+ c, err := NewCacheTTL[int](ctx, 2*time.Millisecond, time.Millisecond)
+ require.NoError(b, err)
+
+ for i := range 1000 {
+ c.Set(strconv.Itoa(i), i)
+ }
+
+ deadline := time.Now().Add(300 * time.Millisecond)
+ for c.Size() != 0 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+
+ if c.Size() != 0 {
+ b.Fatalf("expected empty cache after eviction, got %d", c.Size())
+ }
+
+ b.ReportAllocs()
+ for b.Loop() {
+ if got := c.Size(); got != 0 {
+ b.Fatalf("unexpected size after eviction: got %d", got)
+ }
+ }
+ })
+}
diff --git a/utils/cache/cache_ttl_test.go b/utils/cache/cache_ttl_test.go
new file mode 100644
index 00000000000..e806dcdef3f
--- /dev/null
+++ b/utils/cache/cache_ttl_test.go
@@ -0,0 +1,205 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cache
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNewCacheTTL_ReturnsErrorForInvalidInputs(t *testing.T) {
+ t.Parallel()
+
+ _, err := NewCacheTTL[int](nil, time.Second, time.Second) //nolint:staticcheck
+ require.ErrorIs(t, err, errInvalidContext)
+
+ _, err = NewCacheTTL[int](t.Context(), 0, time.Second)
+ require.ErrorIs(t, err, errInvalidTTLInterval)
+
+ _, err = NewCacheTTL[int](t.Context(), time.Second, 0)
+ require.ErrorIs(t, err, errInvalidCleanupInterval)
+}
+
+func TestNewCacheTTL_ReturnsCacheForValidInputs(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), time.Second, 10*time.Millisecond)
+ require.NoError(t, err)
+ if c == nil {
+ t.Fatal("expected cache instance")
+ }
+}
+
+func TestCacheTTL_CalculateCacheKey_ReturnsSameValueForSameInput(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), time.Second, 10*time.Millisecond)
+ require.NoError(t, err)
+
+ key := "Authorization:Bearer token"
+ first := c.calculateKeyHash(key)
+ second := c.calculateKeyHash(key)
+
+ if first != second {
+ t.Fatalf("expected stable key hash: first=%d second=%d", first, second)
+ }
+}
+
+func TestCacheTTL_CalculateCacheKey_ReturnsDifferentValuesForDifferentInputs(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), time.Second, 10*time.Millisecond)
+ require.NoError(t, err)
+
+ first := c.calculateKeyHash("Authorization:Bearer token-a")
+ second := c.calculateKeyHash("Authorization:Bearer token-b")
+
+ if first == second {
+ t.Fatal("expected different key hashes for different inputs")
+ }
+}
+
+func TestCacheTTL_Set_Get_Delete_StoresReadsAndRemovesValue(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), time.Second, 10*time.Millisecond)
+ require.NoError(t, err)
+
+ c.Set("k", 42)
+
+ got, ok := c.Get("k")
+ if !ok {
+ t.Fatal("expected key to exist")
+ }
+ if got != 42 {
+ t.Fatalf("unexpected value: got %d, want %d", got, 42)
+ }
+
+ c.Delete("k")
+
+ _, ok = c.Get("k")
+ if ok {
+ t.Fatal("expected key to be deleted")
+ }
+}
+
+func TestCacheTTL_Get_ReturnsMissForUnknownKey(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), time.Second, 10*time.Millisecond)
+ require.NoError(t, err)
+
+ got, ok := c.Get("missing")
+ if ok {
+ t.Fatal("expected missing key")
+ }
+ if got != 0 {
+ t.Fatalf("unexpected zero value: got %d", got)
+ }
+}
+
+func TestCacheTTL_Get_ReturnsMissAfterTTLExpiration(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), 10*time.Millisecond, time.Second)
+ require.NoError(t, err)
+
+ c.Set("k", 7)
+ time.Sleep(20 * time.Millisecond)
+
+ _, ok := c.Get("k")
+ if ok {
+ t.Fatal("expected expired key to miss")
+ }
+}
+
+func TestCacheTTL_Size_TracksInsertUpdateDeleteAndMissingDelete(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), time.Second, 10*time.Millisecond)
+ require.NoError(t, err)
+
+ if got := c.Size(); got != 0 {
+ t.Fatalf("unexpected size: got %d, want %d", got, 0)
+ }
+
+ c.Set("a", 1)
+ if got := c.Size(); got != 1 {
+ t.Fatalf("unexpected size after first insert: got %d, want %d", got, 1)
+ }
+
+ c.Set("a", 2)
+ if got := c.Size(); got != 1 {
+ t.Fatalf("unexpected size after update: got %d, want %d", got, 1)
+ }
+
+ c.Set("b", 3)
+ if got := c.Size(); got != 2 {
+ t.Fatalf("unexpected size after second insert: got %d, want %d", got, 2)
+ }
+
+ c.Delete("missing")
+ if got := c.Size(); got != 2 {
+ t.Fatalf("unexpected size after deleting missing key: got %d, want %d", got, 2)
+ }
+
+ c.Delete("a")
+ if got := c.Size(); got != 1 {
+ t.Fatalf("unexpected size after deleting existing key: got %d, want %d", got, 1)
+ }
+}
+
+func TestCacheTTL_EvictionWorker_RemovesExpiredItemsAndUpdatesSize(t *testing.T) {
+ t.Parallel()
+
+ c, err := NewCacheTTL[int](t.Context(), 15*time.Millisecond, 5*time.Millisecond)
+ require.NoError(t, err)
+
+ c.Set("a", 1)
+ c.Set("b", 2)
+ if got := c.Size(); got != 2 {
+ t.Fatalf("unexpected initial size: got %d, want %d", got, 2)
+ }
+
+ eventually(t, 300*time.Millisecond, 5*time.Millisecond, func() bool {
+ return c.Size() == 0
+ })
+
+ if _, ok := c.Get("a"); ok {
+ t.Fatal("expected key a to be evicted")
+ }
+ if _, ok := c.Get("b"); ok {
+ t.Fatal("expected key b to be evicted")
+ }
+}
+
+func eventually(t *testing.T, timeout, interval time.Duration, fn func() bool) {
+ t.Helper()
+
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ if fn() {
+ return
+ }
+ time.Sleep(interval)
+ }
+
+ t.Fatal("condition was not met before timeout")
+}
diff --git a/utils/cache/common.go b/utils/cache/common.go
new file mode 100644
index 00000000000..709841c02c7
--- /dev/null
+++ b/utils/cache/common.go
@@ -0,0 +1,51 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cache
+
+import (
+ "errors"
+ "sync"
+ "time"
+
+ "golang.org/x/sys/cpu"
+)
+
+const (
+ // ShardCount must be a power of 2 for bitwise modulo.
+ // 256 is optimal for typical 16-64 core server deployments.
+ shardCount = 256
+ shardMask = shardCount - 1
+)
+
+var (
+ errInvalidContext = errors.New("context must not be nil")
+ errInvalidTTLInterval = errors.New("ttl must be greater than 0")
+ errInvalidCleanupInterval = errors.New("cleanupInterval must be greater than 0")
+)
+
+// item wraps the value and its expiration timestamp.
+type item[V any] struct {
+ value V
+ expires time.Time
+}
+
+// shard contains the actual map and a lock, padded to prevent false sharing.
+type shard[K comparable, V any] struct {
+ mu sync.RWMutex
+ items map[K]item[V]
+ size int64
+ // Pad to CPU Arch dependant bytes to prevent false sharing in L1 CPU cache.
+ _ cpu.CacheLinePad
+}
diff --git a/utils/rateLimiter/concurrencyLimiter.go b/utils/rateLimiter/concurrencyLimiter.go
new file mode 100644
index 00000000000..f47c6d3ba6d
--- /dev/null
+++ b/utils/rateLimiter/concurrencyLimiter.go
@@ -0,0 +1,62 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package rateLimiter package provides different implementations of high-performance rate limiters.
+package rateLimiter
+
+import (
+ "sync/atomic"
+
+ "golang.org/x/sys/cpu"
+)
+
+// ConcurrencyLimiter is used for limiting total active in-flight operations
+// (e.g., max 50 concurrent database connections or worker threads).
+// It fails fast in case there are no free slots available.
+// It uses atomic operations to minimize lock contention and maximize throughput in concurrent environments.
+// Useful in hot-paths.
+type ConcurrencyLimiter struct {
+ // Leading pad: Prevents false sharing with preceding fields if embedded in a larger struct.
+ // CPU arch dependant.
+ _ cpu.CacheLinePad
+ availableSlots atomic.Int32
+ // Trailing pad: Prevents false sharing with trailing fields or adjacent elements in a slice.
+ // CPU arch dependant.
+ _ cpu.CacheLinePad
+}
+
+// NewConcurrencyLimiter creates a new ConcurrencyLimiter with the specified maximum number of slots.
+func NewConcurrencyLimiter(maxSlots int32) *ConcurrencyLimiter {
+ cl := &ConcurrencyLimiter{}
+ cl.availableSlots.Store(maxSlots)
+ return cl
+}
+
+// TryAcquire claims an active slot. Returns false immediately if 0 slots remain.
+func (cl *ConcurrencyLimiter) TryAcquire() bool {
+ for {
+ current := cl.availableSlots.Load()
+ if current <= 0 {
+ return false // Fail fast
+ }
+ if cl.availableSlots.CompareAndSwap(current, current-1) {
+ return true
+ }
+ }
+}
+
+// Release frees an active slot back to the pool.
+func (cl *ConcurrencyLimiter) Release() {
+ cl.availableSlots.Add(1)
+}
diff --git a/utils/rateLimiter/concurrencyLimiter_bench_test.go b/utils/rateLimiter/concurrencyLimiter_bench_test.go
new file mode 100644
index 00000000000..ce397de8de5
--- /dev/null
+++ b/utils/rateLimiter/concurrencyLimiter_bench_test.go
@@ -0,0 +1,56 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rateLimiter
+
+import (
+ "runtime"
+ "testing"
+)
+
+func BenchmarkConcurrencyLimiter_TryAcquireRelease(b *testing.B) {
+ limiter := NewConcurrencyLimiter(1)
+ b.ReportAllocs()
+
+ for b.Loop() {
+ if !limiter.TryAcquire() {
+ b.Fatal("expected acquire to succeed")
+ }
+ limiter.Release()
+ }
+}
+
+func BenchmarkConcurrencyLimiter_TryAcquireWhenExhausted(b *testing.B) {
+ limiter := NewConcurrencyLimiter(0)
+ b.ReportAllocs()
+
+ for b.Loop() {
+ if limiter.TryAcquire() {
+ b.Fatal("expected acquire to fail for exhausted limiter")
+ }
+ }
+}
+
+func BenchmarkConcurrencyLimiter_ParallelAcquireRelease(b *testing.B) {
+ limiter := NewConcurrencyLimiter(int32(runtime.GOMAXPROCS(0))) //nolint:gosec
+ b.ReportAllocs()
+
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ if limiter.TryAcquire() {
+ limiter.Release()
+ }
+ }
+ })
+}
diff --git a/utils/rateLimiter/concurrencyLimiter_test.go b/utils/rateLimiter/concurrencyLimiter_test.go
new file mode 100644
index 00000000000..fb10a7874c5
--- /dev/null
+++ b/utils/rateLimiter/concurrencyLimiter_test.go
@@ -0,0 +1,111 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rateLimiter
+
+import (
+ "sync"
+ "sync/atomic"
+ "testing"
+)
+
+func TestNewConcurrencyLimiter_TryAcquireSucceedsUpToConfiguredLimit(t *testing.T) {
+ t.Parallel()
+
+ limiter := NewConcurrencyLimiter(3)
+
+ if !limiter.TryAcquire() {
+ t.Fatal("expected first acquire to succeed")
+ }
+ if !limiter.TryAcquire() {
+ t.Fatal("expected second acquire to succeed")
+ }
+ if !limiter.TryAcquire() {
+ t.Fatal("expected third acquire to succeed")
+ }
+ if limiter.TryAcquire() {
+ t.Fatal("expected acquire to fail when limit is exhausted")
+ }
+}
+
+func TestConcurrencyLimiter_ReleaseMakesSlotAvailableAgain(t *testing.T) {
+ t.Parallel()
+
+ limiter := NewConcurrencyLimiter(1)
+
+ if !limiter.TryAcquire() {
+ t.Fatal("expected initial acquire to succeed")
+ }
+ if limiter.TryAcquire() {
+ t.Fatal("expected acquire to fail when slot is already taken")
+ }
+
+ limiter.Release()
+
+ if !limiter.TryAcquire() {
+ t.Fatal("expected acquire to succeed after release")
+ }
+}
+
+func TestNewConcurrencyLimiter_WithZeroSlotsAlwaysRejectsAcquire(t *testing.T) {
+ t.Parallel()
+
+ limiter := NewConcurrencyLimiter(0)
+
+ if limiter.TryAcquire() {
+ t.Fatal("expected acquire to fail for zero-capacity limiter")
+ }
+}
+
+func TestConcurrencyLimiter_ReleaseWithoutPriorAcquireIncreasesAvailableCapacity(t *testing.T) {
+ t.Parallel()
+
+ limiter := NewConcurrencyLimiter(0)
+ limiter.Release()
+
+ if !limiter.TryAcquire() {
+ t.Fatal("expected acquire to succeed after release from zero capacity")
+ }
+ if limiter.TryAcquire() {
+ t.Fatal("expected second acquire to fail after consuming released slot")
+ }
+}
+
+func TestConcurrencyLimiter_TryAcquireConcurrentCallersNeverExceedsLimit(t *testing.T) {
+ t.Parallel()
+
+ const (
+ slots int32 = 8
+ workers int = 128
+ )
+
+ limiter := NewConcurrencyLimiter(slots)
+ var wg sync.WaitGroup
+ var successes atomic.Int32
+
+ wg.Add(workers)
+ for range workers {
+ go func() {
+ defer wg.Done()
+ if limiter.TryAcquire() {
+ successes.Add(1)
+ }
+ }()
+ }
+ wg.Wait()
+
+ if got := successes.Load(); got != slots {
+ t.Fatalf("unexpected number of successful acquires: got %d, want %d", got, slots)
+ }
+}