outposts: Create separate metrics service in Kubernetes (#21229)

* outposts: create separate metrics service

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix service monitor plumbing

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* update docs

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* format

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add some static tests

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* make metrics service ClusterIP

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* update service monitor when labels mismatch

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

---------

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

authentik/outposts/controllers/base.py

@@ -58,6 +58,9 @@ class BaseController:
         self.connection = connection
         self.logger = get_logger()
         self.deployment_ports = []
+        self.metrics_ports = [
+            DeploymentPort(9300, "http-metrics", "tcp"),
+        ]
 
     def up(self):
         """Called by scheduled task to reconcile deployment/service/etc"""

authentik/outposts/controllers/k8s/service.py

@@ -2,7 +2,7 @@
 
 from typing import TYPE_CHECKING
 
-from kubernetes.client import CoreV1Api, V1Service, V1ServicePort, V1ServiceSpec
+from kubernetes.client import CoreV1Api, V1ObjectMeta, V1Service, V1ServicePort, V1ServiceSpec
 
 from authentik.outposts.controllers.base import FIELD_MANAGER
 from authentik.outposts.controllers.k8s.base import KubernetesObjectReconciler
@@ -84,3 +84,47 @@ class ServiceReconciler(KubernetesObjectReconciler[V1Service]):
             reference,
             field_manager=FIELD_MANAGER,
         )
+
+
+class MetricsServiceReconciler(ServiceReconciler):
+    @property
+    def noop(self) -> bool:
+        return self.is_embedded
+
+    @staticmethod
+    def reconciler_name() -> str:
+        return "service-metrics"
+
+    @property
+    def name(self):
+        name_suffix = "-metrics"
+        name = super().name
+        return name[: 63 - len(name_suffix)] + name_suffix
+
+    def get_object_meta(self, **kwargs) -> V1ObjectMeta:
+        meta: V1ObjectMeta = super().get_object_meta(**kwargs)
+        meta.labels["goauthentik.io/service-type"] = "metrics"
+        return meta
+
+    def get_reference_object(self) -> V1Service:
+        """Get deployment object for outpost"""
+        meta = self.get_object_meta(name=self.name)
+        ports = []
+        for port in self.controller.metrics_ports:
+            ports.append(
+                V1ServicePort(
+                    name=port.name,
+                    port=port.port,
+                    protocol=port.protocol.upper(),
+                    target_port=port.inner_port or port.port,
+                )
+            )
+        selector_labels = DeploymentReconciler(self.controller).get_pod_meta()
+        return V1Service(
+            metadata=meta,
+            spec=V1ServiceSpec(
+                ports=ports,
+                selector=selector_labels,
+                type="ClusterIP",
+            ),
+        )

authentik/outposts/controllers/k8s/service_monitor.py

@@ -8,6 +8,8 @@ from kubernetes.client import ApiextensionsV1Api, CustomObjectsApi
 
 from authentik.outposts.controllers.base import FIELD_MANAGER
 from authentik.outposts.controllers.k8s.base import KubernetesObjectReconciler
+from authentik.outposts.controllers.k8s.service import MetricsServiceReconciler
+from authentik.outposts.controllers.k8s.triggers import NeedsUpdate
 
 if TYPE_CHECKING:
     from authentik.outposts.controllers.kubernetes import KubernetesController
@@ -55,6 +57,10 @@ class PrometheusServiceMonitor:
     metadata: PrometheusServiceMonitorMetadata
     spec: PrometheusServiceMonitorSpec
 
+    def to_dict(self):
+        """`to_dict` to conform to how the kubernetes client converts objects to dicts"""
+        return asdict(self)
+
 
 CRD_NAME = "servicemonitors.monitoring.coreos.com"
 CRD_GROUP = "monitoring.coreos.com"
@@ -74,6 +80,11 @@ class PrometheusServiceMonitorReconciler(KubernetesObjectReconciler[PrometheusSe
     def reconciler_name() -> str:
         return "prometheus servicemonitor"
 
+    def reconcile(self, current: PrometheusServiceMonitor, reference: PrometheusServiceMonitor):
+        if current.spec.selector.matchLabels != reference.spec.selector.matchLabels:
+            raise NeedsUpdate()
+        super().reconcile(current, reference)
+
     @property
     def noop(self) -> bool:
         if not self._crd_exists():
@@ -108,7 +119,9 @@ class PrometheusServiceMonitorReconciler(KubernetesObjectReconciler[PrometheusSe
                     )
                 ],
                 selector=PrometheusServiceMonitorSpecSelector(
-                    matchLabels=self.get_object_meta(name=self.name).labels,
+                    matchLabels=MetricsServiceReconciler(self.controller)
+                    .get_object_meta(name=self.name)
+                    .labels,
                 ),
             ),
         )

authentik/outposts/controllers/kubernetes.py

@@ -18,7 +18,7 @@ from authentik.outposts.controllers.base import BaseClient, BaseController, Cont
 from authentik.outposts.controllers.k8s.base import KubernetesObjectReconciler
 from authentik.outposts.controllers.k8s.deployment import DeploymentReconciler
 from authentik.outposts.controllers.k8s.secret import SecretReconciler
-from authentik.outposts.controllers.k8s.service import ServiceReconciler
+from authentik.outposts.controllers.k8s.service import MetricsServiceReconciler, ServiceReconciler
 from authentik.outposts.controllers.k8s.service_monitor import PrometheusServiceMonitorReconciler
 from authentik.outposts.models import (
     KubernetesServiceConnection,
@@ -74,6 +74,7 @@ class KubernetesController(BaseController):
             SecretReconciler.reconciler_name(): SecretReconciler,
             DeploymentReconciler.reconciler_name(): DeploymentReconciler,
             ServiceReconciler.reconciler_name(): ServiceReconciler,
+            MetricsServiceReconciler.reconciler_name(): MetricsServiceReconciler,
             PrometheusServiceMonitorReconciler.reconciler_name(): (
                 PrometheusServiceMonitorReconciler
             ),
@@ -82,6 +83,7 @@ class KubernetesController(BaseController):
             SecretReconciler.reconciler_name(),
             DeploymentReconciler.reconciler_name(),
             ServiceReconciler.reconciler_name(),
+            MetricsServiceReconciler.reconciler_name(),
             PrometheusServiceMonitorReconciler.reconciler_name(),
         ]
 

authentik/outposts/tests/test_controller_k8s.py

@@ -1,11 +1,16 @@
 """Kubernetes controller tests"""
 
+from unittest.mock import MagicMock, patch
+
 from django.test import TestCase
+from kubernetes.client import ApiClient
+from yaml import SafeLoader, load_all
 
 from authentik.blueprints.tests import reconcile_app
 from authentik.lib.generators import generate_id
 from authentik.outposts.apps import MANAGED_OUTPOST
 from authentik.outposts.controllers.k8s.deployment import DeploymentReconciler
+from authentik.outposts.controllers.k8s.service_monitor import PrometheusServiceMonitorReconciler
 from authentik.outposts.controllers.kubernetes import KubernetesController
 from authentik.outposts.models import KubernetesServiceConnection, Outpost, OutpostType
 
@@ -28,7 +33,7 @@ class KubernetesControllerTests(TestCase):
             self.integration,
             # Pass something not-none as client so we don't
             # attempt to connect to K8s as that's not needed
-            client=self,
+            client=ApiClient(),
         )
         rec = DeploymentReconciler(controller)
         self.assertEqual(rec.name, "ak-outpost-authentik-embedded-outpost")
@@ -42,3 +47,18 @@ class KubernetesControllerTests(TestCase):
         controller.outpost.config = _cfg
         self.assertEqual(rec.name, f"outpost-{controller.outpost.uuid.hex}")
         self.assertLess(len(rec.name), 64)
+
+    def test_static(self):
+        self.controller = KubernetesController(
+            self.outpost,
+            self.integration,
+            # Pass something not-none as client so we don't
+            # attempt to connect to K8s as that's not needed
+            client=ApiClient(),
+        )
+        with patch.object(
+            PrometheusServiceMonitorReconciler, "_crd_exists", MagicMock(return_value=True)
+        ):
+            manifest = self.controller.get_static_deployment()
+        manifests = list(load_all(manifest, Loader=SafeLoader))
+        self.assertEqual(len(manifests), 5)

authentik/providers/proxy/controllers/kubernetes.py

@@ -15,7 +15,6 @@ class ProxyKubernetesController(KubernetesController):
         super().__init__(outpost, connection)
         self.deployment_ports = [
             DeploymentPort(9000, "http", "tcp"),
-            DeploymentPort(9300, "http-metrics", "tcp"),
             DeploymentPort(9443, "https", "tcp"),
         ]
         self.reconcilers[IngressReconciler.reconciler_name()] = IngressReconciler

tests/integration/test_proxy_kubernetes.py

@@ -46,7 +46,7 @@ class TestProxyKubernetes(TestCase):
 
         self.controller = ProxyKubernetesController(outpost, service_connection)
         manifest = self.controller.get_static_deployment()
-        self.assertEqual(len(list(yaml.load_all(manifest, Loader=yaml.SafeLoader))), 4)
+        self.assertEqual(len(list(yaml.load_all(manifest, Loader=yaml.SafeLoader))), 5)
 
     @pytest.mark.timeout(120, func_only=True)
     def test_kubernetes_controller_ingress(self):

website/docs/add-secure-apps/outposts/integrations/kubernetes.md

@@ -9,7 +9,8 @@ This integration has the advantage over manual deployments of automatic updates
 This integration creates the following objects:
 
 - Deployment for the outpost container
-- Service
+- Service for protocol access
+- Service for metrics access
 - Secret to store the token
 - Prometheus ServiceMonitor (if the Prometheus Operator is installed in the target cluster)
 - Ingress (only Proxy outposts)
@@ -32,6 +33,7 @@ The following outpost settings are used:
     - 'secret'
     - 'deployment'
     - 'service'
+    - 'service-metrics'
     - 'prometheus servicemonitor'
     - 'ingress'
     - 'traefik middleware'