[17.0][MIG] monitoring_prometheus_queue_job: Migration to 17.0

monitoring_prometheus/README.rst

@@ -15,3 +15,28 @@ Current available metrics are:
 * Longpolling request count
 
 No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo
+
+Gathering metrics from the database
+-----------------------------------
+
+Metrics that require a database query are collected by the *Prometheus: gather metrics*
+scheduled action, which runs every 15 minutes and stores its result as
+``prometheus.metric`` records (one per metric name/labels combination). The */metrics*
+endpoint reads those records and publishes them as gauges. This indirection is required
+because the cron and the HTTP workers run in separate processes and cannot share the
+Prometheus registry.
+
+By default, only users in the *Administration / Settings* group (``base.group_system``)
+can read ``prometheus.metric`` records through the ORM/UI. The */metrics* endpoint
+remains public and reads the records with ``sudo()`` so Prometheus can keep scraping it
+without authentication.
+
+To publish new metrics, extend ``_gather_metrics`` on the ``prometheus.gatherer``
+abstract model and return the result of ``super()`` extended with your own entries::
+
+    {
+        "name": "odoo_some_metric",
+        "documentation": "What this metric measures",
+        "labels": {"label_name": "label_value"},
+        "value": 42,
+    }

monitoring_prometheus/__init__.py

@@ -1,2 +1,2 @@
-from . import controllers
 from . import models
+from . import controllers

monitoring_prometheus/__manifest__.py

@@ -4,12 +4,13 @@
 
 {
     "name": "Monitoring: Prometheus Metrics",
-    "version": "17.0.1.0.1",
+    "version": "17.0.1.1.0",
     "author": "Camptocamp,Odoo Community Association (OCA)",
     "license": "AGPL-3",
     "category": "Extra Tools",
     "depends": ["base", "web"],
     "website": "https://github.com/camptocamp/odoo-cloud-platform",
+    "data": ["security/ir.model.access.csv", "data/ir_cron.xml"],
     "external_dependencies": {
         "python": ["prometheus_client"],
     },

monitoring_prometheus/controllers/prometheus_metrics.py

@@ -1,15 +1,49 @@
-# Copyright 2016-2021 Camptocamp SA
+# Copyright 2016 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
-from prometheus_client import generate_latest
+import json
+import logging
 
-from odoo.http import Controller, route
+from prometheus_client import REGISTRY, CollectorRegistry, Gauge, generate_latest
+
+from odoo.http import Controller, request, route
 
 from ..models.psutils_helpers import get_process_info
 
+_logger = logging.getLogger(__name__)
+
 
 class PrometheusController(Controller):
+    def _build_gathered_registry(self):
+        """Build an isolated registry from the latest gathered metrics."""
+        registry = CollectorRegistry()
+        gauges = {}
+        records = request.env["prometheus.metric"].sudo().search([])
+        for record in records:
+            try:
+                labels = json.loads(record.labels or "{}")
+                gauge = gauges.get(record.name)
+                if gauge is None:
+                    gauge = gauges[record.name] = Gauge(
+                        record.name,
+                        record.documentation or "",
+                        sorted(labels),
+                        registry=registry,
+                    )
+                if labels:
+                    gauge.labels(**labels).set(record.value)
+                else:
+                    gauge.set(record.value)
+            except ValueError:
+                _logger.exception("Could not publish Prometheus metric %s", record.name)
+        return registry
+
     @route("/metrics", auth="public")
     def metrics(self):
         get_process_info()
-        return generate_latest()
+        output = generate_latest(REGISTRY)
+        try:
+            output += generate_latest(self._build_gathered_registry())
+        except Exception:
+            _logger.exception("Could not publish the gathered Prometheus metrics")
+        return output

monitoring_prometheus/data/ir_cron.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<odoo noupdate="1">
+    <record id="cron_prometheus_gather_metrics" model="ir.cron">
+        <field name="name">Prometheus: gather metrics</field>
+        <field name="model_id" ref="model_prometheus_gatherer" />
+        <field name="state">code</field>
+        <field name="code">model._cron_gather_metrics()</field>
+        <field name="interval_number">15</field>
+        <field name="interval_type">minutes</field>
+        <field name="numbercall">-1</field>
+        <field name="doall" eval="False" />
+        <field name="active" eval="True" />
+    </record>
+</odoo>

monitoring_prometheus/models/__init__.py

@@ -1,2 +1,4 @@
 from . import ir_http
+from . import prometheus_gatherer
+from . import prometheus_metric
 from . import psutils_helpers

monitoring_prometheus/models/prometheus_gatherer.py

@@ -0,0 +1,61 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from odoo import api, models
+
+
+class PrometheusGatherer(models.AbstractModel):
+    """Collect application metrics to be exposed on the /metrics endpoint.
+
+    Metrics are gathered by a cron and stored in ``prometheus.metric``
+    records, because crons and HTTP workers run in separate processes and
+    cannot share the Prometheus registry.
+    """
+
+    _name = "prometheus.gatherer"
+    _description = "Prometheus Metrics Gatherer"
+
+    @api.model
+    def _gather_metrics(self):
+        """Return the metrics as a list of dicts.
+
+        Each entry is of the form::
+
+            {
+                "name": "odoo_some_metric",
+                "documentation": "What this metric measures",
+                "labels": {"label_name": "label_value"},
+                "value": 42,
+            }
+
+        ``labels`` may be omitted for an unlabelled metric. All the entries
+        sharing a ``name`` must declare the same label names.
+
+        Modules extending this method must return the result of ``super()``
+        extended with their own entries.
+        """
+        return []
+
+    @api.model
+    def _cron_gather_metrics(self):
+        Metric = self.env["prometheus.metric"].sudo()
+        all_records = Metric.search([])
+        existing = {(record.name, record.labels): record for record in all_records}
+        touched = Metric.browse()
+        values_to_create = []
+        for metric in self._gather_metrics():
+            values = {
+                "name": metric["name"],
+                "documentation": metric.get("documentation", ""),
+                "labels": Metric._labels_key(metric.get("labels")),
+                "value": metric["value"],
+            }
+            record = existing.get((values["name"], values["labels"]))
+            if record:
+                record.write(values)
+                touched |= record
+            else:
+                values_to_create.append(values)
+        touched |= Metric.create(values_to_create)
+        (all_records - touched).unlink()
+        return True

monitoring_prometheus/models/prometheus_metric.py

@@ -0,0 +1,49 @@
+# Copyright 2026 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import json
+
+from odoo import api, fields, models
+
+
+class PrometheusMetric(models.Model):
+    """Store the metrics gathered by ``prometheus.gatherer``.
+
+    A record is stored per metric name/labels combination, because a single
+    metric name can be reported several times with different labels (e.g.
+    one gauge value per queue job state and channel).
+    """
+
+    _name = "prometheus.metric"
+    _description = "Prometheus Metric"
+
+    name = fields.Char(required=True, index=True)
+    documentation = fields.Char()
+    # JSON-serialized dict, used both to publish the metric and as part of
+    # the uniqueness key alongside ``name``.
+    labels = fields.Char(default="{}")
+    value = fields.Float(required=True)
+
+    @staticmethod
+    def _labels_key(labels):
+        return json.dumps(labels or {}, sort_keys=True)
+
+    @api.model
+    def _update_metric(self, metric):
+        """Create or update the record matching ``metric``, return it."""
+        labels_key = self._labels_key(metric.get("labels"))
+        record = self.search(
+            [("name", "=", metric["name"]), ("labels", "=", labels_key)],
+            limit=1,
+        )
+        values = {
+            "name": metric["name"],
+            "documentation": metric.get("documentation", ""),
+            "labels": labels_key,
+            "value": metric["value"],
+        }
+        if record:
+            record.write(values)
+        else:
+            record = self.create(values)
+        return record

monitoring_prometheus/security/ir.model.access.csv

@@ -0,0 +1,2 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_prometheus_metric_system,prometheus.metric system,model_prometheus_metric,base.group_system,1,1,1,1

monitoring_prometheus/tests/__init__.py

@@ -0,0 +1,2 @@
+from . import test_prometheus_controller
+from . import test_prometheus_metric

monitoring_prometheus/tests/test_prometheus_controller.py

@@ -0,0 +1,54 @@
+# Copyright 2026 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from odoo.tests import HttpCase, tagged
+from odoo.tools import mute_logger
+
+
+@tagged("post_install", "-at_install")
+class TestPrometheusController(HttpCase):
+    def test_00_metrics_publishes_current_records(self):
+        """Test the endpoint publishes current rows and drops deleted series."""
+        record = self.env["prometheus.metric"].create(
+            {
+                "name": "odoo_test_gauge",
+                "labels": '{"a": "1"}',
+                "value": 3,
+            }
+        )
+
+        self.assertIn(
+            'odoo_test_gauge{a="1"} 3.0',
+            self.url_open("/metrics").text,
+        )
+
+        record.unlink()
+        self.assertNotIn("odoo_test_gauge", self.url_open("/metrics").text)
+
+    @mute_logger("odoo.addons.monitoring_prometheus.controllers.prometheus_metrics")
+    def test_01_incompatible_metric_does_not_hide_later_records(self):
+        """Test incompatible labels do not hide following valid rows."""
+        self.env["prometheus.metric"].create(
+            [
+                {
+                    "name": "odoo_incompatible_gauge",
+                    "labels": '{"a": "1"}',
+                    "value": 1,
+                },
+                {
+                    "name": "odoo_incompatible_gauge",
+                    "labels": '{"b": "2"}',
+                    "value": 1,
+                },
+                {
+                    "name": "odoo_valid_gauge",
+                    "value": 2,
+                },
+            ]
+        )
+
+        response = self.url_open("/metrics").text
+
+        self.assertIn('odoo_incompatible_gauge{a="1"} 1.0', response)
+        self.assertNotIn('odoo_incompatible_gauge{b="2"}', response)
+        self.assertIn("odoo_valid_gauge 2.0", response)

monitoring_prometheus/tests/test_prometheus_metric.py

@@ -0,0 +1,68 @@
+# Copyright 2026 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from unittest.mock import patch
+
+from odoo.tests import TransactionCase
+
+
+class TestPrometheusMetric(TransactionCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.metric_model = cls.env["prometheus.metric"]
+        cls.gatherer = cls.env["prometheus.gatherer"]
+
+    def test_00_update_metric_reuses_canonical_labels(self):
+        """Test updating a metric reuses its canonical label combination."""
+        metric = self.metric_model._update_metric(
+            {
+                "name": "odoo_test_metric",
+                "labels": {"second": "2", "first": "1"},
+                "value": 1,
+            }
+        )
+
+        updated_metric = self.metric_model._update_metric(
+            {
+                "name": "odoo_test_metric",
+                "labels": {"first": "1", "second": "2"},
+                "value": 2,
+            }
+        )
+
+        self.assertEqual(updated_metric, metric)
+        self.assertEqual(updated_metric.value, 2)
+
+    def test_01_cron_removes_stale_metric_series(self):
+        """Test gathering creates, updates, and removes snapshot series."""
+        stale_metric = self.metric_model.create(
+            {"name": "odoo_stale_metric", "value": 1}
+        )
+        current_metric = self.metric_model.create(
+            {"name": "odoo_current_metric", "value": 1}
+        )
+        gathered_metrics = [
+            {
+                "name": "odoo_current_metric",
+                "documentation": "Current metric",
+                "value": 2,
+            },
+            {
+                "name": "odoo_new_metric",
+                "value": 3,
+            },
+        ]
+
+        with patch.object(
+            type(self.gatherer),
+            "_gather_metrics",
+            return_value=gathered_metrics,
+        ):
+            self.gatherer._cron_gather_metrics()
+
+        self.assertFalse(stale_metric.exists())
+        self.assertEqual(current_metric.value, 2)
+        self.assertEqual(current_metric.documentation, "Current metric")
+        new_metric = self.metric_model.search([("name", "=", "odoo_new_metric")])
+        self.assertEqual(new_metric.value, 3)

monitoring_prometheus_queue_job/README.rst

@@ -0,0 +1,20 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :alt: License
+
+=========================================
+Monitoring: Prometheus metrics queue jobs
+=========================================
+
+Expose the number of queue jobs to Prometheus through the */metrics* endpoint
+of ``monitoring_prometheus``.
+
+The gauge ``odoo_queue_job_count`` is published with the labels *state* and
+*channel*, for the states ``pending``, ``enqueued``, ``started``, ``failed``
+and ``wait_dependencies``. The terminal states ``done`` and ``cancelled`` are
+not reported.
+
+Values are refreshed by the *Prometheus: gather metrics* scheduled action, so
+they are a snapshot and not a real time count.
+
+This module is automatically installed when both ``monitoring_prometheus`` and
+``queue_job`` are installed.

monitoring_prometheus_queue_job/__init__.py

@@ -0,0 +1 @@
+from . import models

monitoring_prometheus_queue_job/__manifest__.py

@@ -0,0 +1,14 @@
+# Copyright 2026 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Monitoring: Prometheus Metrics for Queue Jobs",
+    "version": "17.0.1.0.0",
+    "author": "Camptocamp,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "Extra Tools",
+    "depends": ["monitoring_prometheus", "queue_job"],
+    "website": "https://github.com/camptocamp/odoo-cloud-platform",
+    "auto_install": True,
+}

monitoring_prometheus_queue_job/models/__init__.py

@@ -0,0 +1 @@
+from . import prometheus_gatherer

monitoring_prometheus_queue_job/models/prometheus_gatherer.py

@@ -0,0 +1,40 @@
+# Copyright 2026 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from odoo import api, models
+
+MONITORED_STATES = ("pending", "enqueued", "started", "failed", "wait_dependencies")
+GAUGE_NAME = "odoo_queue_job_count"
+GAUGE_DOC = "Number of queue jobs per state and channel"
+
+
+class PrometheusGatherer(models.AbstractModel):
+    _inherit = "prometheus.gatherer"
+
+    @api.model
+    def _gather_metrics(self):
+        metrics = super()._gather_metrics()
+        groups = self.env["queue.job"].read_group(
+            [("state", "in", list(MONITORED_STATES))],
+            ["state", "channel"],
+            ["state", "channel"],
+            lazy=False,
+        )
+        channels = self.env["queue.job.channel"].search([]).mapped("complete_name")
+        # seed every combination so a series never disappears when its jobs are gone
+        counts = {
+            (state, channel): 0 for state in MONITORED_STATES for channel in channels
+        }
+        for group in groups:
+            key = (group["state"], group["channel"])
+            counts[key] = counts.get(key, 0) + group["__count"]
+        metrics.extend(
+            {
+                "name": GAUGE_NAME,
+                "documentation": GAUGE_DOC,
+                "labels": {"state": state, "channel": channel},
+                "value": count,
+            }
+            for (state, channel), count in counts.items()
+        )
+        return metrics

monitoring_prometheus_queue_job/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"

monitoring_prometheus_queue_job/tests/__init__.py

@@ -0,0 +1 @@
+from . import test_prometheus_gatherer

monitoring_prometheus_queue_job/tests/test_prometheus_gatherer.py

@@ -0,0 +1,49 @@
+# Copyright 2026 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from odoo.tests import TransactionCase
+
+from ..models.prometheus_gatherer import GAUGE_NAME, MONITORED_STATES
+
+
+class TestPrometheusGatherer(TransactionCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.gatherer = cls.env["prometheus.gatherer"]
+        cls.channel = cls.env["queue.job.channel"].create(
+            {
+                "name": "test_prometheus",
+                "parent_id": cls.env.ref("queue_job.channel_root").id,
+            }
+        )
+        Job = cls.env["queue.job"].with_context(
+            _job_edit_sentinel=cls.env["queue.job"].EDIT_SENTINEL
+        )
+        Job.create(
+            [
+                {
+                    "uuid": f"test_prometheus_{index}",
+                    "user_id": cls.env.user.id,
+                    "state": "pending",
+                    "channel": cls.channel.complete_name,
+                    "model_name": "queue.job",
+                    "method_name": "write",
+                }
+                for index in range(2)
+            ]
+        )
+
+    def test_00_gather_queue_job_counts_for_every_state(self):
+        """Test queue metrics include counts and zero series for every state."""
+        metrics = self.gatherer._gather_metrics()
+        counts = {
+            metric["labels"]["state"]: metric["value"]
+            for metric in metrics
+            if metric["name"] == GAUGE_NAME
+            and metric["labels"]["channel"] == self.channel.complete_name
+        }
+        self.assertEqual(
+            counts,
+            {state: 2 if state == "pending" else 0 for state in MONITORED_STATES},
+        )

oca_dependencies.txt

@@ -1 +1,2 @@
 server-env
+queue