From 88cafedadc8cc853dfbc346c166b47316a19b3ac Mon Sep 17 00:00:00 2001 From: Maksym Yankin Date: Wed, 30 Sep 2026 19:23:35 +0300 Subject: [PATCH] [17.0][MIG] monitoring_prometheus_queue_job: Migration to 17.0 --- monitoring_prometheus/README.rst | 25 +++++++ monitoring_prometheus/__init__.py | 2 +- monitoring_prometheus/__manifest__.py | 3 +- .../controllers/prometheus_metrics.py | 42 ++++++++++-- monitoring_prometheus/data/ir_cron.xml | 14 ++++ monitoring_prometheus/models/__init__.py | 2 + .../models/prometheus_gatherer.py | 61 +++++++++++++++++ .../models/prometheus_metric.py | 49 +++++++++++++ .../security/ir.model.access.csv | 2 + monitoring_prometheus/tests/__init__.py | 2 + .../tests/test_prometheus_controller.py | 54 +++++++++++++++ .../tests/test_prometheus_metric.py | 68 +++++++++++++++++++ monitoring_prometheus_queue_job/README.rst | 20 ++++++ monitoring_prometheus_queue_job/__init__.py | 1 + .../__manifest__.py | 14 ++++ .../models/__init__.py | 1 + .../models/prometheus_gatherer.py | 40 +++++++++++ .../pyproject.toml | 3 + .../tests/__init__.py | 1 + .../tests/test_prometheus_gatherer.py | 49 +++++++++++++ oca_dependencies.txt | 1 + 21 files changed, 448 insertions(+), 6 deletions(-) create mode 100644 monitoring_prometheus/data/ir_cron.xml create mode 100644 monitoring_prometheus/models/prometheus_gatherer.py create mode 100644 monitoring_prometheus/models/prometheus_metric.py create mode 100644 monitoring_prometheus/security/ir.model.access.csv create mode 100644 monitoring_prometheus/tests/__init__.py create mode 100644 monitoring_prometheus/tests/test_prometheus_controller.py create mode 100644 monitoring_prometheus/tests/test_prometheus_metric.py create mode 100644 monitoring_prometheus_queue_job/README.rst create mode 100644 monitoring_prometheus_queue_job/__init__.py create mode 100644 monitoring_prometheus_queue_job/__manifest__.py create mode 100644 monitoring_prometheus_queue_job/models/__init__.py create mode 100644 monitoring_prometheus_queue_job/models/prometheus_gatherer.py create mode 100644 monitoring_prometheus_queue_job/pyproject.toml create mode 100644 monitoring_prometheus_queue_job/tests/__init__.py create mode 100644 monitoring_prometheus_queue_job/tests/test_prometheus_gatherer.py diff --git a/monitoring_prometheus/README.rst b/monitoring_prometheus/README.rst index aa98ffe..18da356 100644 --- a/monitoring_prometheus/README.rst +++ b/monitoring_prometheus/README.rst @@ -15,3 +15,28 @@ Current available metrics are: * Longpolling request count No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo + +Gathering metrics from the database +----------------------------------- + +Metrics that require a database query are collected by the *Prometheus: gather metrics* +scheduled action, which runs every 15 minutes and stores its result as +``prometheus.metric`` records (one per metric name/labels combination). The */metrics* +endpoint reads those records and publishes them as gauges. This indirection is required +because the cron and the HTTP workers run in separate processes and cannot share the +Prometheus registry. + +By default, only users in the *Administration / Settings* group (``base.group_system``) +can read ``prometheus.metric`` records through the ORM/UI. The */metrics* endpoint +remains public and reads the records with ``sudo()`` so Prometheus can keep scraping it +without authentication. + +To publish new metrics, extend ``_gather_metrics`` on the ``prometheus.gatherer`` +abstract model and return the result of ``super()`` extended with your own entries:: + + { + "name": "odoo_some_metric", + "documentation": "What this metric measures", + "labels": {"label_name": "label_value"}, + "value": 42, + } diff --git a/monitoring_prometheus/__init__.py b/monitoring_prometheus/__init__.py index 91c5580..f7209b1 100644 --- a/monitoring_prometheus/__init__.py +++ b/monitoring_prometheus/__init__.py @@ -1,2 +1,2 @@ -from . import controllers from . import models +from . import controllers diff --git a/monitoring_prometheus/__manifest__.py b/monitoring_prometheus/__manifest__.py index 02bb5f6..9da50c4 100644 --- a/monitoring_prometheus/__manifest__.py +++ b/monitoring_prometheus/__manifest__.py @@ -4,12 +4,13 @@ { "name": "Monitoring: Prometheus Metrics", - "version": "17.0.1.0.1", + "version": "17.0.1.1.0", "author": "Camptocamp,Odoo Community Association (OCA)", "license": "AGPL-3", "category": "Extra Tools", "depends": ["base", "web"], "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": ["security/ir.model.access.csv", "data/ir_cron.xml"], "external_dependencies": { "python": ["prometheus_client"], }, diff --git a/monitoring_prometheus/controllers/prometheus_metrics.py b/monitoring_prometheus/controllers/prometheus_metrics.py index e3e4cdb..fac09e5 100644 --- a/monitoring_prometheus/controllers/prometheus_metrics.py +++ b/monitoring_prometheus/controllers/prometheus_metrics.py @@ -1,15 +1,49 @@ -# Copyright 2016-2021 Camptocamp SA +# Copyright 2016 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -from prometheus_client import generate_latest +import json +import logging -from odoo.http import Controller, route +from prometheus_client import REGISTRY, CollectorRegistry, Gauge, generate_latest + +from odoo.http import Controller, request, route from ..models.psutils_helpers import get_process_info +_logger = logging.getLogger(__name__) + class PrometheusController(Controller): + def _build_gathered_registry(self): + """Build an isolated registry from the latest gathered metrics.""" + registry = CollectorRegistry() + gauges = {} + records = request.env["prometheus.metric"].sudo().search([]) + for record in records: + try: + labels = json.loads(record.labels or "{}") + gauge = gauges.get(record.name) + if gauge is None: + gauge = gauges[record.name] = Gauge( + record.name, + record.documentation or "", + sorted(labels), + registry=registry, + ) + if labels: + gauge.labels(**labels).set(record.value) + else: + gauge.set(record.value) + except ValueError: + _logger.exception("Could not publish Prometheus metric %s", record.name) + return registry + @route("/metrics", auth="public") def metrics(self): get_process_info() - return generate_latest() + output = generate_latest(REGISTRY) + try: + output += generate_latest(self._build_gathered_registry()) + except Exception: + _logger.exception("Could not publish the gathered Prometheus metrics") + return output diff --git a/monitoring_prometheus/data/ir_cron.xml b/monitoring_prometheus/data/ir_cron.xml new file mode 100644 index 0000000..82b5aba --- /dev/null +++ b/monitoring_prometheus/data/ir_cron.xml @@ -0,0 +1,14 @@ + + + + Prometheus: gather metrics + + code + model._cron_gather_metrics() + 15 + minutes + -1 + + + + diff --git a/monitoring_prometheus/models/__init__.py b/monitoring_prometheus/models/__init__.py index 15f6add..6b79872 100644 --- a/monitoring_prometheus/models/__init__.py +++ b/monitoring_prometheus/models/__init__.py @@ -1,2 +1,4 @@ from . import ir_http +from . import prometheus_gatherer +from . import prometheus_metric from . import psutils_helpers diff --git a/monitoring_prometheus/models/prometheus_gatherer.py b/monitoring_prometheus/models/prometheus_gatherer.py new file mode 100644 index 0000000..811c266 --- /dev/null +++ b/monitoring_prometheus/models/prometheus_gatherer.py @@ -0,0 +1,61 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo import api, models + + +class PrometheusGatherer(models.AbstractModel): + """Collect application metrics to be exposed on the /metrics endpoint. + + Metrics are gathered by a cron and stored in ``prometheus.metric`` + records, because crons and HTTP workers run in separate processes and + cannot share the Prometheus registry. + """ + + _name = "prometheus.gatherer" + _description = "Prometheus Metrics Gatherer" + + @api.model + def _gather_metrics(self): + """Return the metrics as a list of dicts. + + Each entry is of the form:: + + { + "name": "odoo_some_metric", + "documentation": "What this metric measures", + "labels": {"label_name": "label_value"}, + "value": 42, + } + + ``labels`` may be omitted for an unlabelled metric. All the entries + sharing a ``name`` must declare the same label names. + + Modules extending this method must return the result of ``super()`` + extended with their own entries. + """ + return [] + + @api.model + def _cron_gather_metrics(self): + Metric = self.env["prometheus.metric"].sudo() + all_records = Metric.search([]) + existing = {(record.name, record.labels): record for record in all_records} + touched = Metric.browse() + values_to_create = [] + for metric in self._gather_metrics(): + values = { + "name": metric["name"], + "documentation": metric.get("documentation", ""), + "labels": Metric._labels_key(metric.get("labels")), + "value": metric["value"], + } + record = existing.get((values["name"], values["labels"])) + if record: + record.write(values) + touched |= record + else: + values_to_create.append(values) + touched |= Metric.create(values_to_create) + (all_records - touched).unlink() + return True diff --git a/monitoring_prometheus/models/prometheus_metric.py b/monitoring_prometheus/models/prometheus_metric.py new file mode 100644 index 0000000..3f5bb5f --- /dev/null +++ b/monitoring_prometheus/models/prometheus_metric.py @@ -0,0 +1,49 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import json + +from odoo import api, fields, models + + +class PrometheusMetric(models.Model): + """Store the metrics gathered by ``prometheus.gatherer``. + + A record is stored per metric name/labels combination, because a single + metric name can be reported several times with different labels (e.g. + one gauge value per queue job state and channel). + """ + + _name = "prometheus.metric" + _description = "Prometheus Metric" + + name = fields.Char(required=True, index=True) + documentation = fields.Char() + # JSON-serialized dict, used both to publish the metric and as part of + # the uniqueness key alongside ``name``. + labels = fields.Char(default="{}") + value = fields.Float(required=True) + + @staticmethod + def _labels_key(labels): + return json.dumps(labels or {}, sort_keys=True) + + @api.model + def _update_metric(self, metric): + """Create or update the record matching ``metric``, return it.""" + labels_key = self._labels_key(metric.get("labels")) + record = self.search( + [("name", "=", metric["name"]), ("labels", "=", labels_key)], + limit=1, + ) + values = { + "name": metric["name"], + "documentation": metric.get("documentation", ""), + "labels": labels_key, + "value": metric["value"], + } + if record: + record.write(values) + else: + record = self.create(values) + return record diff --git a/monitoring_prometheus/security/ir.model.access.csv b/monitoring_prometheus/security/ir.model.access.csv new file mode 100644 index 0000000..3da8765 --- /dev/null +++ b/monitoring_prometheus/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_prometheus_metric_system,prometheus.metric system,model_prometheus_metric,base.group_system,1,1,1,1 diff --git a/monitoring_prometheus/tests/__init__.py b/monitoring_prometheus/tests/__init__.py new file mode 100644 index 0000000..47d634f --- /dev/null +++ b/monitoring_prometheus/tests/__init__.py @@ -0,0 +1,2 @@ +from . import test_prometheus_controller +from . import test_prometheus_metric diff --git a/monitoring_prometheus/tests/test_prometheus_controller.py b/monitoring_prometheus/tests/test_prometheus_controller.py new file mode 100644 index 0000000..d76c7a0 --- /dev/null +++ b/monitoring_prometheus/tests/test_prometheus_controller.py @@ -0,0 +1,54 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo.tests import HttpCase, tagged +from odoo.tools import mute_logger + + +@tagged("post_install", "-at_install") +class TestPrometheusController(HttpCase): + def test_00_metrics_publishes_current_records(self): + """Test the endpoint publishes current rows and drops deleted series.""" + record = self.env["prometheus.metric"].create( + { + "name": "odoo_test_gauge", + "labels": '{"a": "1"}', + "value": 3, + } + ) + + self.assertIn( + 'odoo_test_gauge{a="1"} 3.0', + self.url_open("/metrics").text, + ) + + record.unlink() + self.assertNotIn("odoo_test_gauge", self.url_open("/metrics").text) + + @mute_logger("odoo.addons.monitoring_prometheus.controllers.prometheus_metrics") + def test_01_incompatible_metric_does_not_hide_later_records(self): + """Test incompatible labels do not hide following valid rows.""" + self.env["prometheus.metric"].create( + [ + { + "name": "odoo_incompatible_gauge", + "labels": '{"a": "1"}', + "value": 1, + }, + { + "name": "odoo_incompatible_gauge", + "labels": '{"b": "2"}', + "value": 1, + }, + { + "name": "odoo_valid_gauge", + "value": 2, + }, + ] + ) + + response = self.url_open("/metrics").text + + self.assertIn('odoo_incompatible_gauge{a="1"} 1.0', response) + self.assertNotIn('odoo_incompatible_gauge{b="2"}', response) + self.assertIn("odoo_valid_gauge 2.0", response) diff --git a/monitoring_prometheus/tests/test_prometheus_metric.py b/monitoring_prometheus/tests/test_prometheus_metric.py new file mode 100644 index 0000000..b6cd482 --- /dev/null +++ b/monitoring_prometheus/tests/test_prometheus_metric.py @@ -0,0 +1,68 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from unittest.mock import patch + +from odoo.tests import TransactionCase + + +class TestPrometheusMetric(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.metric_model = cls.env["prometheus.metric"] + cls.gatherer = cls.env["prometheus.gatherer"] + + def test_00_update_metric_reuses_canonical_labels(self): + """Test updating a metric reuses its canonical label combination.""" + metric = self.metric_model._update_metric( + { + "name": "odoo_test_metric", + "labels": {"second": "2", "first": "1"}, + "value": 1, + } + ) + + updated_metric = self.metric_model._update_metric( + { + "name": "odoo_test_metric", + "labels": {"first": "1", "second": "2"}, + "value": 2, + } + ) + + self.assertEqual(updated_metric, metric) + self.assertEqual(updated_metric.value, 2) + + def test_01_cron_removes_stale_metric_series(self): + """Test gathering creates, updates, and removes snapshot series.""" + stale_metric = self.metric_model.create( + {"name": "odoo_stale_metric", "value": 1} + ) + current_metric = self.metric_model.create( + {"name": "odoo_current_metric", "value": 1} + ) + gathered_metrics = [ + { + "name": "odoo_current_metric", + "documentation": "Current metric", + "value": 2, + }, + { + "name": "odoo_new_metric", + "value": 3, + }, + ] + + with patch.object( + type(self.gatherer), + "_gather_metrics", + return_value=gathered_metrics, + ): + self.gatherer._cron_gather_metrics() + + self.assertFalse(stale_metric.exists()) + self.assertEqual(current_metric.value, 2) + self.assertEqual(current_metric.documentation, "Current metric") + new_metric = self.metric_model.search([("name", "=", "odoo_new_metric")]) + self.assertEqual(new_metric.value, 3) diff --git a/monitoring_prometheus_queue_job/README.rst b/monitoring_prometheus_queue_job/README.rst new file mode 100644 index 0000000..8cac167 --- /dev/null +++ b/monitoring_prometheus_queue_job/README.rst @@ -0,0 +1,20 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License + +========================================= +Monitoring: Prometheus metrics queue jobs +========================================= + +Expose the number of queue jobs to Prometheus through the */metrics* endpoint +of ``monitoring_prometheus``. + +The gauge ``odoo_queue_job_count`` is published with the labels *state* and +*channel*, for the states ``pending``, ``enqueued``, ``started``, ``failed`` +and ``wait_dependencies``. The terminal states ``done`` and ``cancelled`` are +not reported. + +Values are refreshed by the *Prometheus: gather metrics* scheduled action, so +they are a snapshot and not a real time count. + +This module is automatically installed when both ``monitoring_prometheus`` and +``queue_job`` are installed. diff --git a/monitoring_prometheus_queue_job/__init__.py b/monitoring_prometheus_queue_job/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/monitoring_prometheus_queue_job/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/monitoring_prometheus_queue_job/__manifest__.py b/monitoring_prometheus_queue_job/__manifest__.py new file mode 100644 index 0000000..5f806b8 --- /dev/null +++ b/monitoring_prometheus_queue_job/__manifest__.py @@ -0,0 +1,14 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Monitoring: Prometheus Metrics for Queue Jobs", + "version": "17.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": ["monitoring_prometheus", "queue_job"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "auto_install": True, +} diff --git a/monitoring_prometheus_queue_job/models/__init__.py b/monitoring_prometheus_queue_job/models/__init__.py new file mode 100644 index 0000000..18bb3bc --- /dev/null +++ b/monitoring_prometheus_queue_job/models/__init__.py @@ -0,0 +1 @@ +from . import prometheus_gatherer diff --git a/monitoring_prometheus_queue_job/models/prometheus_gatherer.py b/monitoring_prometheus_queue_job/models/prometheus_gatherer.py new file mode 100644 index 0000000..be493f7 --- /dev/null +++ b/monitoring_prometheus_queue_job/models/prometheus_gatherer.py @@ -0,0 +1,40 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo import api, models + +MONITORED_STATES = ("pending", "enqueued", "started", "failed", "wait_dependencies") +GAUGE_NAME = "odoo_queue_job_count" +GAUGE_DOC = "Number of queue jobs per state and channel" + + +class PrometheusGatherer(models.AbstractModel): + _inherit = "prometheus.gatherer" + + @api.model + def _gather_metrics(self): + metrics = super()._gather_metrics() + groups = self.env["queue.job"].read_group( + [("state", "in", list(MONITORED_STATES))], + ["state", "channel"], + ["state", "channel"], + lazy=False, + ) + channels = self.env["queue.job.channel"].search([]).mapped("complete_name") + # seed every combination so a series never disappears when its jobs are gone + counts = { + (state, channel): 0 for state in MONITORED_STATES for channel in channels + } + for group in groups: + key = (group["state"], group["channel"]) + counts[key] = counts.get(key, 0) + group["__count"] + metrics.extend( + { + "name": GAUGE_NAME, + "documentation": GAUGE_DOC, + "labels": {"state": state, "channel": channel}, + "value": count, + } + for (state, channel), count in counts.items() + ) + return metrics diff --git a/monitoring_prometheus_queue_job/pyproject.toml b/monitoring_prometheus_queue_job/pyproject.toml new file mode 100644 index 0000000..4231d0c --- /dev/null +++ b/monitoring_prometheus_queue_job/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/monitoring_prometheus_queue_job/tests/__init__.py b/monitoring_prometheus_queue_job/tests/__init__.py new file mode 100644 index 0000000..2cfd9f4 --- /dev/null +++ b/monitoring_prometheus_queue_job/tests/__init__.py @@ -0,0 +1 @@ +from . import test_prometheus_gatherer diff --git a/monitoring_prometheus_queue_job/tests/test_prometheus_gatherer.py b/monitoring_prometheus_queue_job/tests/test_prometheus_gatherer.py new file mode 100644 index 0000000..a4b0244 --- /dev/null +++ b/monitoring_prometheus_queue_job/tests/test_prometheus_gatherer.py @@ -0,0 +1,49 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo.tests import TransactionCase + +from ..models.prometheus_gatherer import GAUGE_NAME, MONITORED_STATES + + +class TestPrometheusGatherer(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.gatherer = cls.env["prometheus.gatherer"] + cls.channel = cls.env["queue.job.channel"].create( + { + "name": "test_prometheus", + "parent_id": cls.env.ref("queue_job.channel_root").id, + } + ) + Job = cls.env["queue.job"].with_context( + _job_edit_sentinel=cls.env["queue.job"].EDIT_SENTINEL + ) + Job.create( + [ + { + "uuid": f"test_prometheus_{index}", + "user_id": cls.env.user.id, + "state": "pending", + "channel": cls.channel.complete_name, + "model_name": "queue.job", + "method_name": "write", + } + for index in range(2) + ] + ) + + def test_00_gather_queue_job_counts_for_every_state(self): + """Test queue metrics include counts and zero series for every state.""" + metrics = self.gatherer._gather_metrics() + counts = { + metric["labels"]["state"]: metric["value"] + for metric in metrics + if metric["name"] == GAUGE_NAME + and metric["labels"]["channel"] == self.channel.complete_name + } + self.assertEqual( + counts, + {state: 2 if state == "pending" else 0 for state in MONITORED_STATES}, + ) diff --git a/oca_dependencies.txt b/oca_dependencies.txt index 0612c13..2206cba 100644 --- a/oca_dependencies.txt +++ b/oca_dependencies.txt @@ -1 +1,2 @@ server-env +queue