From 8a660b60813b0bc78974280d292f78113cdab8ed Mon Sep 17 00:00:00 2001 From: Alexandre Fayolle Date: Thu, 3 Sep 2026 15:03:17 +0200 Subject: [PATCH] [ADD] monitoring_prometheus_queue_job This commit features a refactoring of monitoring_prometheus to get a cron gathering some metrics from the database, and then publishing the result on the /metrics endpoint The new module monitoring_prometheus_queue_job extends the cron to get information about the queue job states. --- monitoring_prometheus/README.rst | 26 ++++++++++ monitoring_prometheus/__init__.py | 2 +- monitoring_prometheus/__manifest__.py | 4 +- .../controllers/prometheus_metrics.py | 46 +++++++++++++++-- monitoring_prometheus/data/ir_cron.xml | 14 ++++++ monitoring_prometheus/models/__init__.py | 2 + monitoring_prometheus/models/ir_http.py | 2 +- .../models/prometheus_gatherer.py | 47 ++++++++++++++++++ .../models/prometheus_metric.py | 49 +++++++++++++++++++ .../security/ir.model.access.csv | 2 + monitoring_prometheus_queue_job/README.rst | 20 ++++++++ monitoring_prometheus_queue_job/__init__.py | 1 + .../__manifest__.py | 17 +++++++ .../models/__init__.py | 1 + .../models/prometheus_gatherer.py | 40 +++++++++++++++ oca_dependencies.txt | 1 + .../addons/monitoring_prometheus_queue_job | 1 + .../monitoring_prometheus_queue_job/setup.py | 6 +++ 18 files changed, 274 insertions(+), 7 deletions(-) create mode 100644 monitoring_prometheus/data/ir_cron.xml create mode 100644 monitoring_prometheus/models/prometheus_gatherer.py create mode 100644 monitoring_prometheus/models/prometheus_metric.py create mode 100644 monitoring_prometheus/security/ir.model.access.csv create mode 100644 monitoring_prometheus_queue_job/README.rst create mode 100644 monitoring_prometheus_queue_job/__init__.py create mode 100644 monitoring_prometheus_queue_job/__manifest__.py create mode 100644 monitoring_prometheus_queue_job/models/__init__.py create mode 100644 monitoring_prometheus_queue_job/models/prometheus_gatherer.py create mode 120000 setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job create mode 100644 setup/monitoring_prometheus_queue_job/setup.py diff --git a/monitoring_prometheus/README.rst b/monitoring_prometheus/README.rst index aa98ffe..813dae2 100644 --- a/monitoring_prometheus/README.rst +++ b/monitoring_prometheus/README.rst @@ -15,3 +15,29 @@ Current available metrics are: * Longpolling request count No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo + +Gathering metrics from the database +----------------------------------- + +Metrics that require a database query are collected by the *Prometheus: gather metrics* +scheduled action, which runs every 15 minutes and stores its result as +``prometheus.metric`` records (one per metric name/labels combination). The */metrics* +endpoint reads those records and publishes them as gauges. This indirection is required +because the cron and the HTTP workers run in separate processes and cannot share the +Prometheus registry. + +By default, only users in the *Administration / Settings* group (``base.group_system``) +can read ``prometheus.metric`` records through the ORM/UI. The */metrics* endpoint +remains public and reads the records with ``sudo()`` so Prometheus can keep scraping it +without authentication. + +To publish new metrics, extend ``_gather_metrics`` on the ``prometheus.gatherer`` +abstract model and return the result of ``super()`` extended with your own entries:: + + { + "name": "odoo_some_metric", + "documentation": "What this metric measures", + "labels": {"label_name": "label_value"}, + "value": 42, + } + diff --git a/monitoring_prometheus/__init__.py b/monitoring_prometheus/__init__.py index 91c5580..f7209b1 100644 --- a/monitoring_prometheus/__init__.py +++ b/monitoring_prometheus/__init__.py @@ -1,2 +1,2 @@ -from . import controllers from . import models +from . import controllers diff --git a/monitoring_prometheus/__manifest__.py b/monitoring_prometheus/__manifest__.py index 21eca90..0656e73 100644 --- a/monitoring_prometheus/__manifest__.py +++ b/monitoring_prometheus/__manifest__.py @@ -4,13 +4,13 @@ { "name": "Monitoring: Prometheus Metrics", - "version": "14.0.1.0.0", + "version": "14.0.1.1.0", "author": "Camptocamp,Odoo Community Association (OCA)", "license": "AGPL-3", "category": "category", "depends": ["base", "web", "server_environment"], "website": "https://github.com/camptocamp/odoo-cloud-platform", - "data": [], + "data": ["security/ir.model.access.csv", "data/ir_cron.xml"], "external_dependencies": {"python": ["prometheus_client"]}, "installable": True, } diff --git a/monitoring_prometheus/controllers/prometheus_metrics.py b/monitoring_prometheus/controllers/prometheus_metrics.py index 4aa4996..e255d7c 100644 --- a/monitoring_prometheus/controllers/prometheus_metrics.py +++ b/monitoring_prometheus/controllers/prometheus_metrics.py @@ -1,12 +1,52 @@ -# Copyright 2016-2021 Camptocamp SA +# Copyright 2016 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -from prometheus_client import generate_latest +import json +import logging -from odoo.http import Controller, route +from prometheus_client import Gauge, generate_latest + +from odoo.http import Controller, request, route + +_logger = logging.getLogger(__name__) + +# Gauges must be instantiated only once per process, registering the same +# metric name twice raises a "Duplicated timeseries" error. +_GAUGES = {} + + +def _get_gauge(name, documentation, label_names): + gauge = _GAUGES.get(name) + if gauge is None: + gauge = Gauge(name, documentation, label_names) + _GAUGES[name] = gauge + return gauge class PrometheusController(Controller): + def _publish_gathered_metrics(self): + records = request.env["prometheus.metric"].sudo().search([]) + if not records: + return + + cleared = set() + for record in records: + name = record.name + labels = json.loads(record.labels or "{}") + gauge = _get_gauge(name, record.documentation or "", sorted(labels)) + if name not in cleared: + # drop the series that disappeared since the last collection + gauge.clear() + cleared.add(name) + if labels: + gauge.labels(**labels).set(record.value) + else: + gauge.set(record.value) + @route("/metrics", auth="public") def metrics(self): + try: + self._publish_gathered_metrics() + except Exception: + _logger.exception("Could not publish the gathered Prometheus metrics") return generate_latest() diff --git a/monitoring_prometheus/data/ir_cron.xml b/monitoring_prometheus/data/ir_cron.xml new file mode 100644 index 0000000..82b5aba --- /dev/null +++ b/monitoring_prometheus/data/ir_cron.xml @@ -0,0 +1,14 @@ + + + + Prometheus: gather metrics + + code + model._cron_gather_metrics() + 15 + minutes + -1 + + + + diff --git a/monitoring_prometheus/models/__init__.py b/monitoring_prometheus/models/__init__.py index 9a5eb71..472b8a2 100644 --- a/monitoring_prometheus/models/__init__.py +++ b/monitoring_prometheus/models/__init__.py @@ -1 +1,3 @@ from . import ir_http +from . import prometheus_gatherer +from . import prometheus_metric diff --git a/monitoring_prometheus/models/ir_http.py b/monitoring_prometheus/models/ir_http.py index 26375af..3c9ae2c 100644 --- a/monitoring_prometheus/models/ir_http.py +++ b/monitoring_prometheus/models/ir_http.py @@ -1,4 +1,4 @@ -# Copyright 2016-2021 Camptocamp SA +# Copyright 2016 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) from prometheus_client import Counter, Summary diff --git a/monitoring_prometheus/models/prometheus_gatherer.py b/monitoring_prometheus/models/prometheus_gatherer.py new file mode 100644 index 0000000..b5ed748 --- /dev/null +++ b/monitoring_prometheus/models/prometheus_gatherer.py @@ -0,0 +1,47 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo import api, models + + +class PrometheusGatherer(models.AbstractModel): + """Collect application metrics to be exposed on the /metrics endpoint. + + Metrics are gathered by a cron and stored in ``prometheus.metric`` + records, because crons and HTTP workers run in separate processes and + cannot share the Prometheus registry. + """ + + _name = "prometheus.gatherer" + _description = "Prometheus Metrics Gatherer" + + @api.model + def _gather_metrics(self): + """Return the metrics as a list of dicts. + + Each entry is of the form:: + + { + "name": "odoo_some_metric", + "documentation": "What this metric measures", + "labels": {"label_name": "label_value"}, + "value": 42, + } + + ``labels`` may be omitted for an unlabelled metric. All the entries + sharing a ``name`` must declare the same label names. + + Modules extending this method must return the result of ``super()`` + extended with their own entries. + """ + return [] + + @api.model + def _cron_gather_metrics(self): + Metric = self.env["prometheus.metric"].sudo() + touched = self.env["prometheus.metric"] + for metric in self._gather_metrics(): + touched |= Metric._update_metric(metric) + # drop the series that disappeared since the last collection + (Metric.search([]) - touched).unlink() + return True diff --git a/monitoring_prometheus/models/prometheus_metric.py b/monitoring_prometheus/models/prometheus_metric.py new file mode 100644 index 0000000..3f5bb5f --- /dev/null +++ b/monitoring_prometheus/models/prometheus_metric.py @@ -0,0 +1,49 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import json + +from odoo import api, fields, models + + +class PrometheusMetric(models.Model): + """Store the metrics gathered by ``prometheus.gatherer``. + + A record is stored per metric name/labels combination, because a single + metric name can be reported several times with different labels (e.g. + one gauge value per queue job state and channel). + """ + + _name = "prometheus.metric" + _description = "Prometheus Metric" + + name = fields.Char(required=True, index=True) + documentation = fields.Char() + # JSON-serialized dict, used both to publish the metric and as part of + # the uniqueness key alongside ``name``. + labels = fields.Char(default="{}") + value = fields.Float(required=True) + + @staticmethod + def _labels_key(labels): + return json.dumps(labels or {}, sort_keys=True) + + @api.model + def _update_metric(self, metric): + """Create or update the record matching ``metric``, return it.""" + labels_key = self._labels_key(metric.get("labels")) + record = self.search( + [("name", "=", metric["name"]), ("labels", "=", labels_key)], + limit=1, + ) + values = { + "name": metric["name"], + "documentation": metric.get("documentation", ""), + "labels": labels_key, + "value": metric["value"], + } + if record: + record.write(values) + else: + record = self.create(values) + return record diff --git a/monitoring_prometheus/security/ir.model.access.csv b/monitoring_prometheus/security/ir.model.access.csv new file mode 100644 index 0000000..3da8765 --- /dev/null +++ b/monitoring_prometheus/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_prometheus_metric_system,prometheus.metric system,model_prometheus_metric,base.group_system,1,1,1,1 diff --git a/monitoring_prometheus_queue_job/README.rst b/monitoring_prometheus_queue_job/README.rst new file mode 100644 index 0000000..8cac167 --- /dev/null +++ b/monitoring_prometheus_queue_job/README.rst @@ -0,0 +1,20 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License + +========================================= +Monitoring: Prometheus metrics queue jobs +========================================= + +Expose the number of queue jobs to Prometheus through the */metrics* endpoint +of ``monitoring_prometheus``. + +The gauge ``odoo_queue_job_count`` is published with the labels *state* and +*channel*, for the states ``pending``, ``enqueued``, ``started``, ``failed`` +and ``wait_dependencies``. The terminal states ``done`` and ``cancelled`` are +not reported. + +Values are refreshed by the *Prometheus: gather metrics* scheduled action, so +they are a snapshot and not a real time count. + +This module is automatically installed when both ``monitoring_prometheus`` and +``queue_job`` are installed. diff --git a/monitoring_prometheus_queue_job/__init__.py b/monitoring_prometheus_queue_job/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/monitoring_prometheus_queue_job/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/monitoring_prometheus_queue_job/__manifest__.py b/monitoring_prometheus_queue_job/__manifest__.py new file mode 100644 index 0000000..136b61e --- /dev/null +++ b/monitoring_prometheus_queue_job/__manifest__.py @@ -0,0 +1,17 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Monitoring: Prometheus Metrics for Queue Jobs", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "category", + "depends": ["monitoring_prometheus", "queue_job"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "external_dependencies": {"python": ["prometheus_client"]}, + "auto_install": True, + "installable": True, +} diff --git a/monitoring_prometheus_queue_job/models/__init__.py b/monitoring_prometheus_queue_job/models/__init__.py new file mode 100644 index 0000000..18bb3bc --- /dev/null +++ b/monitoring_prometheus_queue_job/models/__init__.py @@ -0,0 +1 @@ +from . import prometheus_gatherer diff --git a/monitoring_prometheus_queue_job/models/prometheus_gatherer.py b/monitoring_prometheus_queue_job/models/prometheus_gatherer.py new file mode 100644 index 0000000..be493f7 --- /dev/null +++ b/monitoring_prometheus_queue_job/models/prometheus_gatherer.py @@ -0,0 +1,40 @@ +# Copyright 2026 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo import api, models + +MONITORED_STATES = ("pending", "enqueued", "started", "failed", "wait_dependencies") +GAUGE_NAME = "odoo_queue_job_count" +GAUGE_DOC = "Number of queue jobs per state and channel" + + +class PrometheusGatherer(models.AbstractModel): + _inherit = "prometheus.gatherer" + + @api.model + def _gather_metrics(self): + metrics = super()._gather_metrics() + groups = self.env["queue.job"].read_group( + [("state", "in", list(MONITORED_STATES))], + ["state", "channel"], + ["state", "channel"], + lazy=False, + ) + channels = self.env["queue.job.channel"].search([]).mapped("complete_name") + # seed every combination so a series never disappears when its jobs are gone + counts = { + (state, channel): 0 for state in MONITORED_STATES for channel in channels + } + for group in groups: + key = (group["state"], group["channel"]) + counts[key] = counts.get(key, 0) + group["__count"] + metrics.extend( + { + "name": GAUGE_NAME, + "documentation": GAUGE_DOC, + "labels": {"state": state, "channel": channel}, + "value": count, + } + for (state, channel), count in counts.items() + ) + return metrics diff --git a/oca_dependencies.txt b/oca_dependencies.txt index 0612c13..2206cba 100644 --- a/oca_dependencies.txt +++ b/oca_dependencies.txt @@ -1 +1,2 @@ server-env +queue diff --git a/setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job b/setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job new file mode 120000 index 0000000..1c21819 --- /dev/null +++ b/setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job @@ -0,0 +1 @@ +../../../../monitoring_prometheus_queue_job \ No newline at end of file diff --git a/setup/monitoring_prometheus_queue_job/setup.py b/setup/monitoring_prometheus_queue_job/setup.py new file mode 100644 index 0000000..28c57bb --- /dev/null +++ b/setup/monitoring_prometheus_queue_job/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)