[ADD] monitoring_prometheus_queue_job

This commit features a refactoring of monitoring_prometheus to get a
cron gathering some metrics from the database, and then publishing the
result on the /metrics endpoint

The new module monitoring_prometheus_queue_job extends the cron to get
information about the queue job states.

monitoring_prometheus/README.rst

@@ -15,3 +15,24 @@ Current available metrics are:
 * Longpolling request count
 
 No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo
+
+Gathering metrics from the database
+-----------------------------------
+
+Metrics that require a database query are collected by the *Prometheus: gather metrics*
+scheduled action, which runs every 15 minutes and stores its result as JSON in the
+``monitoring_prometheus.metrics`` system parameter. The */metrics* endpoint reads that
+parameter and publishes the entries as gauges. This indirection is required because the
+cron and the HTTP workers run in separate processes and cannot share the Prometheus
+registry.
+
+To publish new metrics, extend ``_gather_metrics`` on the ``prometheus.gatherer``
+abstract model and return the result of ``super()`` extended with your own entries::
+
+    {
+        "name": "odoo_some_metric",
+        "documentation": "What this metric measures",
+        "labels": {"label_name": "label_value"},
+        "value": 42,
+    }
+

monitoring_prometheus/__manifest__.py

@@ -4,13 +4,13 @@
 
 {
     "name": "Monitoring: Prometheus Metrics",
-    "version": "14.0.1.0.0",
+    "version": "14.0.1.1.0",
     "author": "Camptocamp,Odoo Community Association (OCA)",
     "license": "AGPL-3",
     "category": "category",
     "depends": ["base", "web", "server_environment"],
     "website": "https://github.com/camptocamp/odoo-cloud-platform",
-    "data": [],
+    "data": ["data/ir_cron.xml"],
     "external_dependencies": {"python": ["prometheus_client"]},
     "installable": True,
 }

monitoring_prometheus/controllers/prometheus_metrics.py

@@ -1,12 +1,54 @@
 # Copyright 2016-2021 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
-from prometheus_client import generate_latest
+import json
+import logging
 
-from odoo.http import Controller, route
+from prometheus_client import Gauge, generate_latest
+
+from odoo.http import Controller, request, route
+
+from ..models.prometheus_gatherer import METRICS_PARAM
+
+_logger = logging.getLogger(__name__)
+
+# Gauges must be instantiated only once per process, registering the same
+# metric name twice raises a "Duplicated timeseries" error.
+_GAUGES = {}
+
+
+def _get_gauge(name, documentation, label_names):
+    gauge = _GAUGES.get(name)
+    if gauge is None:
+        gauge = Gauge(name, documentation, label_names)
+        _GAUGES[name] = gauge
+    return gauge
 
 
 class PrometheusController(Controller):
+    def _publish_gathered_metrics(self):
+        param = request.env["ir.config_parameter"].sudo().get_param(METRICS_PARAM)
+        if not param:
+            return
+
+        cleared = set()
+        for metric in json.loads(param):
+            name = metric["name"]
+            labels = metric.get("labels") or {}
+            gauge = _get_gauge(name, metric.get("documentation", ""), sorted(labels))
+            if name not in cleared:
+                # drop the series that disappeared since the last collection
+                gauge.clear()
+                cleared.add(name)
+            if labels:
+                gauge.labels(**labels).set(metric["value"])
+            else:
+                gauge.set(metric["value"])
+
     @route("/metrics", auth="public")
     def metrics(self):
+        try:
+            self._publish_gathered_metrics()
+        except Exception:
+            _logger.exception("Could not publish the gathered Prometheus metrics")
         return generate_latest()

monitoring_prometheus/data/ir_cron.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<odoo>
+    <data noupdate="1">
+        <record id="cron_prometheus_gather_metrics" model="ir.cron">
+            <field name="name">Prometheus: gather metrics</field>
+            <field name="model_id" ref="model_prometheus_gatherer" />
+            <field name="state">code</field>
+            <field name="code">model._cron_gather_metrics()</field>
+            <field name="interval_number">15</field>
+            <field name="interval_type">minutes</field>
+            <field name="numbercall">-1</field>
+            <field name="doall" eval="False" />
+            <field name="active" eval="True" />
+        </record>
+    </data>
+</odoo>

monitoring_prometheus/models/__init__.py

@@ -1 +1,2 @@
 from . import ir_http
+from . import prometheus_gatherer

monitoring_prometheus/models/prometheus_gatherer.py

@@ -0,0 +1,48 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import json
+
+from odoo import api, models
+
+METRICS_PARAM = "monitoring_prometheus.metrics"
+
+
+class PrometheusGatherer(models.AbstractModel):
+    """Collect application metrics to be exposed on the /metrics endpoint.
+
+    Metrics are gathered by a cron and stored as JSON in an
+    ``ir.config_parameter``, because crons and HTTP workers run in
+    separate processes and cannot share the Prometheus registry.
+    """
+
+    _name = "prometheus.gatherer"
+    _description = "Prometheus Metrics Gatherer"
+
+    @api.model
+    def _gather_metrics(self):
+        """Return the metrics as a list of dicts.
+
+        Each entry is of the form::
+
+            {
+                "name": "odoo_some_metric",
+                "documentation": "What this metric measures",
+                "labels": {"label_name": "label_value"},
+                "value": 42,
+            }
+
+        ``labels`` may be omitted for an unlabelled metric. All the entries
+        sharing a ``name`` must declare the same label names.
+
+        Modules extending this method must return the result of ``super()``
+        extended with their own entries.
+        """
+        return []
+
+    @api.model
+    def _cron_gather_metrics(self):
+        self.env["ir.config_parameter"].sudo().set_param(
+            METRICS_PARAM, json.dumps(self._gather_metrics())
+        )
+        return True

monitoring_prometheus_queue_job/README.rst

@@ -0,0 +1,20 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :alt: License
+
+=========================================
+Monitoring: Prometheus metrics queue jobs
+=========================================
+
+Expose the number of queue jobs to Prometheus through the */metrics* endpoint
+of ``monitoring_prometheus``.
+
+The gauge ``odoo_queue_job_count`` is published with the labels *state* and
+*channel*, for the states ``pending``, ``enqueued``, ``started``, ``failed``
+and ``wait_dependencies``. The terminal states ``done`` and ``cancelled`` are
+not reported.
+
+Values are refreshed by the *Prometheus: gather metrics* scheduled action, so
+they are a snapshot and not a real time count.
+
+This module is automatically installed when both ``monitoring_prometheus`` and
+``queue_job`` are installed.

monitoring_prometheus_queue_job/__init__.py

@@ -0,0 +1 @@
+from . import models

monitoring_prometheus_queue_job/__manifest__.py

@@ -0,0 +1,17 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Monitoring: Prometheus Metrics for Queue Jobs",
+    "version": "14.0.1.0.0",
+    "author": "Camptocamp,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "category",
+    "depends": ["monitoring_prometheus", "queue_job"],
+    "website": "https://github.com/camptocamp/odoo-cloud-platform",
+    "data": [],
+    "external_dependencies": {"python": ["prometheus_client"]},
+    "auto_install": True,
+    "installable": True,
+}

monitoring_prometheus_queue_job/models/__init__.py

@@ -0,0 +1 @@
+from . import prometheus_gatherer

monitoring_prometheus_queue_job/models/prometheus_gatherer.py

@@ -0,0 +1,42 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from odoo import api, models
+
+MONITORED_STATES = ("pending", "enqueued", "started", "failed", "wait_dependencies")
+GAUGE_NAME = "odoo_queue_job_count"
+GAUGE_DOC = "Number of queue jobs per state and channel"
+
+
+class PrometheusGatherer(models.AbstractModel):
+    _inherit = "prometheus.gatherer"
+
+    @api.model
+    def _gather_metrics(self):
+        return super()._gather_metrics() + self._gather_queue_job_metrics()
+
+    @api.model
+    def _gather_queue_job_metrics(self):
+        groups = self.env["queue.job"].read_group(
+            [("state", "in", list(MONITORED_STATES))],
+            ["state", "channel"],
+            ["state", "channel"],
+            lazy=False,
+        )
+        channels = self.env["queue.job.channel"].search([]).mapped("complete_name")
+        # seed every combination so a series never disappears when its jobs are gone
+        counts = {
+            (state, channel): 0 for state in MONITORED_STATES for channel in channels
+        }
+        for group in groups:
+            key = (group["state"], group["channel"])
+            counts[key] = counts.get(key, 0) + group["__count"]
+        return [
+            {
+                "name": GAUGE_NAME,
+                "documentation": GAUGE_DOC,
+                "labels": {"state": state, "channel": channel},
+                "value": count,
+            }
+            for (state, channel), count in counts.items()
+        ]

oca_dependencies.txt

@@ -1 +1,2 @@
 server-env
+queue

setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job

@@ -0,0 +1 @@
+../../../../monitoring_prometheus_queue_job

setup/monitoring_prometheus_queue_job/setup.py

@@ -0,0 +1,6 @@
+import setuptools
+
+setuptools.setup(
+    setup_requires=['setuptools-odoo'],
+    odoo_addon=True,
+)