[ADD] monitoring_prometheus_queue_job

This commit features a refactoring of monitoring_prometheus to get a
cron gathering some metrics from the database, and then publishing the
result on the /metrics endpoint

The new module monitoring_prometheus_queue_job extends the cron to get
information about the queue job states.

monitoring_prometheus_queue_job/README.rst

@@ -0,0 +1,20 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :alt: License
+
+=========================================
+Monitoring: Prometheus metrics queue jobs
+=========================================
+
+Expose the number of queue jobs to Prometheus through the */metrics* endpoint
+of ``monitoring_prometheus``.
+
+The gauge ``odoo_queue_job_count`` is published with the labels *state* and
+*channel*, for the states ``pending``, ``enqueued``, ``started``, ``failed``
+and ``wait_dependencies``. The terminal states ``done`` and ``cancelled`` are
+not reported.
+
+Values are refreshed by the *Prometheus: gather metrics* scheduled action, so
+they are a snapshot and not a real time count.
+
+This module is automatically installed when both ``monitoring_prometheus`` and
+``queue_job`` are installed.

monitoring_prometheus_queue_job/__init__.py

@@ -0,0 +1 @@
+from . import models

monitoring_prometheus_queue_job/__manifest__.py

@@ -0,0 +1,17 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Monitoring: Prometheus Metrics for Queue Jobs",
+    "version": "14.0.1.0.0",
+    "author": "Camptocamp,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "category",
+    "depends": ["monitoring_prometheus", "queue_job"],
+    "website": "https://github.com/camptocamp/odoo-cloud-platform",
+    "data": [],
+    "external_dependencies": {"python": ["prometheus_client"]},
+    "auto_install": True,
+    "installable": True,
+}

monitoring_prometheus_queue_job/models/__init__.py

@@ -0,0 +1 @@
+from . import prometheus_gatherer

monitoring_prometheus_queue_job/models/prometheus_gatherer.py

@@ -0,0 +1,42 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from odoo import api, models
+
+MONITORED_STATES = ("pending", "enqueued", "started", "failed", "wait_dependencies")
+GAUGE_NAME = "odoo_queue_job_count"
+GAUGE_DOC = "Number of queue jobs per state and channel"
+
+
+class PrometheusGatherer(models.AbstractModel):
+    _inherit = "prometheus.gatherer"
+
+    @api.model
+    def _gather_metrics(self):
+        return super()._gather_metrics() + self._gather_queue_job_metrics()
+
+    @api.model
+    def _gather_queue_job_metrics(self):
+        groups = self.env["queue.job"].read_group(
+            [("state", "in", list(MONITORED_STATES))],
+            ["state", "channel"],
+            ["state", "channel"],
+            lazy=False,
+        )
+        channels = self.env["queue.job.channel"].search([]).mapped("complete_name")
+        # seed every combination so a series never disappears when its jobs are gone
+        counts = {
+            (state, channel): 0 for state in MONITORED_STATES for channel in channels
+        }
+        for group in groups:
+            key = (group["state"], group["channel"])
+            counts[key] = counts.get(key, 0) + group["__count"]
+        return [
+            {
+                "name": GAUGE_NAME,
+                "documentation": GAUGE_DOC,
+                "labels": {"state": state, "channel": channel},
+                "value": count,
+            }
+            for (state, channel), count in counts.items()
+        ]