diff --git a/monitoring_prometheus/README.rst b/monitoring_prometheus/README.rst index aa98ffe..6cb7208 100644 --- a/monitoring_prometheus/README.rst +++ b/monitoring_prometheus/README.rst @@ -15,3 +15,24 @@ Current available metrics are: * Longpolling request count No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo + +Gathering metrics from the database +----------------------------------- + +Metrics that require a database query are collected by the *Prometheus: gather metrics* +scheduled action, which runs every 15 minutes and stores its result as JSON in the +``monitoring_prometheus.metrics`` system parameter. The */metrics* endpoint reads that +parameter and publishes the entries as gauges. This indirection is required because the +cron and the HTTP workers run in separate processes and cannot share the Prometheus +registry. + +To publish new metrics, extend ``_gather_metrics`` on the ``prometheus.gatherer`` +abstract model and return the result of ``super()`` extended with your own entries:: + + { + "name": "odoo_some_metric", + "documentation": "What this metric measures", + "labels": {"label_name": "label_value"}, + "value": 42, + } + diff --git a/monitoring_prometheus/__manifest__.py b/monitoring_prometheus/__manifest__.py index 21eca90..65aea1d 100644 --- a/monitoring_prometheus/__manifest__.py +++ b/monitoring_prometheus/__manifest__.py @@ -4,13 +4,13 @@ { "name": "Monitoring: Prometheus Metrics", - "version": "14.0.1.0.0", + "version": "14.0.1.1.0", "author": "Camptocamp,Odoo Community Association (OCA)", "license": "AGPL-3", "category": "category", "depends": ["base", "web", "server_environment"], "website": "https://github.com/camptocamp/odoo-cloud-platform", - "data": [], + "data": ["data/ir_cron.xml"], "external_dependencies": {"python": ["prometheus_client"]}, "installable": True, } diff --git a/monitoring_prometheus/controllers/prometheus_metrics.py b/monitoring_prometheus/controllers/prometheus_metrics.py index 4aa4996..31e063c 100644 --- a/monitoring_prometheus/controllers/prometheus_metrics.py +++ b/monitoring_prometheus/controllers/prometheus_metrics.py @@ -1,12 +1,54 @@ # Copyright 2016-2021 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -from prometheus_client import generate_latest +import json +import logging -from odoo.http import Controller, route +from prometheus_client import Gauge, generate_latest + +from odoo.http import Controller, request, route + +from ..models.prometheus_gatherer import METRICS_PARAM + +_logger = logging.getLogger(__name__) + +# Gauges must be instantiated only once per process, registering the same +# metric name twice raises a "Duplicated timeseries" error. +_GAUGES = {} + + +def _get_gauge(name, documentation, label_names): + gauge = _GAUGES.get(name) + if gauge is None: + gauge = Gauge(name, documentation, label_names) + _GAUGES[name] = gauge + return gauge class PrometheusController(Controller): + def _publish_gathered_metrics(self): + param = request.env["ir.config_parameter"].sudo().get_param(METRICS_PARAM) + if not param: + return + + cleared = set() + for metric in json.loads(param): + name = metric["name"] + labels = metric.get("labels") or {} + gauge = _get_gauge(name, metric.get("documentation", ""), sorted(labels)) + if name not in cleared: + # drop the series that disappeared since the last collection + gauge.clear() + cleared.add(name) + if labels: + gauge.labels(**labels).set(metric["value"]) + else: + gauge.set(metric["value"]) + @route("/metrics", auth="public") def metrics(self): + try: + self._publish_gathered_metrics() + except Exception: + _logger.exception("Could not publish the gathered Prometheus metrics") return generate_latest() diff --git a/monitoring_prometheus/data/ir_cron.xml b/monitoring_prometheus/data/ir_cron.xml new file mode 100644 index 0000000..97c1945 --- /dev/null +++ b/monitoring_prometheus/data/ir_cron.xml @@ -0,0 +1,16 @@ + + + + + Prometheus: gather metrics + + code + model._cron_gather_metrics() + 15 + minutes + -1 + + + + + diff --git a/monitoring_prometheus/models/__init__.py b/monitoring_prometheus/models/__init__.py index 9a5eb71..52ebd90 100644 --- a/monitoring_prometheus/models/__init__.py +++ b/monitoring_prometheus/models/__init__.py @@ -1 +1,2 @@ from . import ir_http +from . import prometheus_gatherer diff --git a/monitoring_prometheus/models/prometheus_gatherer.py b/monitoring_prometheus/models/prometheus_gatherer.py new file mode 100644 index 0000000..b1b0d83 --- /dev/null +++ b/monitoring_prometheus/models/prometheus_gatherer.py @@ -0,0 +1,48 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import json + +from odoo import api, models + +METRICS_PARAM = "monitoring_prometheus.metrics" + + +class PrometheusGatherer(models.AbstractModel): + """Collect application metrics to be exposed on the /metrics endpoint. + + Metrics are gathered by a cron and stored as JSON in an + ``ir.config_parameter``, because crons and HTTP workers run in + separate processes and cannot share the Prometheus registry. + """ + + _name = "prometheus.gatherer" + _description = "Prometheus Metrics Gatherer" + + @api.model + def _gather_metrics(self): + """Return the metrics as a list of dicts. + + Each entry is of the form:: + + { + "name": "odoo_some_metric", + "documentation": "What this metric measures", + "labels": {"label_name": "label_value"}, + "value": 42, + } + + ``labels`` may be omitted for an unlabelled metric. All the entries + sharing a ``name`` must declare the same label names. + + Modules extending this method must return the result of ``super()`` + extended with their own entries. + """ + return [] + + @api.model + def _cron_gather_metrics(self): + self.env["ir.config_parameter"].sudo().set_param( + METRICS_PARAM, json.dumps(self._gather_metrics()) + ) + return True diff --git a/monitoring_prometheus_queue_job/README.rst b/monitoring_prometheus_queue_job/README.rst new file mode 100644 index 0000000..8cac167 --- /dev/null +++ b/monitoring_prometheus_queue_job/README.rst @@ -0,0 +1,20 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License + +========================================= +Monitoring: Prometheus metrics queue jobs +========================================= + +Expose the number of queue jobs to Prometheus through the */metrics* endpoint +of ``monitoring_prometheus``. + +The gauge ``odoo_queue_job_count`` is published with the labels *state* and +*channel*, for the states ``pending``, ``enqueued``, ``started``, ``failed`` +and ``wait_dependencies``. The terminal states ``done`` and ``cancelled`` are +not reported. + +Values are refreshed by the *Prometheus: gather metrics* scheduled action, so +they are a snapshot and not a real time count. + +This module is automatically installed when both ``monitoring_prometheus`` and +``queue_job`` are installed. diff --git a/monitoring_prometheus_queue_job/__init__.py b/monitoring_prometheus_queue_job/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/monitoring_prometheus_queue_job/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/monitoring_prometheus_queue_job/__manifest__.py b/monitoring_prometheus_queue_job/__manifest__.py new file mode 100644 index 0000000..2c51b8f --- /dev/null +++ b/monitoring_prometheus_queue_job/__manifest__.py @@ -0,0 +1,17 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Monitoring: Prometheus Metrics for Queue Jobs", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "category", + "depends": ["monitoring_prometheus", "queue_job"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "external_dependencies": {"python": ["prometheus_client"]}, + "auto_install": True, + "installable": True, +} diff --git a/monitoring_prometheus_queue_job/models/__init__.py b/monitoring_prometheus_queue_job/models/__init__.py new file mode 100644 index 0000000..18bb3bc --- /dev/null +++ b/monitoring_prometheus_queue_job/models/__init__.py @@ -0,0 +1 @@ +from . import prometheus_gatherer diff --git a/monitoring_prometheus_queue_job/models/prometheus_gatherer.py b/monitoring_prometheus_queue_job/models/prometheus_gatherer.py new file mode 100644 index 0000000..f757e5d --- /dev/null +++ b/monitoring_prometheus_queue_job/models/prometheus_gatherer.py @@ -0,0 +1,42 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from odoo import api, models + +MONITORED_STATES = ("pending", "enqueued", "started", "failed", "wait_dependencies") +GAUGE_NAME = "odoo_queue_job_count" +GAUGE_DOC = "Number of queue jobs per state and channel" + + +class PrometheusGatherer(models.AbstractModel): + _inherit = "prometheus.gatherer" + + @api.model + def _gather_metrics(self): + return super()._gather_metrics() + self._gather_queue_job_metrics() + + @api.model + def _gather_queue_job_metrics(self): + groups = self.env["queue.job"].read_group( + [("state", "in", list(MONITORED_STATES))], + ["state", "channel"], + ["state", "channel"], + lazy=False, + ) + channels = self.env["queue.job.channel"].search([]).mapped("complete_name") + # seed every combination so a series never disappears when its jobs are gone + counts = { + (state, channel): 0 for state in MONITORED_STATES for channel in channels + } + for group in groups: + key = (group["state"], group["channel"]) + counts[key] = counts.get(key, 0) + group["__count"] + return [ + { + "name": GAUGE_NAME, + "documentation": GAUGE_DOC, + "labels": {"state": state, "channel": channel}, + "value": count, + } + for (state, channel), count in counts.items() + ] diff --git a/oca_dependencies.txt b/oca_dependencies.txt index 0612c13..2206cba 100644 --- a/oca_dependencies.txt +++ b/oca_dependencies.txt @@ -1 +1,2 @@ server-env +queue diff --git a/setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job b/setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job new file mode 120000 index 0000000..1c21819 --- /dev/null +++ b/setup/monitoring_prometheus_queue_job/odoo/addons/monitoring_prometheus_queue_job @@ -0,0 +1 @@ +../../../../monitoring_prometheus_queue_job \ No newline at end of file diff --git a/setup/monitoring_prometheus_queue_job/setup.py b/setup/monitoring_prometheus_queue_job/setup.py new file mode 100644 index 0000000..28c57bb --- /dev/null +++ b/setup/monitoring_prometheus_queue_job/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)