Merge pull request #3 from guewen/grafana-statsd

Add statsd addon

README.md

@@ -6,14 +6,13 @@ Camptocamp odoo addons used on our Cloud Platform.
 
 On the platform we want to achieve having:
 
-* no data stored on the local filesystem so we can move an instance
+* No data stored on the local filesystem so we can move an instance
   between hosts and even have several running front-ends
-* metrics read from the logs or sent to Prometheus to monitor the instances
+* Metrics read from the logs or sent to Prometheus to monitor the instances
+* Logs sent to ElasticSearch-Kibana structured as JSON for better searching
 
-For the storage, we store all the attachments on a object storage such as S3 or a S3 compatible one, and we store the werkzeug sessions on Redis.
-
-For the metrics, we produce logs as json (TODO) that is sent
-to ELK and send data to Prometheus/Grafana using statsd.
+For the storage, we store all the attachments on a object storage such as S3 or
+a S3 compatible one, and we store the werkzeug sessions on Redis.
 
 ## Setup
 
@@ -25,6 +24,7 @@ Libraries that must be added in ``requirements.txt``:
 boto==2.42.0
 redis==2.10.5
 python-json-logger==0.1.5
+statsd==3.2.1
 ```
 
 ### Server Environment
@@ -46,16 +46,16 @@ The exact naming is important, because the `cloud_platform` addon rely on these
  * `AWS_ACCESS_KEY_ID`: depends of the platform
  * `AWS_SECRET_ACCESS_KEY`: depends of the platform
  * `AWS_BUCKETNAME`: `<client>-odoo-prod`
-* integration: stored RO in the production object storage, fallback
-  on database for the writes
+* integration:
  * `AWS_HOST`: depends of the platform
  * `AWS_ACCESS_KEY_ID`: depends of the platform
  * `AWS_SECRET_ACCESS_KEY`: depends of the platform
  * `AWS_BUCKETNAME`: `<client>-odoo-integration`
 * test: attachments are stored in database
 
-Besides, the 
- * `ir.config_parameter` `ir_attachment.location`: `s3://` (
+Besides, the attachment location should be set to `s3` (but this is
+automatically done by the `install` methods of the `cloud_platform` module.
+ * `ir.config_parameter` `ir_attachment.location`: `s3`
 
 ### Sessions in Redis
 
@@ -83,6 +83,17 @@ At least on production and integration, activate:
 * Add ``logging_json`` in the ``server_wide_modules`` option in the
   configuration file
 
+### Metrics (Statsd/Prometheus for Grafana)
+
+Should be active at least on the production server
+
+* `ODOO_STATSD`: 1
+* `STATSD_CUSTOMER`: `<client>`
+* `STATSD_ENVIRONMENT`: set if you want to send metrics for a special
+  environment which does not match with the `server_environment`
+* `STATSD_HOST`: depends of the platform
+* `STATSD_PORT`: depends of the platform
+
 ### Automatic Configuration
 
 Calling `ctx.env['cloud.platform'].install_exoscale()` in an

cloud_platform/__openerp__.py

@@ -15,6 +15,7 @@
      'monitoring_status',
      'logging_json',
      # 'monitoring_log_requests',
+     'monitoring_statsd',
      'server_environment',  # OCA/server-tools
  ],
  'website': 'http://www.camptocamp.com',

cloud_platform/models/cloud_platform.py

@@ -99,6 +99,11 @@ class CloudPlatform(models.AbstractModel):
                 ", we got: '%s'" % (prefix,)
             )
 
+    @api.model
+    def _check_metrics(self, environment_name):
+        if environment_name == 'prod':
+            assert is_true(os.environ.get('ODOO_STATSD'))
+
     @api.model
     def check(self):
         if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')):
@@ -117,6 +122,7 @@ class CloudPlatform(models.AbstractModel):
         environment_name = config['running_env']
         self._check_s3(environment_name)
         self._check_redis(environment_name)
+        self._check_metrics(environment_name)
 
     @api.cr
     def _register_hook(self, cr):

monitoring_statsd/README.rst

@@ -0,0 +1,31 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :alt: License
+
+============================
+Monitoring: Statsd metrics
+============================
+
+Send metrics to a Statsd (or Statsd compatible) server.
+
+Currently, it sends:
+ * time taken to process a click on a button
+ * time taken to process a workflow signal
+ * time taken by other requests
+
+Configuration
+=============
+
+Activate with the environment variable:
+
+* ``ODOO_STATSD=1``
+
+Configure Statsd:
+
+* ``STATSD_HOST=host``
+* ``STATSD_PORT=port``
+
+Configure differentiator:
+
+* ``STATSD_CUSTOMER`` must contain the name of the client
+* ``STATSD_ENVIRONMENT`` might contain the name of the environment, by
+  default, it will use the ``server_environment``'s one

monitoring_statsd/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import models

monitoring_statsd/__openerp__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{'name': 'Monitoring: Statsd Metrics',
+ 'version': '9.0.1.0.0',
+ 'author': 'Camptocamp',
+ 'license': 'AGPL-3',
+ 'category': 'category',
+ 'depends': ['base',
+             'web',
+             'server_environment',
+             ],
+ 'website': 'http://www.camptocamp.com',
+ 'data': [],
+ 'external_dependencies': {
+     'python': ['statsd'],
+ },
+ 'installable': True,
+ }

monitoring_statsd/models/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import ir_http

monitoring_statsd/models/ir_http.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from openerp import models
+from openerp.http import request
+
+from ..statsd_client import statsd, customer, environment
+
+
+class IrHttp(models.AbstractModel):
+    _inherit = 'ir.http'
+
+    def _dispatch(self):
+        if not statsd:
+            return super(IrHttp, self)._dispatch()
+
+        path_info = request.httprequest.environ.get('PATH_INFO')
+        if path_info.startswith('/longpolling/'):
+            return super(IrHttp, self)._dispatch()
+
+        parts = ['http', ]
+        if path_info.startswith('/web/dataset/call_button'):
+            parts += ['button',
+                      customer, environment,
+                      request.params['model'].replace('.', '_'),
+                      request.params['method'],
+                      ]
+        elif path_info.startswith('/web/dataset/exec_workflow'):
+            parts += ['workflow',
+                      customer, environment,
+                      request.params['model'].replace('.', '_'),
+                      request.params['signal'],
+                      ]
+        else:
+            parts += ['request',
+                      customer, environment,
+                      ]
+
+        with statsd.timer('.'.join(parts)):
+            return super(IrHttp, self)._dispatch()

monitoring_statsd/statsd_client.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+import logging
+import os
+
+from distutils.util import strtobool
+
+from openerp.tools.config import config
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from statsd import defaults
+    from statsd.client import StatsClient
+except ImportError:
+    _logger.warning('statds must be installed')
+    defaults = None  # noqa
+    StatsClient = None  # noqa
+
+
+def is_true(strval):
+    return bool(strtobool(strval or '0'.lower()))
+
+
+statsd_active = is_true(os.environ.get('ODOO_STATSD'))
+
+statsd = None
+customer = None
+environment = None
+if statsd_active and statsd is None and StatsClient is not None:
+    if not os.environ.get('STATSD_CUSTOMER'):
+        raise Exception(
+            'STATSD_CUSTOMER must contain the name of the customer'
+        )
+    if os.environ.get('STATSD_ENVIRONMENT'):
+        environment = os.environ['STATSD_ENVIRONMENT']
+    elif config.get('running_env'):
+        environment = config['running_env']
+    else:
+        raise Exception(
+            'Either STATSD_ENVIRONMENT or configuration option running_env '
+            'must contain the environment (prod, integration, ...)'
+        )
+
+    host = os.getenv('STATSD_HOST', defaults.HOST)
+    port = int(os.getenv('STATSD_PORT', defaults.PORT))
+    prefix = os.getenv('STATSD_PREFIX', defaults.PREFIX)
+    maxudpsize = int(os.getenv('STATSD_MAXUDPSIZE', defaults.MAXUDPSIZE))
+    ipv6 = bool(int(os.getenv('STATSD_IPV6', defaults.IPV6)))
+    statsd = StatsClient(host=host, port=port, prefix='odoo',
+                         maxudpsize=maxudpsize, ipv6=ipv6)

requirements.txt

@@ -1,3 +1,4 @@
 boto==2.42.0
 redis==2.10.5
 python-json-logger==0.1.5
+statsd==3.2.1