Merge pull request #2 from guewen/logging-json

Add logging_json

.travis.yml

@@ -1,13 +1,19 @@
 sudo: false
-cache: pip
+language: python
+
+cache:
+  apt: true
+  directories:
+    - $HOME/.cache/pip
 
 addons:
   apt:
     packages:
       - expect-dev  # provides unbuffer utility
       - python-lxml # because pip installation is slow
-
-language: python
+      - python-simplejson
+      - python-serial
+      - python-yaml
 
 python:
   - '2.7'
@@ -20,11 +26,15 @@ env:
   - LINT_CHECK="1"
   - TESTS="1" ODOO_REPO="odoo/odoo"
   - TESTS="1" ODOO_REPO="OCA/OCB"
+  global:
+  - VERSION="9.0" LINT_CHECK="0" TESTS="0"
 
 install:
   - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools
   - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH}
   - travis_install_nightly
+  - if [ "$LINT_CHECK" != "1" ] ; then ln -s ${HOME}/dependencies/server-tools/server_environment_files_sample ${HOME}/dependencies/server-tools/server_environment_files; fi
+  - printf '[options]\n\nrunning_env = dev' > ${HOME}/.openerp_serverrc
 
 services:
   - postgresql

README.md

@@ -24,6 +24,7 @@ Libraries that must be added in ``requirements.txt``:
 ```
 boto==2.42.0
 redis==2.10.5
+python-json-logger==0.1.5
 ```
 
 ### Server Environment
@@ -75,6 +76,13 @@ Besides, the
  * `ODOO_SESSION_REDIS_PREFIX`: `<client>-odoo-test`
  * `ODOO_SESSION_REDIS_EXPIRATION`: `86400` (1 day)
 
+### JSON Logging
+
+At least on production and integration, activate:
+* `ODOO_LOGGING_JSON`: 1
+* Add ``logging_json`` in the ``server_wide_modules`` option in the
+  configuration file
+
 ### Automatic Configuration
 
 Calling `ctx.env['cloud.platform'].install_exoscale()` in an

attachment_s3/__openerp__.py

@@ -6,12 +6,12 @@
 {'name': 'Attachments on S3 storage',
  'summary': 'Store assets and attachments on a S3 compatible object storage',
  'version': '9.0.1.1.0',
- 'author': 'Camptocamp',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
  'license': 'AGPL-3',
  'category': 'Knowledge Management',
  'depends': ['base'],
  'external_dependencies': {
-    'python': ['boto'],
+     'python': ['boto'],
  },
  'website': 'http://www.camptocamp.com',
  'data': [],

attachment_s3/models/ir_attachment.py

@@ -10,17 +10,20 @@ import xml.dom.minidom
 from contextlib import closing, contextmanager
 from functools import partial
 
-from distutils.util import strtobool
-
-import boto
-from boto.exception import S3ResponseError
-
 import openerp
-from openerp import _, api, exceptions, fields, models, SUPERUSER_ID
+from openerp import _, api, exceptions, models, SUPERUSER_ID
 from ..s3uri import S3Uri
 
 _logger = logging.getLogger(__name__)
 
+try:
+    import boto
+    from boto.exception import S3ResponseError
+except ImportError:
+    boto = None  # noqa
+    S3ResponseError = None  # noqa
+    _logger.debug("Cannot 'import boto'.")
+
 
 class IrAttachment(models.Model):
     _inherit = "ir.attachment"
@@ -217,7 +220,8 @@ class IrAttachment(models.Model):
                 elif attachment.db_datas:
                     _logger.info('moving on the object storage from database')
                     attachment.write({'datas': attachment.datas})
-                new_env.cr.commit()
+                # we are in a new env, this is a valid commit
+                new_env.cr.commit()  # pylint: disable=invalid-commit
 
     @contextmanager
     def do_in_new_env(self):
@@ -238,7 +242,9 @@ class IrAttachment(models.Model):
                     cr.rollback()
                     raise
                 else:
-                    cr.commit()
+                    # disable pylint error because this is a valid commit,
+                    # we are in a new env
+                    cr.commit()  # pylint: disable=invalid-commit
 
     @api.model
     def force_storage(self):

cloud_platform/__openerp__.py

@@ -6,14 +6,15 @@
 {'name': 'Cloud Platform',
  'summary': 'Addons required for the Camptocamp Cloud Platform',
  'version': '9.0.1.0.0',
- 'author': 'Camptocamp',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
  'license': 'AGPL-3',
  'category': 'Extra Tools',
  'depends': [
      'attachment_s3',
      'session_redis',
      'monitoring_status',
-     #'monitoring_log_requests',
+     'logging_json',
+     # 'monitoring_log_requests',
      'server_environment',  # OCA/server-tools
  ],
  'website': 'http://www.camptocamp.com',

cloud_platform/models/cloud_platform.py

@@ -33,7 +33,6 @@ class FilestoreKind(object):
     file = 'file'
 
 
-
 class CloudPlatform(models.AbstractModel):
     _name = 'cloud.platform'
 

cloud_platform/songs.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
 
+
 def install_exoscale(ctx):
     ctx.env['cloud.platform'].install_exoscale()

logging_json/README.rst

@@ -0,0 +1,14 @@
+JSON Logging
+============
+
+This addon allows to output the Odoo logs in JSON.
+
+Configuration
+-------------
+
+The json logging is activated with the environment variable
+``ODOO_LOGGING_JSON`` set to ``1``.
+
+In order to have the logs from the start of the server, you should add
+``logging_json`` in the ``--load`` flag or in the ``server_wide_modules``
+option in the configuration file.

logging_json/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import json_log

logging_json/__openerp__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+{'name': 'JSON Logging',
+ 'version': '9.0.1.0.0',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
+ 'license': 'AGPL-3',
+ 'category': 'Extra Tools',
+ 'depends': ['base',
+             ],
+ 'external_dependencies': {
+     'python': ['pythonjsonlogger'],
+ },
+ 'website': 'http://www.camptocamp.com',
+ 'data': [],
+ 'installable': True,
+ }

logging_json/json_log.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+import logging
+import os
+import threading
+
+from distutils.util import strtobool
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from pythonjsonlogger import jsonlogger
+except ImportError:
+    jsonlogger = None  # noqa
+    _logger.debug("Cannot 'import pythonjsonlogger'.")
+
+
+def is_true(strval):
+    return bool(strtobool(strval or '0'.lower()))
+
+
+class OdooJsonFormatter(jsonlogger.JsonFormatter):
+
+    def add_fields(self, log_record, record, message_dict):
+        record.pid = os.getpid()
+        record.dbname = getattr(threading.currentThread(), 'dbname', '?')
+        _super = super(OdooJsonFormatter, self)
+        return _super.add_fields(log_record, record, message_dict)
+
+
+if is_true(os.environ.get('ODOO_LOGGING_JSON')):
+    format = ('%(asctime)s %(pid)s %(levelname)s'
+              '%(dbname)s %(name)s: %(message)s')
+    formatter = OdooJsonFormatter(format)
+    logging.getLogger().handlers[0].formatter = formatter

monitoring_log_requests/__openerp__.py

@@ -5,7 +5,7 @@
 
 {'name': 'Monitoring: Requests Logging',
  'version': '9.0.1.0.0',
- 'author': 'Camptocamp',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
  'license': 'AGPL-3',
  'category': 'category',
  'depends': ['base', 'web'],

monitoring_status/__openerp__.py

@@ -5,7 +5,7 @@
 
 {'name': 'Monitoring: Status',
  'version': '9.0.1.0.0',
- 'author': 'Camptocamp',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
  'license': 'AGPL-3',
  'category': 'category',
  'depends': ['base', 'web'],

oca_dependencies.txt

@@ -0,0 +1 @@
+server-tools

requirements.txt

@@ -0,0 +1,3 @@
+boto==2.42.0
+redis==2.10.5
+python-json-logger==0.1.5

session_redis/__openerp__.py

@@ -6,12 +6,12 @@
 {'name': 'Sessions in Redis',
  'summary': 'Store web sessions in Redis',
  'version': '9.0.1.0.0',
- 'author': 'Camptocamp',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
  'license': 'AGPL-3',
  'category': 'Extra Tools',
  'depends': ['base'],
  'external_dependencies': {
-    'python': ['redis'],
+     'python': ['redis'],
  },
  'website': 'http://www.camptocamp.com',
  'data': [],

session_redis/http.py

@@ -7,8 +7,6 @@ import os
 
 from distutils.util import strtobool
 
-import redis
-
 import openerp
 from openerp import http
 from openerp.tools.func import lazy_property
@@ -17,6 +15,12 @@ from .session import RedisSessionStore
 
 _logger = logging.getLogger(__name__)
 
+try:
+    import redis
+except ImportError:
+    redis = None  # noqa
+    _logger.debug("Cannot 'import redis'.")
+
 
 def is_true(strval):
     return bool(strtobool(strval or '0'.lower()))