From 46231b3365559f2eca5dc584f4ddd14a0453d466 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 2 Nov 2016 09:55:36 +0100 Subject: [PATCH 1/8] Fix pep8 --- attachment_s3/models/ir_attachment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/attachment_s3/models/ir_attachment.py b/attachment_s3/models/ir_attachment.py index 69a7de1..e39ae79 100644 --- a/attachment_s3/models/ir_attachment.py +++ b/attachment_s3/models/ir_attachment.py @@ -10,13 +10,12 @@ import xml.dom.minidom from contextlib import closing, contextmanager from functools import partial -from distutils.util import strtobool import boto from boto.exception import S3ResponseError import openerp -from openerp import _, api, exceptions, fields, models, SUPERUSER_ID +from openerp import _, api, exceptions, models, SUPERUSER_ID from ..s3uri import S3Uri _logger = logging.getLogger(__name__) From ec1f3fc61a2acfc1d274cbc1d149fad249471b3e Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 2 Nov 2016 09:55:43 +0100 Subject: [PATCH 2/8] Add logging_json Output odoo logs as json --- README.md | 8 ++++++++ cloud_platform/__openerp__.py | 1 + logging_json/README.rst | 14 ++++++++++++++ logging_json/__init__.py | 3 +++ logging_json/__openerp__.py | 18 ++++++++++++++++++ logging_json/json_log.py | 29 +++++++++++++++++++++++++++++ 6 files changed, 73 insertions(+) create mode 100644 logging_json/README.rst create mode 100644 logging_json/__init__.py create mode 100644 logging_json/__openerp__.py create mode 100644 logging_json/json_log.py diff --git a/README.md b/README.md index a379e6e..8302709 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Libraries that must be added in ``requirements.txt``: ``` boto==2.42.0 redis==2.10.5 +python-json-logger==0.1.5 ``` ### Server Environment @@ -75,6 +76,13 @@ Besides, the * `ODOO_SESSION_REDIS_PREFIX`: `-odoo-test` * `ODOO_SESSION_REDIS_EXPIRATION`: `86400` (1 day) +### JSON Logging + +At least on production and integration, activate: +* `ODOO_LOGGING_JSON`: 1 +* Add ``logging_json`` in the ``server_wide_modules`` option in the + configuration file + ### Automatic Configuration Calling `ctx.env['cloud.platform'].install_exoscale()` in an diff --git a/cloud_platform/__openerp__.py b/cloud_platform/__openerp__.py index 16fdaf1..5f22e49 100644 --- a/cloud_platform/__openerp__.py +++ b/cloud_platform/__openerp__.py @@ -13,6 +13,7 @@ 'attachment_s3', 'session_redis', 'monitoring_status', + 'logging_json', #'monitoring_log_requests', 'server_environment', # OCA/server-tools ], diff --git a/logging_json/README.rst b/logging_json/README.rst new file mode 100644 index 0000000..4d60884 --- /dev/null +++ b/logging_json/README.rst @@ -0,0 +1,14 @@ +JSON Logging +============ + +This addon allows to output the Odoo logs in JSON. + +Configuration +------------- + +The json logging is activated with the environment variable +``ODOO_LOGGING_JSON`` set to ``1``. + +In order to have the logs from the start of the server, you should add +``logging_json`` in the ``--load`` flag or in the ``server_wide_modules`` +option in the configuration file. diff --git a/logging_json/__init__.py b/logging_json/__init__.py new file mode 100644 index 0000000..a50f468 --- /dev/null +++ b/logging_json/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import json_log diff --git a/logging_json/__openerp__.py b/logging_json/__openerp__.py new file mode 100644 index 0000000..ec4bd22 --- /dev/null +++ b/logging_json/__openerp__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +{'name': 'JSON Logging', + 'version': '9.0.1.0.0', + 'author': 'Camptocamp', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': ['base', + ], + 'external_dependencies': { + 'python': ['pythonjsonlogger'], + }, + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/logging_json/json_log.py b/logging_json/json_log.py new file mode 100644 index 0000000..2f91915 --- /dev/null +++ b/logging_json/json_log.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import logging +import os +import threading + +from distutils.util import strtobool + +from pythonjsonlogger import jsonlogger + + +def is_true(strval): + return bool(strtobool(strval or '0'.lower())) + + +class OdooJsonFormatter(jsonlogger.JsonFormatter): + + def add_fields(self, log_record, record, message_dict): + record.pid = os.getpid() + record.dbname = getattr(threading.currentThread(), 'dbname', '?') + _super = super(OdooJsonFormatter, self) + return _super.add_fields(log_record, record, message_dict) + + +if is_true(os.environ.get('ODOO_LOGGING_JSON')): + format = ('%(asctime)s %(pid)s %(levelname)s' + '%(dbname)s %(name)s: %(message)s') + formatter = OdooJsonFormatter(format) + logging.getLogger().handlers[0].formatter = formatter From b5f00c26dc2091399e99549eebd61d9830933fcf Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 2 Nov 2016 10:17:39 +0100 Subject: [PATCH 3/8] Fix pep8 --- .travis.yml | 2 ++ attachment_s3/__openerp__.py | 2 +- cloud_platform/__openerp__.py | 2 +- cloud_platform/models/cloud_platform.py | 1 - cloud_platform/songs.py | 1 + logging_json/__openerp__.py | 2 +- session_redis/__openerp__.py | 2 +- 7 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index bc63b1b..195737e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,8 @@ env: - LINT_CHECK="1" - TESTS="1" ODOO_REPO="odoo/odoo" - TESTS="1" ODOO_REPO="OCA/OCB" + global: + - VERSION="9.0" install: - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools diff --git a/attachment_s3/__openerp__.py b/attachment_s3/__openerp__.py index 8456d40..253b075 100644 --- a/attachment_s3/__openerp__.py +++ b/attachment_s3/__openerp__.py @@ -11,7 +11,7 @@ 'category': 'Knowledge Management', 'depends': ['base'], 'external_dependencies': { - 'python': ['boto'], + 'python': ['boto'], }, 'website': 'http://www.camptocamp.com', 'data': [], diff --git a/cloud_platform/__openerp__.py b/cloud_platform/__openerp__.py index 5f22e49..ad9b54c 100644 --- a/cloud_platform/__openerp__.py +++ b/cloud_platform/__openerp__.py @@ -14,7 +14,7 @@ 'session_redis', 'monitoring_status', 'logging_json', - #'monitoring_log_requests', + # 'monitoring_log_requests', 'server_environment', # OCA/server-tools ], 'website': 'http://www.camptocamp.com', diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 907c216..c90ea66 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -33,7 +33,6 @@ class FilestoreKind(object): file = 'file' - class CloudPlatform(models.AbstractModel): _name = 'cloud.platform' diff --git a/cloud_platform/songs.py b/cloud_platform/songs.py index ce5736b..d8ef9b5 100644 --- a/cloud_platform/songs.py +++ b/cloud_platform/songs.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- + def install_exoscale(ctx): ctx.env['cloud.platform'].install_exoscale() diff --git a/logging_json/__openerp__.py b/logging_json/__openerp__.py index ec4bd22..31bca57 100644 --- a/logging_json/__openerp__.py +++ b/logging_json/__openerp__.py @@ -10,7 +10,7 @@ 'depends': ['base', ], 'external_dependencies': { - 'python': ['pythonjsonlogger'], + 'python': ['pythonjsonlogger'], }, 'website': 'http://www.camptocamp.com', 'data': [], diff --git a/session_redis/__openerp__.py b/session_redis/__openerp__.py index 22efc7d..418cb60 100644 --- a/session_redis/__openerp__.py +++ b/session_redis/__openerp__.py @@ -11,7 +11,7 @@ 'category': 'Extra Tools', 'depends': ['base'], 'external_dependencies': { - 'python': ['redis'], + 'python': ['redis'], }, 'website': 'http://www.camptocamp.com', 'data': [], From 7eafdd75f45a68f958fcf4d9ca864de1c847be75 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 3 Nov 2016 16:24:47 +0100 Subject: [PATCH 4/8] Add oca-dependencies --- oca_dependencies.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 oca_dependencies.txt diff --git a/oca_dependencies.txt b/oca_dependencies.txt new file mode 100644 index 0000000..9c8c917 --- /dev/null +++ b/oca_dependencies.txt @@ -0,0 +1 @@ +server-tools From 3fd26cb89495316144b2d33a14d45da30b46078e Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 3 Nov 2016 16:46:42 +0100 Subject: [PATCH 5/8] Fix travis configuration --- .travis.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 195737e..fd21a35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,19 @@ sudo: false -cache: pip +language: python + +cache: + apt: true + directories: + - $HOME/.cache/pip addons: apt: packages: - expect-dev # provides unbuffer utility - python-lxml # because pip installation is slow - -language: python + - python-simplejson + - python-serial + - python-yaml python: - '2.7' @@ -21,7 +27,7 @@ env: - TESTS="1" ODOO_REPO="odoo/odoo" - TESTS="1" ODOO_REPO="OCA/OCB" global: - - VERSION="9.0" + - VERSION="9.0" LINT_CHECK="0" TESTS="0" install: - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools From 3c8b74369bb21306dbe3d276a854e6117944bbf4 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 3 Nov 2016 16:58:36 +0100 Subject: [PATCH 6/8] Make pylint happy --- attachment_s3/__openerp__.py | 2 +- attachment_s3/models/ir_attachment.py | 19 +++++++++++++------ cloud_platform/__openerp__.py | 2 +- logging_json/__openerp__.py | 2 +- logging_json/json_log.py | 8 +++++++- monitoring_log_requests/__openerp__.py | 2 +- monitoring_status/__openerp__.py | 2 +- session_redis/__openerp__.py | 2 +- session_redis/http.py | 8 ++++++-- 9 files changed, 32 insertions(+), 15 deletions(-) diff --git a/attachment_s3/__openerp__.py b/attachment_s3/__openerp__.py index 253b075..f2c51db 100644 --- a/attachment_s3/__openerp__.py +++ b/attachment_s3/__openerp__.py @@ -6,7 +6,7 @@ {'name': 'Attachments on S3 storage', 'summary': 'Store assets and attachments on a S3 compatible object storage', 'version': '9.0.1.1.0', - 'author': 'Camptocamp', + 'author': 'Camptocamp,Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Knowledge Management', 'depends': ['base'], diff --git a/attachment_s3/models/ir_attachment.py b/attachment_s3/models/ir_attachment.py index e39ae79..9bf1c42 100644 --- a/attachment_s3/models/ir_attachment.py +++ b/attachment_s3/models/ir_attachment.py @@ -10,16 +10,20 @@ import xml.dom.minidom from contextlib import closing, contextmanager from functools import partial - -import boto -from boto.exception import S3ResponseError - import openerp from openerp import _, api, exceptions, models, SUPERUSER_ID from ..s3uri import S3Uri _logger = logging.getLogger(__name__) +try: + import boto + from boto.exception import S3ResponseError +except ImportError: + boto = None # noqa + S3ResponseError = None # noqa + _logger.debug("Cannot 'import boto'.") + class IrAttachment(models.Model): _inherit = "ir.attachment" @@ -216,7 +220,8 @@ class IrAttachment(models.Model): elif attachment.db_datas: _logger.info('moving on the object storage from database') attachment.write({'datas': attachment.datas}) - new_env.cr.commit() + # we are in a new env, this is a valid commit + new_env.cr.commit() # pylint: disable=invalid-commit @contextmanager def do_in_new_env(self): @@ -237,7 +242,9 @@ class IrAttachment(models.Model): cr.rollback() raise else: - cr.commit() + # disable pylint error because this is a valid commit, + # we are in a new env + cr.commit() # pylint: disable=invalid-commit @api.model def force_storage(self): diff --git a/cloud_platform/__openerp__.py b/cloud_platform/__openerp__.py index ad9b54c..14400f1 100644 --- a/cloud_platform/__openerp__.py +++ b/cloud_platform/__openerp__.py @@ -6,7 +6,7 @@ {'name': 'Cloud Platform', 'summary': 'Addons required for the Camptocamp Cloud Platform', 'version': '9.0.1.0.0', - 'author': 'Camptocamp', + 'author': 'Camptocamp,Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Extra Tools', 'depends': [ diff --git a/logging_json/__openerp__.py b/logging_json/__openerp__.py index 31bca57..e73ded2 100644 --- a/logging_json/__openerp__.py +++ b/logging_json/__openerp__.py @@ -4,7 +4,7 @@ {'name': 'JSON Logging', 'version': '9.0.1.0.0', - 'author': 'Camptocamp', + 'author': 'Camptocamp,Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Extra Tools', 'depends': ['base', diff --git a/logging_json/json_log.py b/logging_json/json_log.py index 2f91915..3fcc796 100644 --- a/logging_json/json_log.py +++ b/logging_json/json_log.py @@ -6,7 +6,13 @@ import threading from distutils.util import strtobool -from pythonjsonlogger import jsonlogger +_logger = logging.getLogger(__name__) + +try: + from pythonjsonlogger import jsonlogger +except ImportError: + jsonlogger = None # noqa + _logger.debug("Cannot 'import pythonjsonlogger'.") def is_true(strval): diff --git a/monitoring_log_requests/__openerp__.py b/monitoring_log_requests/__openerp__.py index c09fba2..49e73e7 100644 --- a/monitoring_log_requests/__openerp__.py +++ b/monitoring_log_requests/__openerp__.py @@ -5,7 +5,7 @@ {'name': 'Monitoring: Requests Logging', 'version': '9.0.1.0.0', - 'author': 'Camptocamp', + 'author': 'Camptocamp,Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'category', 'depends': ['base', 'web'], diff --git a/monitoring_status/__openerp__.py b/monitoring_status/__openerp__.py index 0e024c2..f64afb9 100644 --- a/monitoring_status/__openerp__.py +++ b/monitoring_status/__openerp__.py @@ -5,7 +5,7 @@ {'name': 'Monitoring: Status', 'version': '9.0.1.0.0', - 'author': 'Camptocamp', + 'author': 'Camptocamp,Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'category', 'depends': ['base', 'web'], diff --git a/session_redis/__openerp__.py b/session_redis/__openerp__.py index 418cb60..3701910 100644 --- a/session_redis/__openerp__.py +++ b/session_redis/__openerp__.py @@ -6,7 +6,7 @@ {'name': 'Sessions in Redis', 'summary': 'Store web sessions in Redis', 'version': '9.0.1.0.0', - 'author': 'Camptocamp', + 'author': 'Camptocamp,Odoo Community Association (OCA)', 'license': 'AGPL-3', 'category': 'Extra Tools', 'depends': ['base'], diff --git a/session_redis/http.py b/session_redis/http.py index 4c814e8..a9c5ec0 100644 --- a/session_redis/http.py +++ b/session_redis/http.py @@ -7,8 +7,6 @@ import os from distutils.util import strtobool -import redis - import openerp from openerp import http from openerp.tools.func import lazy_property @@ -17,6 +15,12 @@ from .session import RedisSessionStore _logger = logging.getLogger(__name__) +try: + import redis +except ImportError: + redis = None # noqa + _logger.debug("Cannot 'import redis'.") + def is_true(strval): return bool(strtobool(strval or '0'.lower())) From e5c1afb09e41d74f424372ce03acc4132ceb4894 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 3 Nov 2016 16:59:58 +0100 Subject: [PATCH 7/8] Fix server_environment_files error --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index fd21a35..2655a84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,8 @@ install: - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH} - travis_install_nightly + - if [ "$LINT_CHECK" != "1" ] ; then ln -s ${HOME}/dependencies/server-tools/server_environment_files_sample ${HOME}/dependencies/server-tools/server_environment_files; fi + - printf '[options]\n\nrunning_env = dev' > ${HOME}/.openerp_serverrc services: - postgresql From bed41d671972aefb45d46d6171ee744bec8cecf1 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 3 Nov 2016 17:37:00 +0100 Subject: [PATCH 8/8] Add requirements.txt --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3d3be20 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +boto==2.42.0 +redis==2.10.5 +python-json-logger==0.1.5