Add attachment_s3

attachment_s3/README.rst

@@ -0,0 +1,41 @@
+Attachments on S3 storage
+=========================
+
+This addon allows to store the attachments (documents and assets) on S3 or any
+other S3-compatible Object Storage.
+
+Configuration
+-------------
+
+Activate S3 storage:
+
+* Create or set the system parameter with the key ``ir_attachment.location``
+  and the value in the form ``s3``.
+
+Configure accesses with environment variables:
+
+* ``AWS_HOST`` (not required if using AWS services)
+* ``AWS_ACCESS_KEY_ID``
+* ``AWS_SECRET_ACCESS_KEY``
+* ``AWS_BUCKETNAME``
+
+Read-only mode:
+
+The bucket and the file key are stored in the attachment. So if you change the
+``AWS_BUCKETNAME`` or the ``ir_attachment.location``, the existing attachments
+will still be read on their former bucket. But as soon as they are written over
+or new attachments are created, they will be created on the new bucket or on
+the other location (db or filesystem). This is a convenient way to be able to
+read the production attachments on a replication (since you have the
+credentials) without any risk to alter the production data.
+
+This addon must be added in the server wide addons with (``--load`` option):
+
+``--load=web,web_kanban,attachment_s3``
+
+Limitations
+-----------
+
+* You need to call ``env['ir.attachment'].force_storage()`` after
+  having changed the ``ir_attachment.location`` configuration in order to
+  migrate the existing attachments to S3.

attachment_s3/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import models

attachment_s3/__openerp__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{'name': 'Attachments on S3 storage',
+ 'summary': 'Store assets and attachments on a S3 compatible object storage',
+ 'version': '8.0.1.0.0',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
+ 'license': 'AGPL-3',
+ 'category': 'Knowledge Management',
+ 'depends': ['base_attachment_object_storage'],
+ 'external_dependencies': {
+     'python': ['boto'],
+ },
+ 'website': 'http://www.camptocamp.com',
+ 'data': [],
+ 'installable': True,
+ }

attachment_s3/models/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+from . import ir_attachment

attachment_s3/models/ir_attachment.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+import base64
+import logging
+import os
+import xml.dom.minidom
+from functools import partial
+
+from openerp import _
+from openerp.osv import osv
+from openerp.osv.orm import except_orm
+from ..s3uri import S3Uri
+
+_logger = logging.getLogger(__name__)
+
+try:
+    import boto
+    from boto.exception import S3ResponseError
+except ImportError:
+    boto = None  # noqa
+    S3ResponseError = None  # noqa
+    _logger.debug("Cannot 'import boto'.")
+
+
+class IrAttachment(osv.osv):
+    _inherit = "ir.attachment"
+
+    def _get_stores(self):
+        return ['s3'] + super(IrAttachment, self)._get_stores()
+
+    def _get_s3_bucket(self, name=None):
+        """Connect to S3 and return the bucket
+
+        The following environment variables can be set:
+        * ``AWS_HOST``
+        * ``AWS_ACCESS_KEY_ID``
+        * ``AWS_SECRET_ACCESS_KEY``
+        * ``AWS_BUCKETNAME``
+
+        If a name is provided, we'll read this bucket, otherwise, the bucket
+        from the environment variable ``AWS_BUCKETNAME`` will be read.
+
+        """
+        host = os.environ.get('AWS_HOST')
+        if host:
+            connect_s3 = partial(boto.connect_s3, host=host)
+        else:
+            connect_s3 = boto.connect_s3
+
+        access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+        secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+        if name:
+            bucket_name = name
+        else:
+            bucket_name = os.environ.get('AWS_BUCKETNAME')
+        if not (access_key and secret_key and bucket_name):
+            msg = _('If you want to read from the %s S3 bucket, the following '
+                    'environment variables must be set:\n'
+                    '* AWS_ACCESS_KEY_ID\n'
+                    '* AWS_SECRET_ACCESS_KEY\n'
+                    'If you want to write in the %s S3 bucket, this variable '
+                    'must be set as well:\n'
+                    '* AWS_BUCKETNAME\n'
+                    'Optionally, the S3 host can be changed with:\n'
+                    '* AWS_HOST\n'
+                    ) % (bucket_name, bucket_name)
+
+            raise except_orm(_('Configuration Error'), msg)
+
+        try:
+            conn = connect_s3(aws_access_key_id=access_key,
+                              aws_secret_access_key=secret_key)
+
+        except S3ResponseError as error:
+            # log verbose error from s3, return short message for user
+            _logger.exception('Error during connection on S3')
+            raise except_orm(_('S3 Error'), self._parse_s3_error(error))
+
+        bucket = conn.lookup(bucket_name)
+        if not bucket:
+            bucket = conn.create_bucket(bucket_name)
+        return bucket
+
+    @staticmethod
+    def _parse_s3_error(s3error):
+        msg = s3error.reason
+        # S3 error message is a XML message...
+        doc = xml.dom.minidom.parseString(s3error.body)
+        msg_node = doc.getElementsByTagName('Message')
+        if msg_node:
+            msg = '%s: %s' % (msg, msg_node[0].childNodes[0].data)
+        return msg
+
+    def _store_file_read(self, fname, bin_size=False):
+        if fname.startswith('s3://'):
+            s3uri = S3Uri(fname)
+            try:
+                bucket = self._get_s3_bucket(name=s3uri.bucket())
+            except except_orm:
+                _logger.exception(
+                    "error reading attachment '%s' from object storage", fname
+                )
+                return ''
+            filekey = bucket.get_key(s3uri.item())
+            if filekey:
+                read = base64.b64encode(filekey.get_contents_as_string())
+            else:
+                read = ''
+                _logger.info(
+                    "attachment '%s' missing on object storage", fname
+                )
+            return read
+        else:
+            return super(IrAttachment, self)._store_file_read(fname, bin_size)
+
+    def _store_file_write(self, storage, key, bin_data):
+        if storage == 's3':
+            bucket = self._get_s3_bucket()
+            filekey = bucket.get_key(key) or bucket.new_key(key)
+            filename = 's3://%s/%s' % (bucket.name, key)
+            try:
+                filekey.set_contents_from_string(bin_data)
+            except S3ResponseError as error:
+                # log verbose error from s3, return short message for user
+                _logger.exception(
+                    'Error during storage of the file %s' % filename
+                )
+                raise except_orm(
+                    _('S3 Error'),
+                    _('The file could not be stored: %s') %
+                    (self._parse_s3_error(error),)
+                )
+        else:
+            _super = super(IrAttachment, self)
+            filename = _super._store_file_write(key, bin_data)
+        return filename
+
+    def _store_file_delete(self, fname):
+        if fname.startswith('s3://'):
+            s3uri = S3Uri(fname)
+            bucket_name = s3uri.bucket()
+            item_name = s3uri.item()
+            # delete the file only if it is on the current configured bucket
+            # otherwise, we might delete files used on a different environment
+            if bucket_name == os.environ.get('AWS_BUCKETNAME'):
+                bucket = self._get_s3_bucket()
+                filekey = bucket.get_key(item_name)
+                if filekey:
+                    try:
+                        filekey.delete()
+                        _logger.info(
+                            'file %s deleted on the object storage' % (fname,)
+                        )
+                    except S3ResponseError:
+                        # log verbose error from s3, return short message for
+                        # user
+                        _logger.exception(
+                            'Error during deletion of the file %s' % fname
+                        )
+        else:
+            super(IrAttachment, self)._file_delete_from_store(fname)

attachment_s3/s3uri.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+
+
+class S3Uri(object):
+
+    _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE)
+
+    def __init__(self, uri):
+        match = self._url_re.match(uri)
+        if not match:
+            raise ValueError("%s: is not a valid S3 URI" % (uri,))
+        self._bucket, self._item = match.groups()
+
+    def bucket(self):
+        return self._bucket
+
+    def item(self):
+        return self._item

requirements.txt

@@ -1,2 +1,3 @@
+boto==2.42.0
 redis==2.10.5
 python-json-logger==0.1.5