Merge pull request #172 from p-tombez/9.0-s3_boto3

[9.0] attachment_s3: Migrate to boto3

attachment_s3/__openerp__.py

@@ -11,7 +11,7 @@
  'category': 'Knowledge Management',
  'depends': ['base'],
  'external_dependencies': {
-     'python': ['boto'],
+     'python': ['boto3'],
  },
  'website': 'http://www.camptocamp.com',
  'data': [

attachment_s3/models/ir_attachment.py

@@ -7,9 +7,8 @@ import base64
 import inspect
 import logging
 import os
-import xml.dom.minidom
 from contextlib import closing, contextmanager
-from functools import partial
+import io
 
 import psycopg2
 
@@ -20,12 +19,13 @@ from ..s3uri import S3Uri
 _logger = logging.getLogger(__name__)
 
 try:
-    import boto
+    import boto3
-    from boto.exception import S3ResponseError
+    from botocore.exceptions import ClientError, EndpointConnectionError
 except ImportError:
-    boto = None  # noqa
+    boto3 = None  # noqa
-    S3ResponseError = None  # noqa
+    ClientError = None  # noqa
-    _logger.debug("Cannot 'import boto'.")
+    EndpointConnectionError = None  # noqa
+    _logger.debug("Cannot 'import boto3'.")
 
 
 def clean_fs(files):
@@ -159,6 +159,7 @@ class IrAttachment(models.Model):
 
         The following environment variables can be set:
         * ``AWS_HOST``
+        * ``AWS_REGION``
         * ``AWS_ACCESS_KEY_ID``
         * ``AWS_SECRET_ACCESS_KEY``
         * ``AWS_BUCKETNAME``
@@ -168,17 +169,20 @@ class IrAttachment(models.Model):
 
         """
         host = os.environ.get('AWS_HOST')
-        if host:
+        region_name = os.environ.get('AWS_REGION')
-            connect_s3 = partial(boto.connect_s3, host=host)
-        else:
-            connect_s3 = boto.connect_s3
-
         access_key = os.environ.get('AWS_ACCESS_KEY_ID')
         secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
-        if name:
+        bucket_name = name or os.environ.get('AWS_BUCKETNAME')
-            bucket_name = name
+
-        else:
+        params = {
-            bucket_name = os.environ.get('AWS_BUCKETNAME')
+            'aws_access_key_id': access_key,
+            'aws_secret_access_key': secret_key,
+        }
+
+        if host:
+            params['endpoint_url'] = host
+        if region_name:
+            params['region_name'] = region_name
         if not (access_key and secret_key and bucket_name):
             msg = _('If you want to read from the %s S3 bucket, the following '
                     'environment variables must be set:\n'
@@ -192,31 +196,34 @@ class IrAttachment(models.Model):
                     ) % (bucket_name, bucket_name)
 
             raise exceptions.UserError(msg)
-
+        # try:
+        s3 = boto3.resource('s3', **params)
+        bucket = s3.Bucket(bucket_name)
+        exists = True
         try:
-            conn = connect_s3(aws_access_key_id=access_key,
+            s3.meta.client.head_bucket(Bucket=bucket_name)
-                              aws_secret_access_key=secret_key)
+        except ClientError as e:
-
+            # If a client error is thrown, then check that it was a 404 error.
-        except S3ResponseError as error:
+            # If it was a 404 error, then the bucket does not exist.
+            error_code = e.response['Error']['Code']
+            if error_code == '404':
+                exists = False
+        except EndpointConnectionError as error:
             # log verbose error from s3, return short message for user
             _logger.exception('Error during connection on S3')
-            raise exceptions.UserError(self._parse_s3_error(error))
+            raise exceptions.UserError(str(error))
 
-        bucket = conn.lookup(bucket_name)
+        if not exists:
-        if not bucket:
+            if not region_name:
-            bucket = conn.create_bucket(bucket_name)
+                bucket = s3.create_bucket(Bucket=bucket_name)
+            else:
+                bucket = s3.create_bucket(
+                    Bucket=bucket_name,
+                    CreateBucketConfiguration={
+                        'LocationConstraint': region_name
+                    })
         return bucket
 
-    @staticmethod
-    def _parse_s3_error(s3error):
-        msg = s3error.reason
-        # S3 error message is a XML message...
-        doc = xml.dom.minidom.parseString(s3error.body)
-        msg_node = doc.getElementsByTagName('Message')
-        if msg_node:
-            msg = '%s: %s' % (msg, msg_node[0].childNodes[0].data)
-        return msg
-
     @api.model
     def _file_read_s3(self, fname, bin_size=False):
         s3uri = S3Uri(fname)
@@ -227,12 +234,20 @@ class IrAttachment(models.Model):
                 "error reading attachment '%s' from object storage", fname
             )
             return ''
-        filekey = bucket.get_key(s3uri.item())
+        try:
-        if filekey:
+            key = s3uri.item()
-            read = base64.b64encode(filekey.get_contents_as_string())
+            bucket.meta.client.head_object(
-        else:
+                Bucket=bucket.name,  Key=key
+            )
+            res = io.BytesIO()
+            bucket.download_fileobj(key, res)
+            res.seek(0)
+            read = base64.b64encode(res.read())
+        except ClientError:
             read = ''
-            _logger.info("attachment '%s' missing on object storage", fname)
+            _logger.info(
+                "attachment '%s' missing on object storage", fname
+            )
         return read
 
     @api.model
@@ -250,19 +265,21 @@ class IrAttachment(models.Model):
             bucket = self._get_s3_bucket()
             bin_data = value.decode('base64')
             key = self._compute_checksum(bin_data)
-            filekey = bucket.get_key(key) or bucket.new_key(key)
+            obj = bucket.Object(key=key)
+            file = io.BytesIO()
+            file.write(bin_data)
+            file.seek(0)
             filename = 's3://%s/%s' % (bucket.name, key)
             try:
-                filekey.set_contents_from_string(bin_data)
+                obj.upload_fileobj(file)
-            except S3ResponseError as error:
+            except ClientError as error:
                 # log verbose error from s3, return short message for user
-                    _logger.exception(
+                _logger.exception(
-                        'Error during storage of the file %s' % filename
+                    'Error during storage of the file %s' % filename
-                    )
+                )
-                    raise exceptions.UserError(
+                raise exceptions.UserError(
-                        _('The file could not be stored: %s') %
+                    _('The file could not be stored: %s') % str(error)
-                        (self._parse_s3_error(error),)
+                )
-                    )
         else:
             filename = super(IrAttachment, self)._file_write(value, checksum)
         return filename
@@ -270,12 +287,6 @@ class IrAttachment(models.Model):
     @api.model
     def _file_delete(self, fname):
         if fname.startswith('s3://'):
-            # using SQL to include files hidden through unlink or due to record
-            # rules
-            cr = self.env.cr
-            cr.execute("SELECT COUNT(*) FROM ir_attachment "
-                       "WHERE store_fname = %s", (fname,))
-            count = cr.fetchone()[0]
             s3uri = S3Uri(fname)
             bucket_name = s3uri.bucket()
             item_name = s3uri.item()
@@ -283,19 +294,21 @@ class IrAttachment(models.Model):
             # otherwise, we might delete files used on a different environment
             if bucket_name == os.environ.get('AWS_BUCKETNAME'):
                 bucket = self._get_s3_bucket()
-                filekey = bucket.get_key(item_name)
+                obj = bucket.Object(key=item_name)
-                if not count and filekey:
+                try:
-                    try:
+                    bucket.meta.client.head_object(
-                        filekey.delete()
+                        Bucket=bucket.name, Key=item_name
-                        _logger.info(
+                    )
-                            'file %s deleted on the object storage' % (fname,)
+                    obj.delete()
-                        )
+                    _logger.info(
-                    except S3ResponseError:
+                        'file %s deleted on the object storage' % (fname,)
-                        # log verbose error from s3, return short message for
+                    )
-                        # user
+                except ClientError:
-                        _logger.exception(
+                    # log verbose error from s3, return short message for
-                            'Error during deletion of the file %s' % fname
+                    # user
-                        )
+                    _logger.exception(
+                        'Error during deletion of the file %s' % fname
+                    )
         else:
             super(IrAttachment, self)._file_delete(fname)
 
@@ -398,7 +411,7 @@ class IrAttachment(models.Model):
                 with closing(registry.cursor()) as cr:
                     try:
                         yield self.env(cr=cr)
-                    except:
+                    except Exception:
                         cr.rollback()
                         raise
                     else:

requirements.txt

@@ -1,4 +1,4 @@
-boto==2.42.0
+boto3==1.9.102
 redis==2.10.5
 python-json-logger==0.1.5
 statsd==3.2.1