Merge pull request #170 from p-tombez/11.0-s3_boto3

[11.0] attachment_s3: Migrate to boto3

attachment_s3/__manifest__.py

@@ -11,7 +11,7 @@
  'category': 'Knowledge Management',
  'depends': ['base', 'base_attachment_object_storage'],
  'external_dependencies': {
-     'python': ['boto'],
+     'python': ['boto3'],
  },
  'website': 'https://www.camptocamp.com',
  'data': [],

attachment_s3/models/ir_attachment.py

@@ -6,7 +6,7 @@
 import base64
 import logging
 import os
-import xml.dom.minidom
+import io
 
 from odoo import _, api, exceptions, models
 from ..s3uri import S3Uri
@@ -14,12 +14,13 @@ from ..s3uri import S3Uri
 _logger = logging.getLogger(__name__)
 
 try:
-    import boto
+    import boto3
-    from boto.exception import S3ResponseError
+    from botocore.exceptions import ClientError, EndpointConnectionError
 except ImportError:
-    boto = None  # noqa
+    boto3 = None  # noqa
-    S3ResponseError = None  # noqa
+    ClientError = None  # noqa
-    _logger.debug("Cannot 'import boto'.")
+    EndpointConnectionError = None  # noqa
+    _logger.debug("Cannot 'import boto3'.")
 
 
 class IrAttachment(models.Model):
@@ -56,13 +57,9 @@ class IrAttachment(models.Model):
             'aws_secret_access_key': secret_key,
         }
         if host:
-            params['host'] = host
+            params['endpoint_url'] = host
         if region_name:
-            # needs specific method for region
-            connect_s3 = boto.s3.connect_to_region
             params['region_name'] = region_name
-        else:
-            connect_s3 = boto.connect_s3
         if not (access_key and secret_key and bucket_name):
             msg = _('If you want to read from the %s S3 bucket, the following '
                     'environment variables must be set:\n'
@@ -76,30 +73,34 @@ class IrAttachment(models.Model):
                     ) % (bucket_name, bucket_name)
 
             raise exceptions.UserError(msg)
-
+        # try:
+        s3 = boto3.resource('s3', **params)
+        bucket = s3.Bucket(bucket_name)
+        exists = True
         try:
-            conn = connect_s3(**params)
+            s3.meta.client.head_bucket(Bucket=bucket_name)
-
+        except ClientError as e:
-        except S3ResponseError as error:
+            # If a client error is thrown, then check that it was a 404 error.
+            # If it was a 404 error, then the bucket does not exist.
+            error_code = e.response['Error']['Code']
+            if error_code == '404':
+                exists = False
+        except EndpointConnectionError as error:
             # log verbose error from s3, return short message for user
             _logger.exception('Error during connection on S3')
-            raise exceptions.UserError(self._parse_s3_error(error))
+            raise exceptions.UserError(str(error))
 
-        bucket = conn.lookup(bucket_name)
+        if not exists:
-        if not bucket:
+            if not region_name:
-            bucket = conn.create_bucket(bucket_name)
+                bucket = s3.create_bucket(Bucket=bucket_name)
+            else:
+                bucket = s3.create_bucket(
+                    Bucket=bucket_name,
+                    CreateBucketConfiguration={
+                        'LocationConstraint': region_name
+                    })
         return bucket
 
-    @staticmethod
-    def _parse_s3_error(s3error):
-        msg = s3error.reason
-        # S3 error message is a XML message...
-        doc = xml.dom.minidom.parseString(s3error.body)
-        msg_node = doc.getElementsByTagName('Message')
-        if msg_node:
-            msg = '%s: %s' % (msg, msg_node[0].childNodes[0].data)
-        return msg
-
     @api.model
     def _store_file_read(self, fname, bin_size=False):
         if fname.startswith('s3://'):
@@ -111,10 +112,16 @@ class IrAttachment(models.Model):
                     "error reading attachment '%s' from object storage", fname
                 )
                 return ''
-            filekey = bucket.get_key(s3uri.item())
+            try:
-            if filekey:
+                key = s3uri.item()
-                read = base64.b64encode(filekey.get_contents_as_string())
+                bucket.meta.client.head_object(
-            else:
+                    Bucket=bucket.name,  Key=key
+                )
+                res = io.BytesIO()
+                bucket.download_fileobj(key, res)
+                res.seek(0)
+                read = base64.b64encode(res.read())
+            except ClientError:
                 read = ''
                 _logger.info(
                     "attachment '%s' missing on object storage", fname
@@ -127,18 +134,20 @@ class IrAttachment(models.Model):
     def _store_file_write(self, key, bin_data):
         if self._storage() == 's3':
             bucket = self._get_s3_bucket()
-            filekey = bucket.get_key(key) or bucket.new_key(key)
+            obj = bucket.Object(key=key)
+            file = io.BytesIO()
+            file.write(bin_data)
+            file.seek(0)
             filename = 's3://%s/%s' % (bucket.name, key)
             try:
-                filekey.set_contents_from_string(bin_data)
+                obj.upload_fileobj(file)
-            except S3ResponseError as error:
+            except ClientError as error:
                 # log verbose error from s3, return short message for user
                 _logger.exception(
                     'Error during storage of the file %s' % filename
                 )
                 raise exceptions.UserError(
-                        _('The file could not be stored: %s') %
+                    _('The file could not be stored: %s') % str(error)
-                        (self._parse_s3_error(error),)
                 )
         else:
             _super = super(IrAttachment, self)
@@ -155,14 +164,16 @@ class IrAttachment(models.Model):
             # otherwise, we might delete files used on a different environment
             if bucket_name == os.environ.get('AWS_BUCKETNAME'):
                 bucket = self._get_s3_bucket()
-                filekey = bucket.get_key(item_name)
+                obj = bucket.Object(key=item_name)
-                if filekey:
                 try:
-                        filekey.delete()
+                    bucket.meta.client.head_object(
+                        Bucket=bucket.name, Key=item_name
+                    )
+                    obj.delete()
                     _logger.info(
                         'file %s deleted on the object storage' % (fname,)
                     )
-                    except S3ResponseError:
+                except ClientError:
                     # log verbose error from s3, return short message for
                     # user
                     _logger.exception(

requirements.txt

@@ -1,4 +1,4 @@
-boto==2.42.0
+boto3==1.9.102
 redis==2.10.5
 python-json-logger==0.1.5
 statsd==3.2.1