mirror of
https://github.com/camptocamp/odoo-cloud-platform.git
synced 2026-06-24 16:48:36 +00:00
Merge pull request #172 from p-tombez/9.0-s3_boto3
[9.0] attachment_s3: Migrate to boto3
This commit is contained in:
Patrick Tombez
GitHub
@@ -11,7 +11,7 @@
|
||||
'category': 'Knowledge Management',
|
||||
'depends': ['base'],
|
||||
'external_dependencies': {
|
||||
'python': ['boto'],
|
||||
'python': ['boto3'],
|
||||
},
|
||||
'website': 'http://www.camptocamp.com',
|
||||
'data': [
|
||||
|
||||
@@ -7,9 +7,8 @@ import base64
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import xml.dom.minidom
|
||||
from contextlib import closing, contextmanager
|
||||
from functools import partial
|
||||
import io
|
||||
|
||||
import psycopg2
|
||||
|
||||
@@ -20,12 +19,13 @@ from ..s3uri import S3Uri
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import boto
|
||||
from boto.exception import S3ResponseError
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError, EndpointConnectionError
|
||||
except ImportError:
|
||||
boto = None # noqa
|
||||
S3ResponseError = None # noqa
|
||||
_logger.debug("Cannot 'import boto'.")
|
||||
boto3 = None # noqa
|
||||
ClientError = None # noqa
|
||||
EndpointConnectionError = None # noqa
|
||||
_logger.debug("Cannot 'import boto3'.")
|
||||
|
||||
|
||||
def clean_fs(files):
|
||||
@@ -159,6 +159,7 @@ class IrAttachment(models.Model):
|
||||
|
||||
The following environment variables can be set:
|
||||
* ``AWS_HOST``
|
||||
* ``AWS_REGION``
|
||||
* ``AWS_ACCESS_KEY_ID``
|
||||
* ``AWS_SECRET_ACCESS_KEY``
|
||||
* ``AWS_BUCKETNAME``
|
||||
@@ -168,17 +169,20 @@ class IrAttachment(models.Model):
|
||||
|
||||
"""
|
||||
host = os.environ.get('AWS_HOST')
|
||||
if host:
|
||||
connect_s3 = partial(boto.connect_s3, host=host)
|
||||
else:
|
||||
connect_s3 = boto.connect_s3
|
||||
|
||||
region_name = os.environ.get('AWS_REGION')
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
if name:
|
||||
bucket_name = name
|
||||
else:
|
||||
bucket_name = os.environ.get('AWS_BUCKETNAME')
|
||||
bucket_name = name or os.environ.get('AWS_BUCKETNAME')
|
||||
|
||||
params = {
|
||||
'aws_access_key_id': access_key,
|
||||
'aws_secret_access_key': secret_key,
|
||||
}
|
||||
|
||||
if host:
|
||||
params['endpoint_url'] = host
|
||||
if region_name:
|
||||
params['region_name'] = region_name
|
||||
if not (access_key and secret_key and bucket_name):
|
||||
msg = _('If you want to read from the %s S3 bucket, the following '
|
||||
'environment variables must be set:\n'
|
||||
@@ -192,31 +196,34 @@ class IrAttachment(models.Model):
|
||||
) % (bucket_name, bucket_name)
|
||||
|
||||
raise exceptions.UserError(msg)
|
||||
|
||||
# try:
|
||||
s3 = boto3.resource('s3', **params)
|
||||
bucket = s3.Bucket(bucket_name)
|
||||
exists = True
|
||||
try:
|
||||
conn = connect_s3(aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key)
|
||||
|
||||
except S3ResponseError as error:
|
||||
s3.meta.client.head_bucket(Bucket=bucket_name)
|
||||
except ClientError as e:
|
||||
# If a client error is thrown, then check that it was a 404 error.
|
||||
# If it was a 404 error, then the bucket does not exist.
|
||||
error_code = e.response['Error']['Code']
|
||||
if error_code == '404':
|
||||
exists = False
|
||||
except EndpointConnectionError as error:
|
||||
# log verbose error from s3, return short message for user
|
||||
_logger.exception('Error during connection on S3')
|
||||
raise exceptions.UserError(self._parse_s3_error(error))
|
||||
raise exceptions.UserError(str(error))
|
||||
|
||||
bucket = conn.lookup(bucket_name)
|
||||
if not bucket:
|
||||
bucket = conn.create_bucket(bucket_name)
|
||||
if not exists:
|
||||
if not region_name:
|
||||
bucket = s3.create_bucket(Bucket=bucket_name)
|
||||
else:
|
||||
bucket = s3.create_bucket(
|
||||
Bucket=bucket_name,
|
||||
CreateBucketConfiguration={
|
||||
'LocationConstraint': region_name
|
||||
})
|
||||
return bucket
|
||||
|
||||
@staticmethod
|
||||
def _parse_s3_error(s3error):
|
||||
msg = s3error.reason
|
||||
# S3 error message is a XML message...
|
||||
doc = xml.dom.minidom.parseString(s3error.body)
|
||||
msg_node = doc.getElementsByTagName('Message')
|
||||
if msg_node:
|
||||
msg = '%s: %s' % (msg, msg_node[0].childNodes[0].data)
|
||||
return msg
|
||||
|
||||
@api.model
|
||||
def _file_read_s3(self, fname, bin_size=False):
|
||||
s3uri = S3Uri(fname)
|
||||
@@ -227,12 +234,20 @@ class IrAttachment(models.Model):
|
||||
"error reading attachment '%s' from object storage", fname
|
||||
)
|
||||
return ''
|
||||
filekey = bucket.get_key(s3uri.item())
|
||||
if filekey:
|
||||
read = base64.b64encode(filekey.get_contents_as_string())
|
||||
else:
|
||||
try:
|
||||
key = s3uri.item()
|
||||
bucket.meta.client.head_object(
|
||||
Bucket=bucket.name, Key=key
|
||||
)
|
||||
res = io.BytesIO()
|
||||
bucket.download_fileobj(key, res)
|
||||
res.seek(0)
|
||||
read = base64.b64encode(res.read())
|
||||
except ClientError:
|
||||
read = ''
|
||||
_logger.info("attachment '%s' missing on object storage", fname)
|
||||
_logger.info(
|
||||
"attachment '%s' missing on object storage", fname
|
||||
)
|
||||
return read
|
||||
|
||||
@api.model
|
||||
@@ -250,18 +265,20 @@ class IrAttachment(models.Model):
|
||||
bucket = self._get_s3_bucket()
|
||||
bin_data = value.decode('base64')
|
||||
key = self._compute_checksum(bin_data)
|
||||
filekey = bucket.get_key(key) or bucket.new_key(key)
|
||||
obj = bucket.Object(key=key)
|
||||
file = io.BytesIO()
|
||||
file.write(bin_data)
|
||||
file.seek(0)
|
||||
filename = 's3://%s/%s' % (bucket.name, key)
|
||||
try:
|
||||
filekey.set_contents_from_string(bin_data)
|
||||
except S3ResponseError as error:
|
||||
obj.upload_fileobj(file)
|
||||
except ClientError as error:
|
||||
# log verbose error from s3, return short message for user
|
||||
_logger.exception(
|
||||
'Error during storage of the file %s' % filename
|
||||
)
|
||||
raise exceptions.UserError(
|
||||
_('The file could not be stored: %s') %
|
||||
(self._parse_s3_error(error),)
|
||||
_('The file could not be stored: %s') % str(error)
|
||||
)
|
||||
else:
|
||||
filename = super(IrAttachment, self)._file_write(value, checksum)
|
||||
@@ -270,12 +287,6 @@ class IrAttachment(models.Model):
|
||||
@api.model
|
||||
def _file_delete(self, fname):
|
||||
if fname.startswith('s3://'):
|
||||
# using SQL to include files hidden through unlink or due to record
|
||||
# rules
|
||||
cr = self.env.cr
|
||||
cr.execute("SELECT COUNT(*) FROM ir_attachment "
|
||||
"WHERE store_fname = %s", (fname,))
|
||||
count = cr.fetchone()[0]
|
||||
s3uri = S3Uri(fname)
|
||||
bucket_name = s3uri.bucket()
|
||||
item_name = s3uri.item()
|
||||
@@ -283,14 +294,16 @@ class IrAttachment(models.Model):
|
||||
# otherwise, we might delete files used on a different environment
|
||||
if bucket_name == os.environ.get('AWS_BUCKETNAME'):
|
||||
bucket = self._get_s3_bucket()
|
||||
filekey = bucket.get_key(item_name)
|
||||
if not count and filekey:
|
||||
obj = bucket.Object(key=item_name)
|
||||
try:
|
||||
filekey.delete()
|
||||
bucket.meta.client.head_object(
|
||||
Bucket=bucket.name, Key=item_name
|
||||
)
|
||||
obj.delete()
|
||||
_logger.info(
|
||||
'file %s deleted on the object storage' % (fname,)
|
||||
)
|
||||
except S3ResponseError:
|
||||
except ClientError:
|
||||
# log verbose error from s3, return short message for
|
||||
# user
|
||||
_logger.exception(
|
||||
@@ -398,7 +411,7 @@ class IrAttachment(models.Model):
|
||||
with closing(registry.cursor()) as cr:
|
||||
try:
|
||||
yield self.env(cr=cr)
|
||||
except:
|
||||
except Exception:
|
||||
cr.rollback()
|
||||
raise
|
||||
else:
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
boto==2.42.0
|
||||
boto3==1.9.102
|
||||
redis==2.10.5
|
||||
python-json-logger==0.1.5
|
||||
statsd==3.2.1
|
||||
|
||||
Reference in New Issue
Block a user