diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df9807c..106246e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ exclude: | (?x) # NOT INSTALLABLE ADDONS + ^attachment_s3/| ^base_fileurl_field/| ^monitoring_log_requests/| ^monitoring_statsd/| - ^test_base_fileurl_field/| # END NOT INSTALLABLE ADDONS # Files and folders generated by bots, to avoid loops ^setup/|/static/description/index\.html$| diff --git a/attachment_s3/README.rst b/attachment_s3/README.rst new file mode 100644 index 0000000..0d03699 --- /dev/null +++ b/attachment_s3/README.rst @@ -0,0 +1,58 @@ +Attachments on S3 storage +========================= + +This addon allows to store the attachments (documents and assets) on S3 or any +other S3-compatible Object Storage. + +Configuration +------------- + +Activate S3 storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` + and the value in the form ``s3``. + +Configure accesses with environment variables: + +* ``AWS_HOST`` (not required if using AWS services) +* ``AWS_REGION`` (required if using AWS services) +* ``AWS_ACCESS_KEY_ID`` +* ``AWS_SECRET_ACCESS_KEY`` +* ``AWS_BUCKETNAME`` (optional {db} placeholder) + +Read-only mode: + +The bucket and the file key are stored in the attachment. So if you change the +``AWS_BUCKETNAME`` or the ``ir_attachment.location``, the existing attachments +will still be read on their former bucket. But as soon as they are written over +or new attachments are created, they will be created on the new bucket or on +the other location (db or filesystem). This is a convenient way to be able to +read the production attachments on a replication (since you have the +credentials) without any risk to alter the production data. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,attachment_s3`` + +The System Parameter ``ir_attachment.storage.force.database`` can be customized to +force storage of files in the database. See the documentation of the module +``base_attachment_object_storage``. + +Multi-tenancy +------------- + +Use the `{db}` placeholder to handle multi-tenancy. + +On instances that hold multiple databases, it's preferable to have one bucket per database. + +To handle this, you can insert the `{db}` placeholder in your bucket name variable ``AWS_BUCKETNAME``. +It will be replaced by the database name. +This will give you a unique bucketname per database. + + +Limitations +----------- + +* You need to call ``env['ir.attachment'].force_storage()`` after + having changed the ``ir_attachment.location`` configuration in order to + migrate the existing attachments to S3. diff --git a/attachment_s3/__init__.py b/attachment_s3/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/attachment_s3/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/attachment_s3/__manifest__.py b/attachment_s3/__manifest__.py new file mode 100644 index 0000000..7958e62 --- /dev/null +++ b/attachment_s3/__manifest__.py @@ -0,0 +1,19 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Attachments on S3 storage", + "summary": "Store assets and attachments on a S3 compatible object storage", + "version": "15.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base", "base_attachment_object_storage"], + "external_dependencies": { + "python": ["boto3"], + }, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": False, +} diff --git a/attachment_s3/models/__init__.py b/attachment_s3/models/__init__.py new file mode 100644 index 0000000..aaf38a1 --- /dev/null +++ b/attachment_s3/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_s3/models/ir_attachment.py b/attachment_s3/models/ir_attachment.py new file mode 100644 index 0000000..d18c0e2 --- /dev/null +++ b/attachment_s3/models/ir_attachment.py @@ -0,0 +1,177 @@ +# Copyright 2016-2019 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +import io +import logging +import os +from urllib.parse import urlsplit + +from odoo import _, api, exceptions, models + +from ..s3uri import S3Uri + +_logger = logging.getLogger(__name__) + +try: + import boto3 + from botocore.exceptions import ClientError, EndpointConnectionError +except ImportError: + boto3 = None # noqa + ClientError = None # noqa + EndpointConnectionError = None # noqa + _logger.debug("Cannot 'import boto3'.") + + +class IrAttachment(models.Model): + _inherit = "ir.attachment" + + def _get_stores(self): + return ["s3"] + super()._get_stores() + + @api.model + def _get_s3_bucket(self, name=None): + """Connect to S3 and return the bucket + + The following environment variables can be set: + * ``AWS_HOST`` + * ``AWS_REGION`` + * ``AWS_ACCESS_KEY_ID`` + * ``AWS_SECRET_ACCESS_KEY`` + * ``AWS_BUCKETNAME`` + + If a name is provided, we'll read this bucket, otherwise, the bucket + from the environment variable ``AWS_BUCKETNAME`` will be read. + + """ + host = os.environ.get("AWS_HOST") + + # Ensure host is prefixed with a scheme (use https as default) + if host and not urlsplit(host).scheme: + host = "https://%s" % host + + region_name = os.environ.get("AWS_REGION") + access_key = os.environ.get("AWS_ACCESS_KEY_ID") + secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") + bucket_name = name or os.environ.get("AWS_BUCKETNAME") + # replaces {db} by the database name to handle multi-tenancy + bucket_name = bucket_name.format(db=self.env.cr.dbname) + + params = { + "aws_access_key_id": access_key, + "aws_secret_access_key": secret_key, + } + if host: + params["endpoint_url"] = host + if region_name: + params["region_name"] = region_name + if not (access_key and secret_key and bucket_name): + msg = _( + "If you want to read from the %(bucket_name)s S3 bucket, the following " + "environment variables must be set:\n" + "* AWS_ACCESS_KEY_ID\n" + "* AWS_SECRET_ACCESS_KEY\n" + "If you want to write in the %(bucket_name)s S3 bucket, this variable " + "must be set as well:\n" + "* AWS_BUCKETNAME\n" + "Optionally, the S3 host can be changed with:\n" + "* AWS_HOST\n" + ).format(bucket_name=bucket_name) + + raise exceptions.UserError(msg) + # try: + s3 = boto3.resource("s3", **params) + bucket = s3.Bucket(bucket_name) + exists = True + try: + s3.meta.client.head_bucket(Bucket=bucket_name) + except ClientError as e: + # If a client error is thrown, then check that it was a 404 error. + # If it was a 404 error, then the bucket does not exist. + error_code = e.response["Error"]["Code"] + if error_code == "404": + exists = False + except EndpointConnectionError as error: + # log verbose error from s3, return short message for user + msg = _logger.exception("Error during connection on S3") + raise exceptions.UserError(str(error)) from None + + if not exists: + if not region_name: + bucket = s3.create_bucket(Bucket=bucket_name) + else: + bucket = s3.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={"LocationConstraint": region_name}, + ) + return bucket + + @api.model + def _store_file_read(self, fname): + if fname.startswith("s3://"): + s3uri = S3Uri(fname) + try: + bucket = self._get_s3_bucket(name=s3uri.bucket()) + except exceptions.UserError: + _logger.exception( + "error reading attachment '%s' from object storage", fname + ) + return "" + try: + key = s3uri.item() + bucket.meta.client.head_object(Bucket=bucket.name, Key=key) + with io.BytesIO() as res: + bucket.download_fileobj(key, res) + res.seek(0) + read = res.read() + except ClientError: + read = "" + _logger.info("attachment '%s' missing on object storage", fname) + return read + else: + return super()._store_file_read(fname) + + @api.model + def _store_file_write(self, key, bin_data): + location = self.env.context.get("storage_location") or self._storage() + if location == "s3": + bucket = self._get_s3_bucket() + obj = bucket.Object(key=key) + with io.BytesIO() as file: + file.write(bin_data) + file.seek(0) + filename = "s3://%s/%s" % (bucket.name, key) + try: + obj.upload_fileobj(file) + except ClientError as error: + # log verbose error from s3, return short message for user + _logger.exception("Error during storage of the file %s" % filename) + raise exceptions.UserError( + _("The file could not be stored: %s") % str(error) + ) from None + else: + _super = super() + filename = _super._store_file_write(key, bin_data) + return filename + + @api.model + def _store_file_delete(self, fname): + if fname.startswith("s3://"): + s3uri = S3Uri(fname) + bucket_name = s3uri.bucket() + item_name = s3uri.item() + # delete the file only if it is on the current configured bucket + # otherwise, we might delete files used on a different environment + if bucket_name == os.environ.get("AWS_BUCKETNAME"): + bucket = self._get_s3_bucket() + obj = bucket.Object(key=item_name) + try: + bucket.meta.client.head_object(Bucket=bucket.name, Key=item_name) + obj.delete() + _logger.info("file %s deleted on the object storage" % (fname,)) + except ClientError: + # log verbose error from s3, return short message for + # user + _logger.exception("Error during deletion of the file %s" % fname) + else: + return super()._store_file_delete(fname) diff --git a/attachment_s3/s3uri.py b/attachment_s3/s3uri.py new file mode 100644 index 0000000..5bbe11a --- /dev/null +++ b/attachment_s3/s3uri.py @@ -0,0 +1,20 @@ +# Copyright 2016-2019 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class S3Uri: + _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid S3 URI" % (uri,)) + self._bucket, self._item = match.groups() + + def bucket(self): + return self._bucket + + def item(self): + return self._item diff --git a/test_base_fileurl_field/README.rst b/test_base_fileurl_field/README.rst deleted file mode 100644 index 0061d73..0000000 --- a/test_base_fileurl_field/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -Test Base FileURL Field -======================= - -This module serves as implementation example for `base_fileurl_field` and to run its tests. diff --git a/test_base_fileurl_field/__init__.py b/test_base_fileurl_field/__init__.py deleted file mode 100644 index 7967689..0000000 --- a/test_base_fileurl_field/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright 2019 Camptocamp SA -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) -from . import models diff --git a/test_base_fileurl_field/__manifest__.py b/test_base_fileurl_field/__manifest__.py deleted file mode 100644 index 4c885b0..0000000 --- a/test_base_fileurl_field/__manifest__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2019 Camptocamp SA -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) -{ - "name": "test base fileurl fields", - "summary": """A module to verify fileurl field.""", - "version": "17.0.1.0.0", - "category": "Tests", - "author": "Camptocamp,Odoo Community Association (OCA)", - "website": "https://github.com/camptocamp/odoo-cloud-platform", - "license": "AGPL-3", - "depends": ["base_fileurl_field"], - "data": [ - "views/res_partner.xml", - "views/res_users.xml", - ], - "installable": False, - "auto_install": False, -} diff --git a/test_base_fileurl_field/data/pattern.png b/test_base_fileurl_field/data/pattern.png deleted file mode 100644 index ce52599..0000000 Binary files a/test_base_fileurl_field/data/pattern.png and /dev/null differ diff --git a/test_base_fileurl_field/data/sample.txt b/test_base_fileurl_field/data/sample.txt deleted file mode 100644 index 8a03e0e..0000000 --- a/test_base_fileurl_field/data/sample.txt +++ /dev/null @@ -1 +0,0 @@ -This is a simple text file. diff --git a/test_base_fileurl_field/models/__init__.py b/test_base_fileurl_field/models/__init__.py deleted file mode 100644 index 906bfc7..0000000 --- a/test_base_fileurl_field/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import res_partner -from . import res_users diff --git a/test_base_fileurl_field/models/res_partner.py b/test_base_fileurl_field/models/res_partner.py deleted file mode 100644 index 48ca0a4..0000000 --- a/test_base_fileurl_field/models/res_partner.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 Camptocamp SA -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) -from odoo import _, api, fields, models -from odoo.exceptions import ValidationError - - -class ResPartner(models.Model): - - _inherit = "res.partner" - - name = fields.Char() - url_file = fields.FileURL( - storage_location="s3", filename="url_file_fname", storage_path="partner" - ) - url_file_fname = fields.Char() - - url_image = fields.FileURL( - storage_location="s3", - filename="url_image_fname", - storage_path="partner_image", - ) - url_image_fname = fields.Char() - - @api.constrains("url_file", "url_file_fname") - def _check_url_file_fname(self): - rec = self.search([("url_file_fname", "=", self.url_file_fname)]) - if len(rec) > 1: - raise ValidationError( - _( - "This file name is already used on an existing record. " - "Please use another file name or delete the url_file on :\n" - "Model: %s Id: %s" % (self._name, rec.id) - ) - ) - - @api.constrains("url_image", "url_image_fname") - def _check_url_image_fname(self): - rec = self.search([("url_image_fname", "=", self.url_image_fname)]) - if len(rec) > 1: - raise ValidationError( - _( - "This file name is already used on an existing record. " - "Please use another file name or delete the url_image on :\n" - "Model: %s Id: %s" % (self._name, rec.id) - ) - ) diff --git a/test_base_fileurl_field/models/res_users.py b/test_base_fileurl_field/models/res_users.py deleted file mode 100644 index c8bc324..0000000 --- a/test_base_fileurl_field/models/res_users.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2019 Camptocamp SA -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) -from odoo import fields, models - - -class ResUsers(models.Model): - - _inherit = "res.users" - - partner_url_file = fields.FileURL(related="partner_id.url_file") - partner_url_file_fname = fields.Char(related="partner_id.url_file_fname") diff --git a/test_base_fileurl_field/tests/__init__.py b/test_base_fileurl_field/tests/__init__.py deleted file mode 100644 index 56a55f4..0000000 --- a/test_base_fileurl_field/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import ir_attachment -from . import test_fileurl_fields diff --git a/test_base_fileurl_field/tests/ir_attachment.py b/test_base_fileurl_field/tests/ir_attachment.py deleted file mode 100644 index 8803571..0000000 --- a/test_base_fileurl_field/tests/ir_attachment.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2019 Camptocamp SA -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) - -import logging - -from odoo import api, models - -_logger = logging.getLogger(__name__) - -FAKE_S3_BUCKET = {} - - -class IrAttachment(models.Model): - _inherit = "ir.attachment" - - def _get_stores(self): - l = ["s3"] - l += super(IrAttachment, self)._get_stores() - return l - - @api.model - def _store_file_read(self, fname, bin_size=False): - if fname.startswith("s3://"): - return FAKE_S3_BUCKET.get(fname) - else: - return super(IrAttachment, self)._store_file_read(fname, bin_size) - - @api.model - def _store_file_write(self, key, bin_data): - location = self.env.context.get("storage_location") or self._storage() - if location == "s3": - FAKE_S3_BUCKET[key] = bin_data - filename = "s3://fake_bucket/%s" % key - else: - _super = super(IrAttachment, self) - filename = _super._store_file_write(key, bin_data) - return filename - - @api.model - def _store_file_delete(self, fname): - if fname.startswith("s3://"): - FAKE_S3_BUCKET.pop(fname) - else: - super(IrAttachment, self)._store_file_delete(fname) diff --git a/test_base_fileurl_field/tests/test_fileurl_fields.py b/test_base_fileurl_field/tests/test_fileurl_fields.py deleted file mode 100644 index 8be2bec..0000000 --- a/test_base_fileurl_field/tests/test_fileurl_fields.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2019 Camptocamp SA -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) -import base64 - -from odoo.exceptions import ValidationError -from odoo.modules.module import get_module_resource -from odoo.tests import TransactionCase - - -class TestFileUrlFields(TransactionCase): - def test_fileurl_fields(self): - file_path = get_module_resource("test_base_fileurl_field", "data", "sample.txt") - image_path = get_module_resource( - "test_base_fileurl_field", "data", "pattern.png" - ) - partner = self.env.ref("base.main_partner") - with open(file_path, "rb") as f: - with open(image_path, "rb") as i: - partner.write( - { - "url_file": base64.b64encode(f.read()), - "url_file_fname": "sample.txt", - "url_image": base64.b64encode(i.read()), - "url_image_fname": "pattern.png", - } - ) - - with open(file_path, "rb") as f: - self.assertEqual(base64.decodebytes(partner.url_file), f.read()) - - with open(image_path, "rb") as i: - self.assertEqual(base64.decodebytes(partner.url_image), i.read()) - - partner2 = self.env.ref("base.partner_admin") - with open(file_path, "rb") as f: - with self.assertRaises(ValidationError): - partner2.write( - { - "url_file": base64.b64encode(f.read()), - "url_file_fname": "sample.txt", - } - ) diff --git a/test_base_fileurl_field/views/res_partner.xml b/test_base_fileurl_field/views/res_partner.xml deleted file mode 100644 index f3367e9..0000000 --- a/test_base_fileurl_field/views/res_partner.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - res.partner.form.inherit - res.partner - - - - - - - - - - - - - - - - - diff --git a/test_base_fileurl_field/views/res_users.xml b/test_base_fileurl_field/views/res_users.xml deleted file mode 100644 index cbfdc72..0000000 --- a/test_base_fileurl_field/views/res_users.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - res.users.form.inherit - res.users - - - - - - - - - - - - -