mirror of
https://github.com/camptocamp/odoo-cloud-platform.git
synced 2026-06-24 02:08:36 +00:00
Merge pull request #286 from nilshamerlinck/9.0-backport-attachment_azure
[MIG] attachment_azure: Backport from 10.0 to 9.0
This commit is contained in:
Yannick Vaucher
GitHub
@@ -0,0 +1,43 @@
|
||||
===========================================
|
||||
Attachments on Microsoft Azure Blob Storage
|
||||
===========================================
|
||||
|
||||
This addon allows to store the attachments (documents and assets) on `Microsoft Azure
|
||||
Blob Storage <https://docs.microsoft.com/azure/storage/blobs/>`_.
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
|
||||
Activate Azure Blob storage:
|
||||
|
||||
* Create or set the system parameter with the key ``ir_attachment.location``
|
||||
and the value in the form ``azure``.
|
||||
|
||||
Configure accesses with environment variables:
|
||||
|
||||
* ``AZURE_STORAGE_CONNECTION_STRING`` or
|
||||
* ``AZURE_STORAGE_ACCOUNT_NAME``
|
||||
* ``AZURE_STORAGE_ACCOUNT_URL``
|
||||
* ``AZURE_STORAGE_ACCOUNT_KEY``
|
||||
|
||||
One container will be created per database using the `RUNNING_ENV` environment variable
|
||||
and the name of the database. By default, `RUNNING_ENV` is set to `dev`.
|
||||
|
||||
The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``.
|
||||
The strings ``{db}`` and ``{env}`` can be used inside that variable and the values
|
||||
will be replaced respectively by the database name and environment name.
|
||||
|
||||
This addon must be added in the server wide addons with (``--load`` option):
|
||||
|
||||
``--load=web,attachment_azure``
|
||||
|
||||
The System Parameter ``ir_attachment.storage.force.database`` can be customized to
|
||||
force storage of files in the database. See the documentation of the module
|
||||
``base_attachment_object_storage``.
|
||||
|
||||
Limitations
|
||||
-----------
|
||||
|
||||
* You need to call ``env['ir.attachment'].force_storage()`` after
|
||||
having changed the ``ir_attachment.location`` configuration in order to
|
||||
migrate the existing attachments to Azure Blob Storage.
|
||||
@@ -0,0 +1 @@
|
||||
from . import models
|
||||
@@ -0,0 +1,25 @@
|
||||
# Copyright 2016-2021 Camptocamp SA
|
||||
# Copyright 2021 Open Source Integrators
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
{
|
||||
"name": "Attachments on Azure storage",
|
||||
"summary": "Store assets and attachments on a Azure compatible object storage",
|
||||
"version": "9.0.1.0.0",
|
||||
"author": "Camptocamp, "
|
||||
"Open Source Integrators, "
|
||||
"Serpent Consulting Services, "
|
||||
"Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Knowledge Management",
|
||||
"depends": ["base_attachment_object_storage"],
|
||||
"external_dependencies": {
|
||||
"python": [
|
||||
"azure.storage.blob", # pip: azure-storage-blob
|
||||
"azure.identity" # pip: azure-identity
|
||||
],
|
||||
},
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"installable": True,
|
||||
"development_status": "Beta",
|
||||
"maintainers": ["max3903"],
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
from . import ir_attachment
|
||||
@@ -0,0 +1,205 @@
|
||||
# Copyright 2016-2021 Camptocamp SA
|
||||
# Copyright 2021 Open Source Integrators
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from openerp import _, api, exceptions, models
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
from azure.storage.blob import (
|
||||
BlobServiceClient,
|
||||
generate_account_sas,
|
||||
ResourceTypes,
|
||||
AccountSasPermissions,
|
||||
)
|
||||
from azure.core.exceptions import ResourceExistsError, HttpResponseError
|
||||
except ImportError:
|
||||
_logger.debug("Cannot 'import azure-storage-blob'.")
|
||||
|
||||
try:
|
||||
from azure.identity import DefaultAzureCredential
|
||||
except ImportError:
|
||||
_logger.debug("Cannot 'import azure-identity'.")
|
||||
|
||||
|
||||
class IrAttachment(models.Model):
|
||||
_inherit = "ir.attachment"
|
||||
|
||||
def _get_stores(self):
|
||||
l = ["azure"]
|
||||
l += super(IrAttachment, self)._get_stores()
|
||||
return l
|
||||
|
||||
@api.model
|
||||
def _get_blob_service_client(self):
|
||||
"""Connect to Azure and return the blob service client
|
||||
|
||||
The following environment variables must be set:
|
||||
* ``AZURE_STORAGE_CONNECTION_STRING``
|
||||
or
|
||||
* ``AZURE_STORAGE_ACCOUNT_NAME``
|
||||
* ``AZURE_STORAGE_ACCOUNT_URL``
|
||||
* ``AZURE_STORAGE_ACCOUNT_KEY``
|
||||
or if you want to use AAD (pod identity), set it to 1 or 0
|
||||
* ``AZURE_STORAGE_USE_AAD``
|
||||
|
||||
"""
|
||||
connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
|
||||
account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME")
|
||||
account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
|
||||
account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY")
|
||||
account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD")
|
||||
if not (
|
||||
connect_str
|
||||
or (account_name and account_url and account_key)
|
||||
or account_use_aad
|
||||
):
|
||||
msg = _(
|
||||
"If you want to read from the Azure container, you must provide the "
|
||||
"following environment variables:\n"
|
||||
"* AZURE_STORAGE_CONNECTION_STRING\n"
|
||||
"or\n"
|
||||
"* AZURE_STORAGE_ACCOUNT_NAME\n"
|
||||
"* AZURE_STORAGE_ACCOUNT_URL\n"
|
||||
"* AZURE_STORAGE_ACCOUNT_KEY\n"
|
||||
"or\n"
|
||||
"* AZURE_STORAGE_USE_AAD\n"
|
||||
)
|
||||
raise exceptions.UserError(msg)
|
||||
blob_service_client = None
|
||||
if account_use_aad:
|
||||
token_credential = DefaultAzureCredential()
|
||||
blob_service_client = BlobServiceClient(
|
||||
account_url=account_url, credential=token_credential
|
||||
)
|
||||
elif connect_str:
|
||||
try:
|
||||
blob_service_client = BlobServiceClient.from_connection_string(
|
||||
connect_str
|
||||
)
|
||||
except HttpResponseError as error:
|
||||
_logger.exception(
|
||||
"Error during the connection to Azure container using the "
|
||||
"connection string."
|
||||
)
|
||||
raise exceptions.UserError(str(error))
|
||||
else:
|
||||
try:
|
||||
sas_token = generate_account_sas(
|
||||
account_name=account_name,
|
||||
account_key=account_key,
|
||||
resource_types=ResourceTypes(container=True, object=True),
|
||||
permission=AccountSasPermissions(read=True, write=True),
|
||||
expiry=datetime.utcnow() + timedelta(hours=1),
|
||||
)
|
||||
blob_service_client = BlobServiceClient(
|
||||
account_url=account_url,
|
||||
credential=sas_token,
|
||||
)
|
||||
except HttpResponseError as error:
|
||||
_logger.exception(
|
||||
"Error during the connection to Azure container using the Shared "
|
||||
"Access Signature (SAS)"
|
||||
)
|
||||
raise exceptions.UserError(str(error))
|
||||
return blob_service_client
|
||||
|
||||
@api.model
|
||||
def _get_container_name(self):
|
||||
"""
|
||||
Container naming rules:
|
||||
https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
|
||||
"""
|
||||
running_env = os.environ.get("RUNNING_ENV", "dev")
|
||||
storage_name = os.environ.get('AZURE_STORAGE_NAME', r'{env}-{db}')
|
||||
storage_name = storage_name.format(
|
||||
env=running_env,
|
||||
db=self.env.cr.dbname
|
||||
)
|
||||
# replace invalid characters by _
|
||||
storage_name = re.sub(r"[\W_]+", "-", storage_name)
|
||||
# lowercase, max 63 chars
|
||||
return str.lower(storage_name)[:63]
|
||||
|
||||
@api.model
|
||||
def _get_azure_container(self):
|
||||
container_name = self._get_container_name()
|
||||
blob_service_client = self._get_blob_service_client()
|
||||
container_client = blob_service_client.get_container_client(container_name)
|
||||
try:
|
||||
# Create the container
|
||||
container_client.create_container()
|
||||
except ResourceExistsError:
|
||||
pass
|
||||
except HttpResponseError as error:
|
||||
_logger.exception("Error during the creation of the Azure container")
|
||||
raise exceptions.UserError(str(error))
|
||||
return container_client
|
||||
|
||||
@api.model
|
||||
def _store_file_read(self, fname, bin_size=False):
|
||||
if fname.startswith("azure://"):
|
||||
container_client = self._get_azure_container()
|
||||
key = fname.replace("azure://", "", 1).lower()
|
||||
try:
|
||||
blob_client = container_client.get_blob_client(key)
|
||||
if bin_size:
|
||||
return blob_client.get_blob_properties()['size']
|
||||
else:
|
||||
read = base64.b64encode(blob_client.download_blob().readall())
|
||||
except HttpResponseError:
|
||||
read = ""
|
||||
_logger.info("Attachment '%s' missing on object storage", fname)
|
||||
return read
|
||||
else:
|
||||
return super(IrAttachment, self)._store_file_read(fname, bin_size)
|
||||
|
||||
@api.model
|
||||
def _store_file_write(self, key, bin_data):
|
||||
location = self.env.context.get("storage_location") or self._storage()
|
||||
if location == "azure":
|
||||
container_client = self._get_azure_container()
|
||||
with io.BytesIO() as file:
|
||||
blob_client = container_client.get_blob_client(key.lower())
|
||||
file.write(bin_data)
|
||||
file.seek(0)
|
||||
filename = "azure://%s" % (key)
|
||||
try:
|
||||
blob_client.upload_blob(file, blob_type="BlockBlob")
|
||||
except ResourceExistsError:
|
||||
pass
|
||||
except HttpResponseError as error:
|
||||
# log verbose error from azure, return short message for user
|
||||
_logger.exception("Error during storage of the file %s" % filename)
|
||||
raise exceptions.UserError(
|
||||
_("The file could not be stored: %s") % str(error)
|
||||
)
|
||||
else:
|
||||
_super = super(IrAttachment, self)
|
||||
filename = _super._store_file_write(key, bin_data)
|
||||
return filename
|
||||
|
||||
@api.model
|
||||
def _store_file_delete(self, fname):
|
||||
if fname.startswith("azure://"):
|
||||
container_client = self._get_azure_container()
|
||||
key = fname.replace("azure://", "", 1).lower()
|
||||
# delete the file only if it is on the current configured container
|
||||
# otherwise, we might delete files used on a different environment
|
||||
try:
|
||||
blob_client = container_client.get_blob_client(key)
|
||||
blob_client.delete_blob()
|
||||
_logger.info("File %s deleted on the object storage" % (fname))
|
||||
except HttpResponseError:
|
||||
# log verbose error from azure, return short message for
|
||||
# user
|
||||
_logger.exception("Error during deletion of the file %s" % fname)
|
||||
else:
|
||||
super(IrAttachment, self)._store_file_delete(fname)
|
||||
@@ -1,3 +1,5 @@
|
||||
azure-storage-blob==12.8.1
|
||||
azure-identity==1.6.0
|
||||
boto3==1.9.102
|
||||
redis==2.10.5
|
||||
python-json-logger==0.1.5
|
||||
|
||||
Reference in New Issue
Block a user