mirror of
https://github.com/camptocamp/odoo-cloud-platform.git
synced 2026-06-24 02:08:36 +00:00
Merge pull request #421 from camptocamp/13.0-ci-to-gha
[13.0] Change CI to GitHub actions
This commit is contained in:
Yannick Payot
GitHub
@@ -0,0 +1,27 @@
|
||||
# Do NOT update manually; changes here will be overwritten by Copier
|
||||
_commit: v1.14.2
|
||||
_src_path: https://github.com/OCA/oca-addons-repo-template.git
|
||||
ci: GitHub
|
||||
dependency_installation_mode: PIP
|
||||
generate_requirements_txt: false
|
||||
github_check_license: true
|
||||
github_ci_extra_env: {}
|
||||
github_enable_codecov: true
|
||||
github_enable_makepot: true
|
||||
github_enable_stale_action: true
|
||||
github_enforce_dev_status_compatibility: false
|
||||
include_wkhtmltopdf: false
|
||||
odoo_version: 13.0
|
||||
org_name: Camptocamp
|
||||
org_slug: camptocamp
|
||||
rebel_module_groups:
|
||||
- attachment_s3,cloud_platform_exoscale
|
||||
- attachment_swift,cloud_platform_ovh
|
||||
- attachment_azure,cloud_platform_azure
|
||||
repo_description: None
|
||||
repo_name: Odoo Cloud Addons
|
||||
repo_slug: odoo-cloud-platform
|
||||
repo_website: https://github.com/camptocamp/odoo-cloud-platform
|
||||
travis_apt_packages: []
|
||||
travis_apt_sources: []
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Configuration for known file extensions
|
||||
[*.{css,js,json,less,md,py,rst,sass,scss,xml,yaml,yml}]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
indent_size = 4
|
||||
indent_style = space
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[*.{json,yml,yaml,rst,md}]
|
||||
indent_size = 2
|
||||
|
||||
# Do not configure editor for libs and autogenerated content
|
||||
[{*/static/{lib,src/lib}/**,*/static/description/index.html,*/readme/../README.rst}]
|
||||
charset = unset
|
||||
end_of_line = unset
|
||||
indent_size = unset
|
||||
indent_style = unset
|
||||
insert_final_newline = false
|
||||
trim_trailing_whitespace = false
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
env:
|
||||
browser: true
|
||||
es6: true
|
||||
|
||||
# See https://github.com/OCA/odoo-community.org/issues/37#issuecomment-470686449
|
||||
parserOptions:
|
||||
ecmaVersion: 2019
|
||||
|
||||
overrides:
|
||||
- files:
|
||||
- "**/*.esm.js"
|
||||
parserOptions:
|
||||
sourceType: module
|
||||
|
||||
# Globals available in Odoo that shouldn't produce errorings
|
||||
globals:
|
||||
_: readonly
|
||||
$: readonly
|
||||
fuzzy: readonly
|
||||
jQuery: readonly
|
||||
moment: readonly
|
||||
odoo: readonly
|
||||
openerp: readonly
|
||||
owl: readonly
|
||||
|
||||
# Styling is handled by Prettier, so we only need to enable AST rules;
|
||||
# see https://github.com/OCA/maintainer-quality-tools/pull/618#issuecomment-558576890
|
||||
rules:
|
||||
accessor-pairs: warn
|
||||
array-callback-return: warn
|
||||
callback-return: warn
|
||||
capitalized-comments:
|
||||
- warn
|
||||
- always
|
||||
- ignoreConsecutiveComments: true
|
||||
ignoreInlineComments: true
|
||||
complexity:
|
||||
- warn
|
||||
- 15
|
||||
constructor-super: warn
|
||||
dot-notation: warn
|
||||
eqeqeq: warn
|
||||
global-require: warn
|
||||
handle-callback-err: warn
|
||||
id-blacklist: warn
|
||||
id-match: warn
|
||||
init-declarations: error
|
||||
max-depth: warn
|
||||
max-nested-callbacks: warn
|
||||
max-statements-per-line: warn
|
||||
no-alert: warn
|
||||
no-array-constructor: warn
|
||||
no-caller: warn
|
||||
no-case-declarations: warn
|
||||
no-class-assign: warn
|
||||
no-cond-assign: error
|
||||
no-const-assign: error
|
||||
no-constant-condition: warn
|
||||
no-control-regex: warn
|
||||
no-debugger: error
|
||||
no-delete-var: warn
|
||||
no-div-regex: warn
|
||||
no-dupe-args: error
|
||||
no-dupe-class-members: error
|
||||
no-dupe-keys: error
|
||||
no-duplicate-case: error
|
||||
no-duplicate-imports: error
|
||||
no-else-return: warn
|
||||
no-empty-character-class: warn
|
||||
no-empty-function: error
|
||||
no-empty-pattern: error
|
||||
no-empty: warn
|
||||
no-eq-null: error
|
||||
no-eval: error
|
||||
no-ex-assign: error
|
||||
no-extend-native: warn
|
||||
no-extra-bind: warn
|
||||
no-extra-boolean-cast: warn
|
||||
no-extra-label: warn
|
||||
no-fallthrough: warn
|
||||
no-func-assign: error
|
||||
no-global-assign: error
|
||||
no-implicit-coercion:
|
||||
- warn
|
||||
- allow: ["~"]
|
||||
no-implicit-globals: warn
|
||||
no-implied-eval: warn
|
||||
no-inline-comments: warn
|
||||
no-inner-declarations: warn
|
||||
no-invalid-regexp: warn
|
||||
no-irregular-whitespace: warn
|
||||
no-iterator: warn
|
||||
no-label-var: warn
|
||||
no-labels: warn
|
||||
no-lone-blocks: warn
|
||||
no-lonely-if: error
|
||||
no-mixed-requires: error
|
||||
no-multi-str: warn
|
||||
no-native-reassign: error
|
||||
no-negated-condition: warn
|
||||
no-negated-in-lhs: error
|
||||
no-new-func: warn
|
||||
no-new-object: warn
|
||||
no-new-require: warn
|
||||
no-new-symbol: warn
|
||||
no-new-wrappers: warn
|
||||
no-new: warn
|
||||
no-obj-calls: warn
|
||||
no-octal-escape: warn
|
||||
no-octal: warn
|
||||
no-param-reassign: warn
|
||||
no-path-concat: warn
|
||||
no-process-env: warn
|
||||
no-process-exit: warn
|
||||
no-proto: warn
|
||||
no-prototype-builtins: warn
|
||||
no-redeclare: warn
|
||||
no-regex-spaces: warn
|
||||
no-restricted-globals: warn
|
||||
no-restricted-imports: warn
|
||||
no-restricted-modules: warn
|
||||
no-restricted-syntax: warn
|
||||
no-return-assign: error
|
||||
no-script-url: warn
|
||||
no-self-assign: warn
|
||||
no-self-compare: warn
|
||||
no-sequences: warn
|
||||
no-shadow-restricted-names: warn
|
||||
no-shadow: warn
|
||||
no-sparse-arrays: warn
|
||||
no-sync: warn
|
||||
no-this-before-super: warn
|
||||
no-throw-literal: warn
|
||||
no-undef-init: warn
|
||||
no-undef: error
|
||||
no-unmodified-loop-condition: warn
|
||||
no-unneeded-ternary: error
|
||||
no-unreachable: error
|
||||
no-unsafe-finally: error
|
||||
no-unused-expressions: error
|
||||
no-unused-labels: error
|
||||
no-unused-vars: error
|
||||
no-use-before-define: error
|
||||
no-useless-call: warn
|
||||
no-useless-computed-key: warn
|
||||
no-useless-concat: warn
|
||||
no-useless-constructor: warn
|
||||
no-useless-escape: warn
|
||||
no-useless-rename: warn
|
||||
no-void: warn
|
||||
no-with: warn
|
||||
operator-assignment: [error, always]
|
||||
prefer-const: warn
|
||||
radix: warn
|
||||
require-yield: warn
|
||||
sort-imports: warn
|
||||
spaced-comment: [error, always]
|
||||
strict: [error, function]
|
||||
use-isnan: error
|
||||
valid-jsdoc:
|
||||
- warn
|
||||
- prefer:
|
||||
arg: param
|
||||
argument: param
|
||||
augments: extends
|
||||
constructor: class
|
||||
exception: throws
|
||||
func: function
|
||||
method: function
|
||||
prop: property
|
||||
return: returns
|
||||
virtual: abstract
|
||||
yield: yields
|
||||
preferType:
|
||||
array: Array
|
||||
bool: Boolean
|
||||
boolean: Boolean
|
||||
number: Number
|
||||
object: Object
|
||||
str: String
|
||||
string: String
|
||||
requireParamDescription: false
|
||||
requireReturn: false
|
||||
requireReturnDescription: false
|
||||
requireReturnType: false
|
||||
valid-typeof: warn
|
||||
yoda: warn
|
||||
@@ -0,0 +1,12 @@
|
||||
[flake8]
|
||||
max-line-length = 88
|
||||
max-complexity = 16
|
||||
# B = bugbear
|
||||
# B9 = bugbear opinionated (incl line length)
|
||||
select = C,E,F,W,B,B9
|
||||
# E203: whitespace before ':' (black behaviour)
|
||||
# E501: flake8 line length (covered by bugbear B950)
|
||||
# W503: line break before binary operator (black behaviour)
|
||||
ignore = E203,E501,W503
|
||||
per-file-ignores=
|
||||
__init__.py:F401
|
||||
@@ -0,0 +1,37 @@
|
||||
name: pre-commit
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- "13.0*"
|
||||
push:
|
||||
branches:
|
||||
- "13.0"
|
||||
- "13.0-ocabot-*"
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.8"
|
||||
- name: Get python version
|
||||
run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
|
||||
- uses: actions/cache@v1
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
|
||||
- name: Install pre-commit
|
||||
run: pip install pre-commit
|
||||
- name: Run pre-commit
|
||||
run: pre-commit run --all-files --show-diff-on-failure --color=always
|
||||
- name: Check that all files generated by pre-commit are in git
|
||||
run: |
|
||||
newfiles="$(git ls-files --others --exclude-from=.gitignore)"
|
||||
if [ "$newfiles" != "" ] ; then
|
||||
echo "Please check-in the following files:"
|
||||
echo "$newfiles"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,69 @@
|
||||
name: Mark stale issues and pull requests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 12 * * 0"
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Stale PRs and issues policy
|
||||
uses: actions/stale@v4
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
# General settings.
|
||||
ascending: true
|
||||
remove-stale-when-updated: true
|
||||
# Pull Requests settings.
|
||||
# 120+30 day stale policy for PRs
|
||||
# * Except PRs marked as "no stale"
|
||||
days-before-pr-stale: 120
|
||||
days-before-pr-close: 30
|
||||
exempt-pr-labels: "no stale"
|
||||
stale-pr-label: "stale"
|
||||
stale-pr-message: >
|
||||
There hasn't been any activity on this pull request in the past 4 months, so
|
||||
it has been marked as stale and it will be closed automatically if no
|
||||
further activity occurs in the next 30 days.
|
||||
|
||||
If you want this PR to never become stale, please ask a PSC member to apply
|
||||
the "no stale" label.
|
||||
# Issues settings.
|
||||
# 180+30 day stale policy for open issues
|
||||
# * Except Issues marked as "no stale"
|
||||
days-before-issue-stale: 180
|
||||
days-before-issue-close: 30
|
||||
exempt-issue-labels: "no stale,needs more information"
|
||||
stale-issue-label: "stale"
|
||||
stale-issue-message: >
|
||||
There hasn't been any activity on this issue in the past 6 months, so it has
|
||||
been marked as stale and it will be closed automatically if no further
|
||||
activity occurs in the next 30 days.
|
||||
|
||||
If you want this issue to never become stale, please ask a PSC member to
|
||||
apply the "no stale" label.
|
||||
|
||||
# 15+30 day stale policy for issues pending more information
|
||||
# * Issues that are pending more information
|
||||
# * Except Issues marked as "no stale"
|
||||
- name: Needs more information stale issues policy
|
||||
uses: actions/stale@v4
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
ascending: true
|
||||
only-labels: "needs more information"
|
||||
exempt-issue-labels: "no stale"
|
||||
days-before-stale: 15
|
||||
days-before-close: 30
|
||||
days-before-pr-stale: -1
|
||||
days-before-pr-close: -1
|
||||
remove-stale-when-updated: true
|
||||
stale-issue-label: "stale"
|
||||
stale-issue-message: >
|
||||
This issue needs more information and there hasn't been any activity
|
||||
recently, so it has been marked as stale and it will be closed automatically
|
||||
if no further activity occurs in the next 30 days.
|
||||
|
||||
If you think this is a mistake, please ask a PSC member to remove the "needs
|
||||
more information" label.
|
||||
@@ -0,0 +1,103 @@
|
||||
name: tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- "13.0*"
|
||||
push:
|
||||
branches:
|
||||
- "13.0"
|
||||
- "13.0-ocabot-*"
|
||||
|
||||
jobs:
|
||||
unreleased-deps:
|
||||
runs-on: ubuntu-latest
|
||||
name: Detect unreleased dependencies
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: |
|
||||
for reqfile in requirements.txt test-requirements.txt ; do
|
||||
if [ -f ${reqfile} ] ; then
|
||||
result=0
|
||||
# reject non-comment lines that contain a / (i.e. URLs, relative paths)
|
||||
grep "^[^#].*/" ${reqfile} || result=$?
|
||||
if [ $result -eq 0 ] ; then
|
||||
echo "Unreleased dependencies found in ${reqfile}."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
test:
|
||||
runs-on: ubuntu-20.04
|
||||
container: ${{ matrix.container }}
|
||||
name: ${{ matrix.name }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest
|
||||
include: "attachment_s3,cloud_platform_exoscale"
|
||||
makepot: "true"
|
||||
name: test exoscale S3 with Odoo
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-ocb13.0:latest
|
||||
include: "attachment_s3,cloud_platform_exoscale"
|
||||
name: test exoscale S3 with OCB
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest
|
||||
include: "attachment_swift,cloud_platform_ovh"
|
||||
makepot: "true"
|
||||
name: test OVH with Odoo
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-ocb13.0:latest
|
||||
include: "attachment_swift,cloud_platform_ovh"
|
||||
name: test OVH with OCB
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest
|
||||
include: "attachment_azure,cloud_platform_azure"
|
||||
makepot: "true"
|
||||
name: test azure with Odoo
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-ocb13.0:latest
|
||||
include: "attachment_azure,cloud_platform_azure"
|
||||
name: test fileurl with OCB
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest
|
||||
include: "base_fileurl_field,test_base_fileurl_field"
|
||||
makepot: "true"
|
||||
name: test fileurl with Odoo
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-ocb13.0:latest
|
||||
include: "base_fileurl_field,test_base_fileurl_field"
|
||||
name: test azure with OCB
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest
|
||||
exclude: "attachment_azure,attachment_s3,attachment_swift,cloud_platform_exoscale,cloud_platform_ovh,cloud_platform_azure"
|
||||
makepot: "true"
|
||||
name: test others with Odoo
|
||||
- container: ghcr.io/oca/oca-ci/py3.6-ocb13.0:latest
|
||||
exclude: "attachment_azure,attachment_s3,attachment_swift,cloud_platform_exoscale,cloud_platform_ovh,cloud_platform_azure"
|
||||
name: test others with OCB
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:9.6
|
||||
env:
|
||||
POSTGRES_USER: odoo
|
||||
POSTGRES_PASSWORD: odoo
|
||||
POSTGRES_DB: odoo
|
||||
ports:
|
||||
- 5432:5432
|
||||
env:
|
||||
INCLUDE: "${{ matrix.include }}"
|
||||
EXCLUDE: "${{ matrix.exclude }}"
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install addons and dependencies
|
||||
run: oca_install_addons
|
||||
- name: Check licenses
|
||||
run: manifestoo -d . check-licenses
|
||||
- name: Check development status
|
||||
run: manifestoo -d . check-dev-status --default-dev-status=Beta
|
||||
continue-on-error: true
|
||||
- name: Initialize test db
|
||||
run: oca_init_test_database
|
||||
- name: Run tests
|
||||
run: oca_run_tests
|
||||
- uses: codecov/codecov-action@v1
|
||||
- name: Update .pot files
|
||||
run: oca_export_and_push_pot https://x-access-token:${{ secrets.GIT_PUSH_TOKEN }}@github.com/${{ github.repository }}
|
||||
if: ${{ matrix.makepot == 'true' && github.event_name == 'push' && github.repository_owner == 'camptocamp' }}
|
||||
+20
-3
@@ -1,6 +1,8 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
/.venv
|
||||
/.pytest_cache
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
@@ -13,8 +15,6 @@ build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
@@ -22,6 +22,7 @@ var/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
*.eggs
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
@@ -41,6 +42,19 @@ coverage.xml
|
||||
# Pycharm
|
||||
.idea
|
||||
|
||||
# Eclipse
|
||||
.settings
|
||||
|
||||
# Visual Studio cache/options directory
|
||||
.vs/
|
||||
.vscode
|
||||
|
||||
# OSX Files
|
||||
.DS_Store
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
|
||||
# Mr Developer
|
||||
.mr.developer.cfg
|
||||
.project
|
||||
@@ -50,8 +64,11 @@ coverage.xml
|
||||
.ropeproject
|
||||
|
||||
# Sphinx documentation
|
||||
connector/doc/_build/
|
||||
docs/_build/
|
||||
|
||||
# Backup files
|
||||
*~
|
||||
*.swp
|
||||
|
||||
# OCA rules
|
||||
!static/lib/
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
[settings]
|
||||
; see https://github.com/psf/black
|
||||
multi_line_output=3
|
||||
include_trailing_comma=True
|
||||
force_grid_wrap=0
|
||||
combine_as_imports=True
|
||||
use_parentheses=True
|
||||
line_length=88
|
||||
known_odoo=odoo
|
||||
known_odoo_addons=odoo.addons
|
||||
sections=FUTURE,STDLIB,THIRDPARTY,ODOO,ODOO_ADDONS,FIRSTPARTY,LOCALFOLDER
|
||||
default_section=THIRDPARTY
|
||||
ensure_newline_before_comments = True
|
||||
@@ -0,0 +1,134 @@
|
||||
exclude: |
|
||||
(?x)
|
||||
# NOT INSTALLABLE ADDONS
|
||||
# END NOT INSTALLABLE ADDONS
|
||||
# Files and folders generated by bots, to avoid loops
|
||||
^setup/|/static/description/index\.html$|
|
||||
# We don't want to mess with tool-generated files
|
||||
.svg$|/tests/([^/]+/)?cassettes/|^.copier-answers.yml$|^.github/|
|
||||
# Maybe reactivate this when all README files include prettier ignore tags?
|
||||
^README\.md$|
|
||||
# Library files can have extraneous formatting (even minimized)
|
||||
/static/(src/)?lib/|
|
||||
# Repos using Sphinx to generate docs don't need prettying
|
||||
^docs/_templates/.*\.html$|
|
||||
# You don't usually want a bot to modify your legal texts
|
||||
(LICENSE.*|COPYING.*)
|
||||
default_language_version:
|
||||
python: python3.8
|
||||
node: "14.13.0"
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
# These files are most likely copier diff rejection junks; if found,
|
||||
# review them manually, fix the problem (if needed) and remove them
|
||||
- id: forbidden-files
|
||||
name: forbidden files
|
||||
entry: found forbidden files; remove them
|
||||
language: fail
|
||||
files: "\\.rej$"
|
||||
- id: en-po-files
|
||||
name: en.po files cannot exist
|
||||
entry: found a en.po file
|
||||
language: fail
|
||||
files: '[a-zA-Z0-9_]*/i18n/en\.po$'
|
||||
- repo: https://github.com/oca/maintainer-tools
|
||||
rev: ab1d7f6
|
||||
hooks:
|
||||
# update the NOT INSTALLABLE ADDONS section above
|
||||
- id: oca-update-pre-commit-excluded-addons
|
||||
- id: oca-fix-manifest-website
|
||||
args: ["https://github.com/camptocamp/odoo-cloud-platform"]
|
||||
- repo: https://github.com/myint/autoflake
|
||||
rev: v1.4
|
||||
hooks:
|
||||
- id: autoflake
|
||||
args:
|
||||
- --expand-star-imports
|
||||
- --ignore-init-module-imports
|
||||
- --in-place
|
||||
- --remove-all-unused-imports
|
||||
- --remove-duplicate-keys
|
||||
- --remove-unused-variables
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 19.10b0
|
||||
hooks:
|
||||
- id: black
|
||||
additional_dependencies: ["click<8.1.0"]
|
||||
- repo: https://github.com/pre-commit/mirrors-prettier
|
||||
rev: v1.19.1
|
||||
hooks:
|
||||
- id: prettier
|
||||
name: prettier (with plugin-xml)
|
||||
entry: prettier --write --list-different
|
||||
additional_dependencies:
|
||||
- "prettier@1.19.1"
|
||||
- "@prettier/plugin-xml@0.7.2"
|
||||
files: \.(css|htm|html|js|json|jsx|less|md|scss|toml|ts|xml|yaml|yml)$
|
||||
- repo: https://github.com/pre-commit/mirrors-eslint
|
||||
rev: v6.8.0
|
||||
hooks:
|
||||
- id: eslint
|
||||
verbose: true
|
||||
args:
|
||||
- --color
|
||||
- --fix
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v2.4.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
# exclude autogenerated files
|
||||
exclude: /README\.rst$|\.pot?$
|
||||
- id: end-of-file-fixer
|
||||
# exclude autogenerated files
|
||||
exclude: /README\.rst$|\.pot?$
|
||||
- id: debug-statements
|
||||
- id: fix-encoding-pragma
|
||||
args: ["--remove"]
|
||||
- id: check-case-conflict
|
||||
- id: check-docstring-first
|
||||
- id: check-executables-have-shebangs
|
||||
- id: check-merge-conflict
|
||||
# exclude files where underlines are not distinguishable from merge conflicts
|
||||
exclude: /README\.rst$|^docs/.*\.rst$
|
||||
- id: check-symlinks
|
||||
- id: check-xml
|
||||
- id: mixed-line-ending
|
||||
args: ["--fix=lf"]
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v1.26.2
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: ["--keep-percent-format"]
|
||||
- repo: https://github.com/pre-commit/mirrors-isort
|
||||
rev: v4.3.21
|
||||
hooks:
|
||||
- id: isort
|
||||
name: isort except __init__.py
|
||||
exclude: /__init__\.py$
|
||||
- repo: https://github.com/acsone/setuptools-odoo
|
||||
rev: 3.1.8
|
||||
hooks:
|
||||
- id: setuptools-odoo-make-default
|
||||
- repo: https://github.com/PyCQA/flake8
|
||||
rev: 3.7.9
|
||||
hooks:
|
||||
- id: flake8
|
||||
name: flake8
|
||||
additional_dependencies: ["flake8-bugbear==19.8.0"]
|
||||
- repo: https://github.com/OCA/pylint-odoo
|
||||
rev: v8.0.19
|
||||
hooks:
|
||||
- id: pylint_odoo
|
||||
name: pylint with optional checks
|
||||
args:
|
||||
- --rcfile=.pylintrc
|
||||
- --exit-zero
|
||||
verbose: true
|
||||
additional_dependencies: &pylint_deps
|
||||
- pylint-odoo==8.0.19
|
||||
- id: pylint_odoo
|
||||
name: pylint with mandatory checks
|
||||
args:
|
||||
- --rcfile=.pylintrc-mandatory
|
||||
additional_dependencies: *pylint_deps
|
||||
@@ -0,0 +1,8 @@
|
||||
# Defaults for all prettier-supported languages.
|
||||
# Prettier will complete this with settings from .editorconfig file.
|
||||
bracketSpacing: false
|
||||
printWidth: 88
|
||||
proseWrap: always
|
||||
semi: true
|
||||
trailingComma: "es5"
|
||||
xmlWhitespaceSensitivity: "strict"
|
||||
@@ -0,0 +1,90 @@
|
||||
|
||||
|
||||
[MASTER]
|
||||
load-plugins=pylint_odoo
|
||||
score=n
|
||||
|
||||
[ODOOLINT]
|
||||
readme_template_url="https://github.com/OCA/maintainer-tools/blob/master/template/module/README.rst"
|
||||
manifest_required_authors=Camptocamp
|
||||
manifest_required_keys=license
|
||||
manifest_deprecated_keys=description,active
|
||||
license_allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3
|
||||
valid_odoo_versions=13.0
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
disable=all
|
||||
|
||||
# This .pylintrc contains optional AND mandatory checks and is meant to be
|
||||
# loaded in an IDE to have it check everything, in the hope this will make
|
||||
# optional checks more visible to contributors who otherwise never look at a
|
||||
# green travis to see optional checks that failed.
|
||||
# .pylintrc-mandatory containing only mandatory checks is used the pre-commit
|
||||
# config as a blocking check.
|
||||
|
||||
enable=anomalous-backslash-in-string,
|
||||
api-one-deprecated,
|
||||
api-one-multi-together,
|
||||
assignment-from-none,
|
||||
attribute-deprecated,
|
||||
class-camelcase,
|
||||
dangerous-default-value,
|
||||
dangerous-view-replace-wo-priority,
|
||||
development-status-allowed,
|
||||
duplicate-id-csv,
|
||||
duplicate-key,
|
||||
duplicate-xml-fields,
|
||||
duplicate-xml-record-id,
|
||||
eval-referenced,
|
||||
eval-used,
|
||||
incoherent-interpreter-exec-perm,
|
||||
license-allowed,
|
||||
manifest-author-string,
|
||||
manifest-deprecated-key,
|
||||
manifest-required-author,
|
||||
manifest-required-key,
|
||||
manifest-version-format,
|
||||
method-compute,
|
||||
method-inverse,
|
||||
method-required-super,
|
||||
method-search,
|
||||
openerp-exception-warning,
|
||||
pointless-statement,
|
||||
pointless-string-statement,
|
||||
print-used,
|
||||
redundant-keyword-arg,
|
||||
redundant-modulename-xml,
|
||||
reimported,
|
||||
relative-import,
|
||||
return-in-init,
|
||||
rst-syntax-error,
|
||||
sql-injection,
|
||||
too-few-format-args,
|
||||
translation-field,
|
||||
translation-required,
|
||||
unreachable,
|
||||
use-vim-comment,
|
||||
wrong-tabs-instead-of-spaces,
|
||||
xml-syntax-error,
|
||||
# messages that do not cause the lint step to fail
|
||||
consider-merging-classes-inherited,
|
||||
create-user-wo-reset-password,
|
||||
dangerous-filter-wo-user,
|
||||
deprecated-module,
|
||||
file-not-used,
|
||||
invalid-commit,
|
||||
missing-manifest-dependency,
|
||||
missing-newline-extrafiles,
|
||||
missing-readme,
|
||||
no-utf8-coding-comment,
|
||||
odoo-addons-relative-import,
|
||||
old-api7-method-defined,
|
||||
redefined-builtin,
|
||||
too-complex,
|
||||
unnecessary-utf8-coding-comment
|
||||
|
||||
|
||||
[REPORTS]
|
||||
msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}
|
||||
output-format=colorized
|
||||
reports=no
|
||||
@@ -0,0 +1,65 @@
|
||||
|
||||
[MASTER]
|
||||
load-plugins=pylint_odoo
|
||||
score=n
|
||||
|
||||
[ODOOLINT]
|
||||
readme_template_url="https://github.com/OCA/maintainer-tools/blob/master/template/module/README.rst"
|
||||
manifest_required_authors=Camptocamp
|
||||
manifest_required_keys=license
|
||||
manifest_deprecated_keys=description,active
|
||||
license_allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3
|
||||
valid_odoo_versions=13.0
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
disable=all
|
||||
|
||||
enable=anomalous-backslash-in-string,
|
||||
api-one-deprecated,
|
||||
api-one-multi-together,
|
||||
assignment-from-none,
|
||||
attribute-deprecated,
|
||||
class-camelcase,
|
||||
dangerous-default-value,
|
||||
dangerous-view-replace-wo-priority,
|
||||
development-status-allowed,
|
||||
duplicate-id-csv,
|
||||
duplicate-key,
|
||||
duplicate-xml-fields,
|
||||
duplicate-xml-record-id,
|
||||
eval-referenced,
|
||||
eval-used,
|
||||
incoherent-interpreter-exec-perm,
|
||||
license-allowed,
|
||||
manifest-author-string,
|
||||
manifest-deprecated-key,
|
||||
manifest-required-author,
|
||||
manifest-required-key,
|
||||
manifest-version-format,
|
||||
method-compute,
|
||||
method-inverse,
|
||||
method-required-super,
|
||||
method-search,
|
||||
openerp-exception-warning,
|
||||
pointless-statement,
|
||||
pointless-string-statement,
|
||||
print-used,
|
||||
redundant-keyword-arg,
|
||||
redundant-modulename-xml,
|
||||
reimported,
|
||||
relative-import,
|
||||
return-in-init,
|
||||
rst-syntax-error,
|
||||
sql-injection,
|
||||
too-few-format-args,
|
||||
translation-field,
|
||||
translation-required,
|
||||
unreachable,
|
||||
use-vim-comment,
|
||||
wrong-tabs-instead-of-spaces,
|
||||
xml-syntax-error
|
||||
|
||||
[REPORTS]
|
||||
msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}
|
||||
output-format=colorized
|
||||
reports=no
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
language: python
|
||||
sudo: false
|
||||
cache: pip
|
||||
|
||||
branches:
|
||||
only:
|
||||
- "/^[[:digit:]]{1,2}.[[:digit:]]$/"
|
||||
|
||||
python:
|
||||
- "3.6"
|
||||
|
||||
addons:
|
||||
postgresql: "9.5"
|
||||
apt:
|
||||
packages:
|
||||
- expect-dev # provides unbuffer utility
|
||||
- python-lxml # because pip installation is slow
|
||||
- python-simplejson
|
||||
- python-serial
|
||||
|
||||
env:
|
||||
matrix:
|
||||
- LINT_CHECK="1"
|
||||
- TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_exoscale"
|
||||
- TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_exoscale"
|
||||
- TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_ovh"
|
||||
- TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_ovh"
|
||||
- TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale"
|
||||
- TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale"
|
||||
global:
|
||||
- VERSION="13.0" LINT_CHECK="0" TESTS="0"
|
||||
|
||||
install:
|
||||
- git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools
|
||||
- export PATH=${HOME}/maintainer-quality-tools/travis:${PATH}
|
||||
- travis_install_nightly
|
||||
|
||||
script:
|
||||
- travis_run_tests
|
||||
|
||||
after_success:
|
||||
- travis_after_test_success
|
||||
@@ -1,7 +1,7 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
@@ -633,8 +633,8 @@ the "copyright" line and a pointer to where the full notice is found.
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published
|
||||
by the Free Software Foundation, either version 3 of the License, or
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
@@ -643,7 +643,7 @@ the "copyright" line and a pointer to where the full notice is found.
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
@@ -658,4 +658,4 @@ specific requirements.
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<http://www.gnu.org/licenses/>.
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
[](https://travis-ci.com/camptocamp/odoo-cloud-platform)
|
||||
|
||||
<!-- /!\ Non OCA Context : Set here the badge of your runbot / runboat instance. -->
|
||||
[](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/pre-commit.yml?query=branch%3A13.0)
|
||||
[](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/test.yml?query=branch%3A13.0)
|
||||
[](https://codecov.io/gh/camptocamp/odoo-cloud-platform)
|
||||
<!-- /!\ Non OCA Context : Set here the badge of your translation instance. -->
|
||||
|
||||
<!-- /!\ do not modify above this line -->
|
||||
|
||||
# Odoo Cloud Addons
|
||||
|
||||
@@ -162,3 +169,26 @@ environment. It will refuse to start if anything is badly configured.
|
||||
|
||||
The checks can be bypassed with the environment variable
|
||||
`ODOO_CLOUD_PLATFORM_UNSAFE` set to `1`.
|
||||
|
||||
<!-- /!\ do not modify below this line -->
|
||||
|
||||
<!-- prettier-ignore-start -->
|
||||
|
||||
[//]: # (addons)
|
||||
|
||||
This part will be replaced when running the oca-gen-addons-table script from OCA/maintainer-tools.
|
||||
|
||||
[//]: # (end addons)
|
||||
|
||||
<!-- prettier-ignore-end -->
|
||||
|
||||
## Licenses
|
||||
|
||||
This repository is licensed under [AGPL-3.0](LICENSE).
|
||||
|
||||
However, each module can have a totally different license, as long as they adhere to Camptocamp
|
||||
policy. Consult each module's `__manifest__.py` file, which contains a `license` key
|
||||
that explains its license.
|
||||
|
||||
----
|
||||
<!-- /!\ Non OCA Context : Set here the full description of your organization. -->
|
||||
|
||||
@@ -9,13 +9,11 @@
|
||||
"Open Source Integrators, "
|
||||
"Serpent Consulting Services, "
|
||||
"Odoo Community Association (OCA)",
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"license": "AGPL-3",
|
||||
"category": "Knowledge Management",
|
||||
"depends": ["base_attachment_object_storage"],
|
||||
"external_dependencies": {
|
||||
"python": ["azure-storage-blob", "azure-identity"],
|
||||
},
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"external_dependencies": {"python": ["azure-storage-blob", "azure-identity"]},
|
||||
"installable": True,
|
||||
"development_status": "Beta",
|
||||
"maintainers": ["max3903"],
|
||||
|
||||
@@ -33,9 +33,9 @@ class IrAttachment(models.Model):
|
||||
_inherit = "ir.attachment"
|
||||
|
||||
def _get_stores(self):
|
||||
l = ["azure"]
|
||||
l += super(IrAttachment, self)._get_stores()
|
||||
return l
|
||||
stores = ["azure"]
|
||||
stores += super()._get_stores()
|
||||
return stores
|
||||
|
||||
@api.model
|
||||
def _get_blob_service_client(self):
|
||||
@@ -100,8 +100,7 @@ class IrAttachment(models.Model):
|
||||
expiry=datetime.utcnow() + timedelta(hours=1),
|
||||
)
|
||||
blob_service_client = BlobServiceClient(
|
||||
account_url=account_url,
|
||||
credential=sas_token,
|
||||
account_url=account_url, credential=sas_token,
|
||||
)
|
||||
except HttpResponseError as error:
|
||||
_logger.exception(
|
||||
@@ -113,16 +112,11 @@ class IrAttachment(models.Model):
|
||||
|
||||
@api.model
|
||||
def _get_container_name(self):
|
||||
"""
|
||||
Container naming rules:
|
||||
https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
|
||||
"""
|
||||
# Container naming rules:
|
||||
# https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names # noqa: B950
|
||||
running_env = os.environ.get("RUNNING_ENV", "dev")
|
||||
storage_name = os.environ.get('AZURE_STORAGE_NAME', r'{env}-{db}')
|
||||
storage_name = storage_name.format(
|
||||
env=running_env,
|
||||
db=self.env.cr.dbname
|
||||
)
|
||||
storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}")
|
||||
storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname)
|
||||
# replace invalid characters by _
|
||||
storage_name = re.sub(r"[\W_]+", "-", storage_name)
|
||||
# lowercase, max 63 chars
|
||||
@@ -137,7 +131,7 @@ class IrAttachment(models.Model):
|
||||
except exceptions.UserError:
|
||||
_logger.exception(
|
||||
"error accessing to storage '%s' please check credentials ",
|
||||
container_name
|
||||
container_name,
|
||||
)
|
||||
return False
|
||||
container_client = blob_service_client.get_container_client(container_name)
|
||||
@@ -154,14 +148,14 @@ class IrAttachment(models.Model):
|
||||
def _store_file_read(self, fname, bin_size=False):
|
||||
if fname.startswith("azure://"):
|
||||
key = fname.replace("azure://", "", 1).lower()
|
||||
if '/' in key:
|
||||
container_name, key = key.split('/', 1)
|
||||
if "/" in key:
|
||||
container_name, key = key.split("/", 1)
|
||||
else:
|
||||
container_name = None
|
||||
container_client = self._get_azure_container(container_name)
|
||||
# if container cannot be retrived, abort reading from azure storage
|
||||
if not container_client:
|
||||
return ''
|
||||
return ""
|
||||
try:
|
||||
blob_client = container_client.get_blob_client(key)
|
||||
read = base64.b64encode(blob_client.download_blob().readall())
|
||||
@@ -201,13 +195,13 @@ class IrAttachment(models.Model):
|
||||
def _store_file_delete(self, fname):
|
||||
if fname.startswith("azure://"):
|
||||
key = fname.replace("azure://", "", 1).lower()
|
||||
if '/' in key:
|
||||
container_name, key = key.split('/', 1)
|
||||
if "/" in key:
|
||||
container_name, key = key.split("/", 1)
|
||||
else:
|
||||
container_name = None
|
||||
container_client = self._get_azure_container(container_name)
|
||||
if not container_client:
|
||||
return ''
|
||||
return ""
|
||||
# delete the file only if it is on the current configured container
|
||||
# otherwise, we might delete files used on a different environment
|
||||
try:
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import models
|
||||
|
||||
@@ -2,17 +2,16 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Attachments on S3 storage',
|
||||
'summary': 'Store assets and attachments on a S3 compatible object storage',
|
||||
'version': '13.0.1.0.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'Knowledge Management',
|
||||
'depends': ['base', 'base_attachment_object_storage'],
|
||||
'external_dependencies': {
|
||||
'python': ['boto3'],
|
||||
},
|
||||
'website': 'https://www.camptocamp.com',
|
||||
'data': [],
|
||||
'installable': True,
|
||||
}
|
||||
{
|
||||
"name": "Attachments on S3 storage",
|
||||
"summary": "Store assets and attachments on a S3 compatible object storage",
|
||||
"version": "13.0.1.0.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Knowledge Management",
|
||||
"depends": ["base", "base_attachment_object_storage"],
|
||||
"external_dependencies": {"python": ["boto3"]},
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from contextlib import closing
|
||||
|
||||
import odoo
|
||||
@@ -14,32 +13,38 @@ _logger = logging.getLogger(__name__)
|
||||
def migrate(cr, version):
|
||||
if not version:
|
||||
return
|
||||
cr.execute("""
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT value FROM ir_config_parameter
|
||||
WHERE key = 'ir_attachment.location'
|
||||
""")
|
||||
"""
|
||||
)
|
||||
row = cr.fetchone()
|
||||
bucket = os.environ.get('AWS_BUCKETNAME')
|
||||
bucket = os.environ.get("AWS_BUCKETNAME")
|
||||
|
||||
if row[0] == 's3' and bucket:
|
||||
if row[0] == "s3" and bucket:
|
||||
uid = odoo.SUPERUSER_ID
|
||||
registry = odoo.modules.registry.Registry(cr.dbname)
|
||||
new_cr = registry.cursor()
|
||||
with closing(new_cr):
|
||||
with odoo.api.Environment.manage():
|
||||
env = odoo.api.Environment(new_cr, uid, {})
|
||||
store_local = env['ir.attachment'].search(
|
||||
[('store_fname', '=like', 's3://%'),
|
||||
'|', ('res_model', '=', 'ir.ui.view'),
|
||||
('res_field', 'in', ['image_small',
|
||||
'image_medium',
|
||||
'web_icon_data'])
|
||||
store_local = env["ir.attachment"].search(
|
||||
[
|
||||
("store_fname", "=like", "s3://%"),
|
||||
"|",
|
||||
("res_model", "=", "ir.ui.view"),
|
||||
(
|
||||
"res_field",
|
||||
"in",
|
||||
["image_small", "image_medium", "web_icon_data"],
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
_logger.info(
|
||||
'Moving %d attachments from S3 to DB for fast access',
|
||||
len(store_local)
|
||||
"Moving %d attachments from S3 to DB for fast access",
|
||||
len(store_local),
|
||||
)
|
||||
for attachment_id in store_local.ids:
|
||||
# force re-storing the document, will move
|
||||
@@ -52,10 +57,13 @@ def migrate(cr, version):
|
||||
# attachments on each loop.
|
||||
try:
|
||||
env.clear()
|
||||
attachment = env['ir.attachment'].browse(attachment_id)
|
||||
_logger.info('Moving attachment %s (id: %s)',
|
||||
attachment.name, attachment.id)
|
||||
attachment.write({'datas': attachment.datas})
|
||||
attachment = env["ir.attachment"].browse(attachment_id)
|
||||
_logger.info(
|
||||
"Moving attachment %s (id: %s)",
|
||||
attachment.name,
|
||||
attachment.id,
|
||||
)
|
||||
attachment.write({"datas": attachment.datas})
|
||||
new_cr.commit()
|
||||
except:
|
||||
except Exception:
|
||||
new_cr.rollback()
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import ir_attachment
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import io
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from odoo import _, api, exceptions, models
|
||||
|
||||
from ..s3uri import S3Uri
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
@@ -27,9 +28,9 @@ class IrAttachment(models.Model):
|
||||
_inherit = "ir.attachment"
|
||||
|
||||
def _get_stores(self):
|
||||
l = ['s3']
|
||||
l += super()._get_stores()
|
||||
return l
|
||||
stores = ["s3"]
|
||||
stores += super()._get_stores()
|
||||
return stores
|
||||
|
||||
@api.model
|
||||
def _get_s3_bucket(self, name=None):
|
||||
@@ -46,42 +47,43 @@ class IrAttachment(models.Model):
|
||||
from the environment variable ``AWS_BUCKETNAME`` will be read.
|
||||
|
||||
"""
|
||||
host = os.environ.get('AWS_HOST')
|
||||
host = os.environ.get("AWS_HOST")
|
||||
|
||||
# Ensure host is prefixed with a scheme (use https as default)
|
||||
if host and not urlsplit(host).scheme:
|
||||
host = 'https://%s' % host
|
||||
host = "https://%s" % host
|
||||
|
||||
region_name = os.environ.get('AWS_REGION')
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
bucket_name = name or os.environ.get('AWS_BUCKETNAME')
|
||||
region_name = os.environ.get("AWS_REGION")
|
||||
access_key = os.environ.get("AWS_ACCESS_KEY_ID")
|
||||
secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
|
||||
bucket_name = name or os.environ.get("AWS_BUCKETNAME")
|
||||
# replaces {db} by the database name to handle multi-tenancy
|
||||
bucket_name = bucket_name.format(db=self.env.cr.dbname)
|
||||
|
||||
params = {
|
||||
'aws_access_key_id': access_key,
|
||||
'aws_secret_access_key': secret_key,
|
||||
"aws_access_key_id": access_key,
|
||||
"aws_secret_access_key": secret_key,
|
||||
}
|
||||
if host:
|
||||
params['endpoint_url'] = host
|
||||
params["endpoint_url"] = host
|
||||
if region_name:
|
||||
params['region_name'] = region_name
|
||||
params["region_name"] = region_name
|
||||
if not (access_key and secret_key and bucket_name):
|
||||
msg = _('If you want to read from the %s S3 bucket, the following '
|
||||
'environment variables must be set:\n'
|
||||
'* AWS_ACCESS_KEY_ID\n'
|
||||
'* AWS_SECRET_ACCESS_KEY\n'
|
||||
'If you want to write in the %s S3 bucket, this variable '
|
||||
'must be set as well:\n'
|
||||
'* AWS_BUCKETNAME\n'
|
||||
'Optionally, the S3 host can be changed with:\n'
|
||||
'* AWS_HOST\n'
|
||||
msg = _(
|
||||
"If you want to read from the %s S3 bucket, the following "
|
||||
"environment variables must be set:\n"
|
||||
"* AWS_ACCESS_KEY_ID\n"
|
||||
"* AWS_SECRET_ACCESS_KEY\n"
|
||||
"If you want to write in the %s S3 bucket, this variable "
|
||||
"must be set as well:\n"
|
||||
"* AWS_BUCKETNAME\n"
|
||||
"Optionally, the S3 host can be changed with:\n"
|
||||
"* AWS_HOST\n"
|
||||
) % (bucket_name, bucket_name)
|
||||
|
||||
raise exceptions.UserError(msg)
|
||||
# try:
|
||||
s3 = boto3.resource('s3', **params)
|
||||
s3 = boto3.resource("s3", **params)
|
||||
bucket = s3.Bucket(bucket_name)
|
||||
exists = True
|
||||
try:
|
||||
@@ -89,12 +91,12 @@ class IrAttachment(models.Model):
|
||||
except ClientError as e:
|
||||
# If a client error is thrown, then check that it was a 404 error.
|
||||
# If it was a 404 error, then the bucket does not exist.
|
||||
error_code = e.response['Error']['Code']
|
||||
if error_code == '404':
|
||||
error_code = e.response["Error"]["Code"]
|
||||
if error_code == "404":
|
||||
exists = False
|
||||
except EndpointConnectionError as error:
|
||||
# log verbose error from s3, return short message for user
|
||||
_logger.exception('Error during connection on S3')
|
||||
_logger.exception("Error during connection on S3")
|
||||
raise exceptions.UserError(str(error))
|
||||
|
||||
if not exists:
|
||||
@@ -103,14 +105,13 @@ class IrAttachment(models.Model):
|
||||
else:
|
||||
bucket = s3.create_bucket(
|
||||
Bucket=bucket_name,
|
||||
CreateBucketConfiguration={
|
||||
'LocationConstraint': region_name
|
||||
})
|
||||
CreateBucketConfiguration={"LocationConstraint": region_name},
|
||||
)
|
||||
return bucket
|
||||
|
||||
@api.model
|
||||
def _store_file_read(self, fname, bin_size=False):
|
||||
if fname.startswith('s3://'):
|
||||
if fname.startswith("s3://"):
|
||||
s3uri = S3Uri(fname)
|
||||
try:
|
||||
bucket = self._get_s3_bucket(name=s3uri.bucket())
|
||||
@@ -118,12 +119,10 @@ class IrAttachment(models.Model):
|
||||
_logger.exception(
|
||||
"error reading attachment '%s' from object storage", fname
|
||||
)
|
||||
return ''
|
||||
return ""
|
||||
try:
|
||||
key = s3uri.item()
|
||||
bucket.meta.client.head_object(
|
||||
Bucket=bucket.name, Key=key
|
||||
)
|
||||
bucket.meta.client.head_object(Bucket=bucket.name, Key=key)
|
||||
if bin_size:
|
||||
return bucket.Object(key).content_length
|
||||
with io.BytesIO() as res:
|
||||
@@ -131,33 +130,29 @@ class IrAttachment(models.Model):
|
||||
res.seek(0)
|
||||
read = base64.b64encode(res.read())
|
||||
except ClientError:
|
||||
read = ''
|
||||
_logger.info(
|
||||
"attachment '%s' missing on object storage", fname
|
||||
)
|
||||
read = ""
|
||||
_logger.info("attachment '%s' missing on object storage", fname)
|
||||
return read
|
||||
else:
|
||||
return super()._store_file_read(fname, bin_size)
|
||||
|
||||
@api.model
|
||||
def _store_file_write(self, key, bin_data):
|
||||
location = self.env.context.get('storage_location') or self._storage()
|
||||
if location == 's3':
|
||||
location = self.env.context.get("storage_location") or self._storage()
|
||||
if location == "s3":
|
||||
bucket = self._get_s3_bucket()
|
||||
obj = bucket.Object(key=key)
|
||||
with io.BytesIO() as file:
|
||||
file.write(bin_data)
|
||||
file.seek(0)
|
||||
filename = 's3://%s/%s' % (bucket.name, key)
|
||||
filename = "s3://%s/%s" % (bucket.name, key)
|
||||
try:
|
||||
obj.upload_fileobj(file)
|
||||
except ClientError as error:
|
||||
# log verbose error from s3, return short message for user
|
||||
_logger.exception(
|
||||
'Error during storage of the file %s' % filename
|
||||
)
|
||||
_logger.exception("Error during storage of the file %s" % filename)
|
||||
raise exceptions.UserError(
|
||||
_('The file could not be stored: %s') % str(error)
|
||||
_("The file could not be stored: %s") % str(error)
|
||||
)
|
||||
else:
|
||||
_super = super()
|
||||
@@ -166,28 +161,22 @@ class IrAttachment(models.Model):
|
||||
|
||||
@api.model
|
||||
def _store_file_delete(self, fname):
|
||||
if fname.startswith('s3://'):
|
||||
if fname.startswith("s3://"):
|
||||
s3uri = S3Uri(fname)
|
||||
bucket_name = s3uri.bucket()
|
||||
item_name = s3uri.item()
|
||||
# delete the file only if it is on the current configured bucket
|
||||
# otherwise, we might delete files used on a different environment
|
||||
if bucket_name == os.environ.get('AWS_BUCKETNAME'):
|
||||
if bucket_name == os.environ.get("AWS_BUCKETNAME"):
|
||||
bucket = self._get_s3_bucket()
|
||||
obj = bucket.Object(key=item_name)
|
||||
try:
|
||||
bucket.meta.client.head_object(
|
||||
Bucket=bucket.name, Key=item_name
|
||||
)
|
||||
bucket.meta.client.head_object(Bucket=bucket.name, Key=item_name)
|
||||
obj.delete()
|
||||
_logger.info(
|
||||
'file %s deleted on the object storage' % (fname,)
|
||||
)
|
||||
_logger.info("file %s deleted on the object storage" % (fname,))
|
||||
except ClientError:
|
||||
# log verbose error from s3, return short message for
|
||||
# user
|
||||
_logger.exception(
|
||||
'Error during deletion of the file %s' % fname
|
||||
)
|
||||
_logger.exception("Error during deletion of the file %s" % fname)
|
||||
else:
|
||||
super()._store_file_delete(fname)
|
||||
|
||||
@@ -2,20 +2,18 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Attachments on Swift storage',
|
||||
'summary': 'Store assets and attachments on a Swift compatible object store',
|
||||
'version': '13.0.1.0.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'Knowledge Management',
|
||||
'depends': ['base_attachment_object_storage'],
|
||||
'external_dependencies': {
|
||||
'python': ['swiftclient',
|
||||
'keystoneclient',
|
||||
'keystoneauth1',
|
||||
],
|
||||
{
|
||||
"name": "Attachments on Swift storage",
|
||||
"summary": "Store assets and attachments on a Swift compatible object store",
|
||||
"version": "13.0.1.0.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Knowledge Management",
|
||||
"depends": ["base_attachment_object_storage"],
|
||||
"external_dependencies": {
|
||||
"python": ["swiftclient", "keystoneclient", "keystoneauth1"],
|
||||
},
|
||||
'website': 'https://www.camptocamp.com',
|
||||
'data': [],
|
||||
'installable': True,
|
||||
}
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
from ..swift_uri import SwiftUri
|
||||
|
||||
from odoo import api, exceptions, models, _
|
||||
from odoo import _, api, exceptions, models
|
||||
|
||||
from ..swift_uri import SwiftUri
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -49,8 +50,9 @@ class SwiftSessionStore(object):
|
||||
def _get_key(self, auth_url, username, password, project_name):
|
||||
return (auth_url, username, password, project_name)
|
||||
|
||||
def get_session(self, auth_url=None, username=None, password=None,
|
||||
project_name=None):
|
||||
def get_session(
|
||||
self, auth_url=None, username=None, password=None, project_name=None
|
||||
):
|
||||
key = self._get_key(auth_url, username, password, project_name)
|
||||
session = self._sessions.get(key)
|
||||
if not session:
|
||||
@@ -59,13 +61,10 @@ class SwiftSessionStore(object):
|
||||
password=password,
|
||||
project_name=project_name,
|
||||
auth_url=auth_url,
|
||||
project_domain_id='default',
|
||||
user_domain_id='default',
|
||||
)
|
||||
session = keystoneauth1.session.Session(
|
||||
auth=auth,
|
||||
timeout=SWIFT_TIMEOUT,
|
||||
project_domain_id="default",
|
||||
user_domain_id="default",
|
||||
)
|
||||
session = keystoneauth1.session.Session(auth=auth, timeout=SWIFT_TIMEOUT,)
|
||||
self._sessions[key] = session
|
||||
return session
|
||||
|
||||
@@ -74,36 +73,38 @@ swift_session_store = SwiftSessionStore()
|
||||
|
||||
|
||||
class IrAttachment(models.Model):
|
||||
_inherit = 'ir.attachment'
|
||||
_inherit = "ir.attachment"
|
||||
|
||||
def _get_stores(self):
|
||||
l = ['swift']
|
||||
l += super()._get_stores()
|
||||
return l
|
||||
stores = ["swift"]
|
||||
stores += super()._get_stores()
|
||||
return stores
|
||||
|
||||
@api.model
|
||||
def _get_swift_connection(self):
|
||||
""" Returns a connection object for the Swift object store """
|
||||
host = os.environ.get('SWIFT_AUTH_URL')
|
||||
account = os.environ.get('SWIFT_ACCOUNT')
|
||||
password = os.environ.get('SWIFT_PASSWORD')
|
||||
project_name = os.environ.get('SWIFT_PROJECT_NAME')
|
||||
if not project_name and os.environ.get('SWIFT_TENANT_NAME'):
|
||||
project_name = os.environ['SWIFT_TENANT_NAME']
|
||||
host = os.environ.get("SWIFT_AUTH_URL")
|
||||
account = os.environ.get("SWIFT_ACCOUNT")
|
||||
password = os.environ.get("SWIFT_PASSWORD")
|
||||
project_name = os.environ.get("SWIFT_PROJECT_NAME")
|
||||
if not project_name and os.environ.get("SWIFT_TENANT_NAME"):
|
||||
project_name = os.environ["SWIFT_TENANT_NAME"]
|
||||
_logger.warning(
|
||||
"SWIFT_TENANT_NAME is deprecated and "
|
||||
"must be replaced by SWIFT_PROJECT_NAME"
|
||||
)
|
||||
region = os.environ.get('SWIFT_REGION_NAME')
|
||||
region = os.environ.get("SWIFT_REGION_NAME")
|
||||
os_options = {}
|
||||
if region:
|
||||
os_options['region_name'] = region
|
||||
os_options["region_name"] = region
|
||||
if not (host and account and password and project_name):
|
||||
raise exceptions.UserError(_(
|
||||
raise exceptions.UserError(
|
||||
_(
|
||||
"Problem connecting to Swift store, are the env variables "
|
||||
"(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, "
|
||||
"SWIFT_TENANT_NAME) properly set?"
|
||||
))
|
||||
)
|
||||
)
|
||||
try:
|
||||
session = swift_session_store.get_session(
|
||||
username=account,
|
||||
@@ -112,17 +113,16 @@ class IrAttachment(models.Model):
|
||||
auth_url=host,
|
||||
)
|
||||
conn = swiftclient.client.Connection(
|
||||
session=session,
|
||||
os_options=os_options,
|
||||
session=session, os_options=os_options,
|
||||
)
|
||||
except ClientException:
|
||||
_logger.exception('Error connecting to Swift object store')
|
||||
raise exceptions.UserError(_('Error on Swift connection'))
|
||||
_logger.exception("Error connecting to Swift object store")
|
||||
raise exceptions.UserError(_("Error on Swift connection"))
|
||||
return conn
|
||||
|
||||
@api.model
|
||||
def _store_file_read(self, fname, bin_size=False):
|
||||
if fname.startswith('swift://'):
|
||||
if fname.startswith("swift://"):
|
||||
swifturi = SwiftUri(fname)
|
||||
try:
|
||||
conn = self._get_swift_connection()
|
||||
@@ -130,32 +130,32 @@ class IrAttachment(models.Model):
|
||||
_logger.exception(
|
||||
"error reading attachment '%s' from object storage", fname
|
||||
)
|
||||
return ''
|
||||
return ""
|
||||
try:
|
||||
resp, obj_content = conn.get_object(swifturi.container(),
|
||||
swifturi.item())
|
||||
resp, obj_content = conn.get_object(
|
||||
swifturi.container(), swifturi.item()
|
||||
)
|
||||
read = base64.b64encode(obj_content)
|
||||
except ClientException:
|
||||
read = ''
|
||||
_logger.exception(
|
||||
'Error reading object from Swift object store')
|
||||
read = ""
|
||||
_logger.exception("Error reading object from Swift object store")
|
||||
return read
|
||||
else:
|
||||
return super()._store_file_read(fname, bin_size)
|
||||
|
||||
def _store_file_write(self, key, bin_data):
|
||||
if self._storage() == 'swift':
|
||||
container = os.environ.get('SWIFT_WRITE_CONTAINER')
|
||||
if self._storage() == "swift":
|
||||
container = os.environ.get("SWIFT_WRITE_CONTAINER")
|
||||
# replace {db} with the database name to handle multi-tenancy
|
||||
container = container.format(db=self.env.cr.dbname)
|
||||
conn = self._get_swift_connection()
|
||||
conn.put_container(container)
|
||||
filename = 'swift://{}/{}'.format(container, key)
|
||||
filename = "swift://{}/{}".format(container, key)
|
||||
try:
|
||||
conn.put_object(container, key, bin_data)
|
||||
except ClientException:
|
||||
_logger.exception('Error writing to Swift object store')
|
||||
raise exceptions.UserError(_('Error writing to Swift'))
|
||||
_logger.exception("Error writing to Swift object store")
|
||||
raise exceptions.UserError(_("Error writing to Swift"))
|
||||
else:
|
||||
_super = super()
|
||||
filename = _super._store_file_write(key, bin_data)
|
||||
@@ -163,18 +163,17 @@ class IrAttachment(models.Model):
|
||||
|
||||
@api.model
|
||||
def _store_file_delete(self, fname):
|
||||
if fname.startswith('swift://'):
|
||||
if fname.startswith("swift://"):
|
||||
swifturi = SwiftUri(fname)
|
||||
container = swifturi.container()
|
||||
# delete the file only if it is on the current configured bucket
|
||||
# otherwise, we might delete files used on a different environment
|
||||
if container == os.environ.get('SWIFT_WRITE_CONTAINER'):
|
||||
if container == os.environ.get("SWIFT_WRITE_CONTAINER"):
|
||||
conn = self._get_swift_connection()
|
||||
try:
|
||||
conn.delete_object(container, swifturi.item())
|
||||
except ClientException:
|
||||
_logger.exception(
|
||||
_('Error deleting an object on the Swift store'))
|
||||
_logger.exception(_("Error deleting an object on the Swift store"))
|
||||
# we ignore the error, file will stay on the object
|
||||
# storage but won't disrupt the process
|
||||
else:
|
||||
|
||||
@@ -6,8 +6,7 @@ import re
|
||||
|
||||
class SwiftUri(object):
|
||||
|
||||
_url_re = re.compile("^swift:///*([^/]*)/?(.*)",
|
||||
re.IGNORECASE | re.UNICODE)
|
||||
_url_re = re.compile("^swift:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE)
|
||||
|
||||
def __init__(self, uri):
|
||||
match = self._url_re.match(uri)
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import test_mock_swift_api
|
||||
|
||||
@@ -2,30 +2,27 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
import base64
|
||||
import mock
|
||||
import os
|
||||
|
||||
import keystoneauth1
|
||||
import mock
|
||||
from mock import patch
|
||||
|
||||
import keystoneauth1
|
||||
|
||||
from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
|
||||
from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore
|
||||
from odoo.addons.attachment_swift.swift_uri import SwiftUri
|
||||
from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
|
||||
|
||||
|
||||
class TestAttachmentSwift(TestIrAttachment):
|
||||
|
||||
def setup(self):
|
||||
super().setUp()
|
||||
self.env['ir.config_parameter'].set_param('ir_attachment.location',
|
||||
'swift')
|
||||
self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift")
|
||||
|
||||
def test_session_store_get_session(self):
|
||||
auth_url = 'auth_url'
|
||||
username = 'username'
|
||||
password = 'password'
|
||||
project_name = 'project_name'
|
||||
auth_url = "auth_url"
|
||||
username = "username"
|
||||
password = "password"
|
||||
project_name = "project_name"
|
||||
store = SwiftSessionStore()
|
||||
session = store.get_session(
|
||||
auth_url=auth_url,
|
||||
@@ -34,10 +31,12 @@ class TestAttachmentSwift(TestIrAttachment):
|
||||
project_name=project_name,
|
||||
)
|
||||
self.assertEqual(session.auth.auth_url, auth_url)
|
||||
self.assertEqual(session.auth.get_cache_id_elements().get(
|
||||
'password_username'), username)
|
||||
self.assertEqual(session.auth.get_cache_id_elements().get(
|
||||
'password_password'), password)
|
||||
self.assertEqual(
|
||||
session.auth.get_cache_id_elements().get("password_username"), username
|
||||
)
|
||||
self.assertEqual(
|
||||
session.auth.get_cache_id_elements().get("password_password"), password
|
||||
)
|
||||
self.assertEqual(session.auth.project_name, project_name)
|
||||
|
||||
# get the same session on a second call
|
||||
@@ -48,73 +47,73 @@ class TestAttachmentSwift(TestIrAttachment):
|
||||
password=password,
|
||||
project_name=project_name,
|
||||
),
|
||||
session
|
||||
session,
|
||||
)
|
||||
|
||||
@patch('swiftclient.client')
|
||||
@patch("swiftclient.client")
|
||||
def test_connection(self, mock_swift_client):
|
||||
""" Test the connection to the store"""
|
||||
os.environ['SWIFT_AUTH_URL'] = 'auth_url'
|
||||
os.environ['SWIFT_ACCOUNT'] = 'account'
|
||||
os.environ['SWIFT_PASSWORD'] = 'password'
|
||||
os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
|
||||
os.environ['SWIFT_REGION_NAME'] = 'NOWHERE'
|
||||
os.environ["SWIFT_AUTH_URL"] = "auth_url"
|
||||
os.environ["SWIFT_ACCOUNT"] = "account"
|
||||
os.environ["SWIFT_PASSWORD"] = "password"
|
||||
os.environ["SWIFT_PROJECT_NAME"] = "project_name"
|
||||
os.environ["SWIFT_REGION_NAME"] = "NOWHERE"
|
||||
attachment = self.Attachment
|
||||
attachment._get_swift_connection()
|
||||
mock_swift_client.Connection.assert_called_once_with(
|
||||
session=mock.ANY,
|
||||
os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')},
|
||||
os_options={"region_name": os.environ.get("SWIFT_REGION_NAME")},
|
||||
)
|
||||
__, kwargs = mock_swift_client.Connection.call_args
|
||||
session = kwargs['session']
|
||||
session = kwargs["session"]
|
||||
self.assertTrue(isinstance(session, keystoneauth1.session.Session))
|
||||
self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL'])
|
||||
self.assertEqual(session.auth.get_cache_id_elements().get(
|
||||
'password_username'), os.environ['SWIFT_ACCOUNT'])
|
||||
self.assertEqual(session.auth.get_cache_id_elements().get(
|
||||
'password_password'), os.environ['SWIFT_PASSWORD'])
|
||||
self.assertEqual(session.auth.project_name,
|
||||
os.environ['SWIFT_PROJECT_NAME'])
|
||||
self.assertEqual(session.auth.auth_url, os.environ["SWIFT_AUTH_URL"])
|
||||
self.assertEqual(
|
||||
session.auth.get_cache_id_elements().get("password_username"),
|
||||
os.environ["SWIFT_ACCOUNT"],
|
||||
)
|
||||
self.assertEqual(
|
||||
session.auth.get_cache_id_elements().get("password_password"),
|
||||
os.environ["SWIFT_PASSWORD"],
|
||||
)
|
||||
self.assertEqual(session.auth.project_name, os.environ["SWIFT_PROJECT_NAME"])
|
||||
|
||||
def test_store_file_on_swift(self):
|
||||
"""
|
||||
Test writing a file
|
||||
"""
|
||||
(self.env['ir.config_parameter'].
|
||||
set_param('ir_attachment.location', 'swift'))
|
||||
os.environ['SWIFT_AUTH_URL'] = 'auth_url'
|
||||
os.environ['SWIFT_ACCOUNT'] = 'account'
|
||||
os.environ['SWIFT_PASSWORD'] = 'password'
|
||||
os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
|
||||
os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
|
||||
container = os.environ.get('SWIFT_WRITE_CONTAINER')
|
||||
(self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift"))
|
||||
os.environ["SWIFT_AUTH_URL"] = "auth_url"
|
||||
os.environ["SWIFT_ACCOUNT"] = "account"
|
||||
os.environ["SWIFT_PASSWORD"] = "password"
|
||||
os.environ["SWIFT_PROJECT_NAME"] = "project_name"
|
||||
os.environ["SWIFT_WRITE_CONTAINER"] = "my_container"
|
||||
container = os.environ.get("SWIFT_WRITE_CONTAINER")
|
||||
attachment = self.Attachment
|
||||
bin_data = base64.b64decode(self.blob1_b64)
|
||||
with patch('swiftclient.client.Connection') as MockConnection:
|
||||
with patch("swiftclient.client.Connection") as MockConnection:
|
||||
conn = MockConnection.return_value
|
||||
attachment.create({'name': 'a5', 'datas': self.blob1_b64})
|
||||
attachment.create({"name": "a5", "datas": self.blob1_b64})
|
||||
conn.put_object.assert_called_with(
|
||||
container,
|
||||
attachment._compute_checksum(bin_data),
|
||||
bin_data)
|
||||
container, attachment._compute_checksum(bin_data), bin_data
|
||||
)
|
||||
|
||||
def test_delete_file_on_swift(self):
|
||||
"""
|
||||
Test deleting a file
|
||||
"""
|
||||
(self.env['ir.config_parameter'].
|
||||
set_param('ir_attachment.location', 'swift'))
|
||||
os.environ['SWIFT_AUTH_URL'] = 'auth_url'
|
||||
os.environ['SWIFT_ACCOUNT'] = 'account'
|
||||
os.environ['SWIFT_PASSWORD'] = 'password'
|
||||
os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
|
||||
os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
|
||||
(self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift"))
|
||||
os.environ["SWIFT_AUTH_URL"] = "auth_url"
|
||||
os.environ["SWIFT_ACCOUNT"] = "account"
|
||||
os.environ["SWIFT_PASSWORD"] = "password"
|
||||
os.environ["SWIFT_PROJECT_NAME"] = "project_name"
|
||||
os.environ["SWIFT_WRITE_CONTAINER"] = "my_container"
|
||||
|
||||
attachment = self.Attachment
|
||||
container = os.environ.get('SWIFT_WRITE_CONTAINER')
|
||||
with patch('swiftclient.client.Connection') as MockConnection:
|
||||
container = os.environ.get("SWIFT_WRITE_CONTAINER")
|
||||
with patch("swiftclient.client.Connection") as MockConnection:
|
||||
conn = MockConnection.return_value
|
||||
a5 = attachment.create({'name': 'a5', 'datas': self.blob1_b64})
|
||||
a5 = attachment.create({"name": "a5", "datas": self.blob1_b64})
|
||||
uri = SwiftUri(a5.store_fname)
|
||||
a5.unlink()
|
||||
conn.delete_object.assert_called_with(container, uri.item())
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# Copyright 2017-2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
|
||||
from ..swift_uri import SwiftUri
|
||||
from swiftclient.exceptions import ClientException
|
||||
|
||||
from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
|
||||
|
||||
from ..swift_uri import SwiftUri
|
||||
|
||||
|
||||
class TestAttachmentSwift(TestIrAttachment):
|
||||
"""
|
||||
@@ -13,8 +15,7 @@ class TestAttachmentSwift(TestIrAttachment):
|
||||
|
||||
def setup(self):
|
||||
super().setUp()
|
||||
self.env['ir.config_parameter'].set_param('ir_attachment.location',
|
||||
'swift')
|
||||
self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift")
|
||||
|
||||
def test_connection(self):
|
||||
""" Test the connection to the Swift object store """
|
||||
@@ -23,17 +24,15 @@ class TestAttachmentSwift(TestIrAttachment):
|
||||
|
||||
def test_store_file_on_swift(self):
|
||||
""" Test writing a file and then reading it """
|
||||
(self.env['ir.config_parameter'].
|
||||
set_param('ir_attachment.location', 'swift'))
|
||||
a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
|
||||
(self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift"))
|
||||
a5 = self.Attachment.create({"name": "a5", "datas": self.blob1_b64})
|
||||
a5bis = self.Attachment.browse(a5.id)[0]
|
||||
self.assertEqual(a5.datas, a5bis.datas)
|
||||
|
||||
def test_delete_file_on_swift(self):
|
||||
""" Create a file and then test the deletion """
|
||||
(self.env['ir.config_parameter'].
|
||||
set_param('ir_attachment.location', 'swift'))
|
||||
a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
|
||||
(self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift"))
|
||||
a5 = self.Attachment.create({"name": "a5", "datas": self.blob1_b64})
|
||||
uri = SwiftUri(a5.store_fname)
|
||||
con = self.Attachment._get_swift_connection()
|
||||
con.get_object(uri.container(), uri.item())
|
||||
|
||||
@@ -2,15 +2,16 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Base Attachment Object Store',
|
||||
'summary': 'Base module for the implementation of external object store.',
|
||||
'version': '13.0.1.1.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'Knowledge Management',
|
||||
'depends': ['base'],
|
||||
'website': 'http://www.camptocamp.com',
|
||||
'data': ['data/res_config_settings_data.xml'],
|
||||
'installable': True,
|
||||
'auto_install': True,
|
||||
}
|
||||
{
|
||||
"name": "Base Attachment Object Store",
|
||||
"summary": "Base module for the implementation of external object store.",
|
||||
"version": "13.0.1.1.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Knowledge Management",
|
||||
"depends": ["base"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": ["data/res_config_settings_data.xml"],
|
||||
"installable": True,
|
||||
"auto_install": True,
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<?xml version='1.0' encoding='utf-8' ?>
|
||||
<odoo noupdate="1">
|
||||
|
||||
<record id="ir_attachment_storage_force_database" model="ir.config_parameter">
|
||||
<field name="key">ir_attachment.storage.force.database</field>
|
||||
<field name="value">{"image/": 51200, "application/javascript": 0, "text/css": 0}</field>
|
||||
<field
|
||||
name="value"
|
||||
>{"image/": 51200, "application/javascript": 0, "text/css": 0}</field>
|
||||
</record>
|
||||
|
||||
</odoo>
|
||||
|
||||
@@ -6,44 +6,41 @@ import inspect
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from contextlib import closing, contextmanager
|
||||
|
||||
import psycopg2
|
||||
import odoo
|
||||
|
||||
from contextlib import closing, contextmanager
|
||||
from odoo import api, exceptions, models, _
|
||||
import odoo
|
||||
from odoo import _, api, exceptions, models
|
||||
from odoo.osv.expression import AND, OR, normalize_domain
|
||||
from odoo.tools.safe_eval import const_eval
|
||||
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def clean_fs(files):
|
||||
_logger.info('cleaning old files from filestore')
|
||||
_logger.info("cleaning old files from filestore")
|
||||
for full_path in files:
|
||||
if os.path.exists(full_path):
|
||||
try:
|
||||
os.unlink(full_path)
|
||||
except OSError:
|
||||
_logger.info(
|
||||
"_file_delete could not unlink %s",
|
||||
full_path, exc_info=True
|
||||
"_file_delete could not unlink %s", full_path, exc_info=True
|
||||
)
|
||||
except IOError:
|
||||
# Harmless and needed for race conditions
|
||||
_logger.info(
|
||||
"_file_delete could not unlink %s",
|
||||
full_path, exc_info=True
|
||||
"_file_delete could not unlink %s", full_path, exc_info=True
|
||||
)
|
||||
|
||||
|
||||
class IrAttachment(models.Model):
|
||||
_inherit = 'ir.attachment'
|
||||
_inherit = "ir.attachment"
|
||||
|
||||
def _register_hook(self):
|
||||
super()._register_hook()
|
||||
location = self.env.context.get('storage_location') or self._storage()
|
||||
location = self.env.context.get("storage_location") or self._storage()
|
||||
# ignore if we are not using an object storage
|
||||
if location not in self._get_stores():
|
||||
return
|
||||
@@ -57,7 +54,7 @@ class IrAttachment(models.Model):
|
||||
# done during the initialization. We need to move the attachments that
|
||||
# could have been created or updated in other addons before this addon
|
||||
# was loaded
|
||||
update_module = load_modules_frame.f_locals.get('update_module')
|
||||
update_module = load_modules_frame.f_locals.get("update_module")
|
||||
|
||||
# We need to call the migration on the loading of the model because
|
||||
# when we are upgrading addons, some of them might add attachments.
|
||||
@@ -66,15 +63,17 @@ class IrAttachment(models.Model):
|
||||
# Typical example is images of ir.ui.menu which are updated in
|
||||
# ir.attachment at every upgrade of the addons
|
||||
if update_module:
|
||||
self.env['ir.attachment'].sudo()._force_storage_to_object_storage()
|
||||
self.env["ir.attachment"].sudo()._force_storage_to_object_storage()
|
||||
|
||||
@property
|
||||
def _object_storage_default_force_db_config(self):
|
||||
return {"image/": 51200, "application/javascript": 0, "text/css": 0}
|
||||
|
||||
def _get_storage_force_db_config(self):
|
||||
param = self.env['ir.config_parameter'].sudo().get_param(
|
||||
'ir_attachment.storage.force.database',
|
||||
param = (
|
||||
self.env["ir.config_parameter"]
|
||||
.sudo()
|
||||
.get_param("ir_attachment.storage.force.database",)
|
||||
)
|
||||
storage_config = None
|
||||
if param:
|
||||
@@ -84,7 +83,8 @@ class IrAttachment(models.Model):
|
||||
_logger.exception(
|
||||
"Could not parse system parameter"
|
||||
" 'ir_attachment.storage.force.database', reverting to the"
|
||||
" default configuration.")
|
||||
" default configuration."
|
||||
)
|
||||
|
||||
if not storage_config:
|
||||
storage_config = self._object_storage_default_force_db_config
|
||||
@@ -157,22 +157,22 @@ class IrAttachment(models.Model):
|
||||
if mimetype.startswith(mimetype_key):
|
||||
if not limit:
|
||||
return True
|
||||
bin_data = base64.b64decode(data) if data else b''
|
||||
bin_data = base64.b64decode(data) if data else b""
|
||||
return len(bin_data) <= limit
|
||||
return False
|
||||
|
||||
def _get_datas_related_values(self, data, mimetype):
|
||||
storage = self.env.context.get('storage_location') or self._storage()
|
||||
storage = self.env.context.get("storage_location") or self._storage()
|
||||
if data and storage in self._get_stores():
|
||||
if self._store_in_db_instead_of_object_storage(data, mimetype):
|
||||
# compute the fields that depend on datas
|
||||
bin_data = base64.b64decode(data) if data else b''
|
||||
bin_data = base64.b64decode(data) if data else b""
|
||||
values = {
|
||||
'file_size': len(bin_data),
|
||||
'checksum': self._compute_checksum(bin_data),
|
||||
'index_content': self._index(bin_data, mimetype),
|
||||
'store_fname': False,
|
||||
'db_datas': data,
|
||||
"file_size": len(bin_data),
|
||||
"checksum": self._compute_checksum(bin_data),
|
||||
"index_content": self._index(bin_data, mimetype),
|
||||
"store_fname": False,
|
||||
"db_datas": data,
|
||||
}
|
||||
return values
|
||||
return super()._get_datas_related_values(data, mimetype)
|
||||
@@ -185,28 +185,22 @@ class IrAttachment(models.Model):
|
||||
return super()._file_read(fname, bin_size=bin_size)
|
||||
|
||||
def _store_file_read(self, fname, bin_size=False):
|
||||
storage = fname.partition('://')[0]
|
||||
raise NotImplementedError(
|
||||
'No implementation for %s' % (storage,)
|
||||
)
|
||||
storage = fname.partition("://")[0]
|
||||
raise NotImplementedError("No implementation for %s" % (storage,))
|
||||
|
||||
def _store_file_write(self, key, bin_data):
|
||||
raise NotImplementedError(
|
||||
'No implementation for %s' % (self.storage(),)
|
||||
)
|
||||
raise NotImplementedError("No implementation for %s" % (self.storage(),))
|
||||
|
||||
def _store_file_delete(self, fname):
|
||||
storage = fname.partition('://')[0]
|
||||
raise NotImplementedError(
|
||||
'No implementation for %s' % (storage,)
|
||||
)
|
||||
storage = fname.partition("://")[0]
|
||||
raise NotImplementedError("No implementation for %s" % (storage,))
|
||||
|
||||
@api.model
|
||||
def _file_write(self, value, checksum):
|
||||
location = self.env.context.get('storage_location') or self._storage()
|
||||
location = self.env.context.get("storage_location") or self._storage()
|
||||
if location in self._get_stores():
|
||||
bin_data = base64.b64decode(value)
|
||||
key = self.env.context.get('force_storage_key')
|
||||
key = self.env.context.get("force_storage_key")
|
||||
if not key:
|
||||
key = self._compute_checksum(bin_data)
|
||||
filename = self._store_file_write(key, bin_data)
|
||||
@@ -220,8 +214,9 @@ class IrAttachment(models.Model):
|
||||
cr = self.env.cr
|
||||
# using SQL to include files hidden through unlink or due to record
|
||||
# rules
|
||||
cr.execute("SELECT COUNT(*) FROM ir_attachment "
|
||||
"WHERE store_fname = %s", (fname,))
|
||||
cr.execute(
|
||||
"SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,)
|
||||
)
|
||||
count = cr.fetchone()[0]
|
||||
if not count:
|
||||
self._store_file_delete(fname)
|
||||
@@ -231,7 +226,7 @@ class IrAttachment(models.Model):
|
||||
@api.model
|
||||
def _is_file_from_a_store(self, fname):
|
||||
for store_name in self._get_stores():
|
||||
uri = '{}://'.format(store_name)
|
||||
uri = "{}://".format(store_name)
|
||||
if fname.startswith(uri):
|
||||
return True
|
||||
return False
|
||||
@@ -244,9 +239,7 @@ class IrAttachment(models.Model):
|
||||
"""
|
||||
with api.Environment.manage():
|
||||
if new_cr:
|
||||
registry = odoo.modules.registry.Registry.new(
|
||||
self.env.cr.dbname
|
||||
)
|
||||
registry = odoo.modules.registry.Registry.new(self.env.cr.dbname)
|
||||
with closing(registry.cursor()) as cr:
|
||||
try:
|
||||
yield self.env(cr=cr)
|
||||
@@ -263,30 +256,35 @@ class IrAttachment(models.Model):
|
||||
|
||||
def _move_attachment_to_store(self):
|
||||
self.ensure_one()
|
||||
_logger.info('inspecting attachment %s (%d)', self.name, self.id)
|
||||
_logger.info("inspecting attachment %s (%d)", self.name, self.id)
|
||||
fname = self.store_fname
|
||||
if fname:
|
||||
# migrating from filesystem filestore
|
||||
# or from the old 'store_fname' without the bucket name
|
||||
_logger.info('moving %s on the object storage', fname)
|
||||
self.write({'datas': self.datas,
|
||||
_logger.info("moving %s on the object storage", fname)
|
||||
self.write(
|
||||
{
|
||||
"datas": self.datas,
|
||||
# this is required otherwise the
|
||||
# mimetype gets overriden with
|
||||
# 'application/octet-stream'
|
||||
# on assets
|
||||
'mimetype': self.mimetype})
|
||||
_logger.info('moved %s on the object storage', fname)
|
||||
"mimetype": self.mimetype,
|
||||
}
|
||||
)
|
||||
_logger.info("moved %s on the object storage", fname)
|
||||
return self._full_path(fname)
|
||||
elif self.db_datas:
|
||||
_logger.info('moving on the object storage from database')
|
||||
self.write({'datas': self.datas})
|
||||
_logger.info("moving on the object storage from database")
|
||||
self.write({"datas": self.datas})
|
||||
|
||||
@api.model
|
||||
def force_storage(self):
|
||||
if not self.env['res.users'].browse(self.env.uid)._is_admin():
|
||||
if not self.env["res.users"].browse(self.env.uid)._is_admin():
|
||||
raise exceptions.AccessError(
|
||||
_('Only administrators can execute this action.'))
|
||||
location = self.env.context.get('storage_location') or self._storage()
|
||||
_("Only administrators can execute this action.")
|
||||
)
|
||||
location = self.env.context.get("storage_location") or self._storage()
|
||||
if location not in self._get_stores():
|
||||
return super().force_storage()
|
||||
self._force_storage_to_object_storage()
|
||||
@@ -310,30 +308,32 @@ class IrAttachment(models.Model):
|
||||
if storage not in self._get_stores():
|
||||
return
|
||||
|
||||
domain = AND((
|
||||
domain = AND(
|
||||
(
|
||||
normalize_domain(
|
||||
[('store_fname', '=like', '{}://%'.format(storage)),
|
||||
[
|
||||
("store_fname", "=like", "{}://%".format(storage)),
|
||||
# for res_field, see comment in
|
||||
# _force_storage_to_object_storage
|
||||
'|',
|
||||
('res_field', '=', False),
|
||||
('res_field', '!=', False),
|
||||
"|",
|
||||
("res_field", "=", False),
|
||||
("res_field", "!=", False),
|
||||
]
|
||||
),
|
||||
normalize_domain(self._store_in_db_instead_of_object_storage_domain())
|
||||
))
|
||||
normalize_domain(self._store_in_db_instead_of_object_storage_domain()),
|
||||
)
|
||||
)
|
||||
|
||||
with self.do_in_new_env(new_cr=new_cr) as new_env:
|
||||
model_env = new_env['ir.attachment'].with_context(
|
||||
prefetch_fields=False
|
||||
)
|
||||
model_env = new_env["ir.attachment"].with_context(prefetch_fields=False)
|
||||
attachment_ids = model_env.search(domain).ids
|
||||
if not attachment_ids:
|
||||
return
|
||||
total = len(attachment_ids)
|
||||
start_time = time.time()
|
||||
_logger.info('Moving %d attachments from %s to'
|
||||
' DB for fast access', total, storage)
|
||||
_logger.info(
|
||||
"Moving %d attachments from %s to" " DB for fast access", total, storage
|
||||
)
|
||||
current = 0
|
||||
for attachment_id in attachment_ids:
|
||||
current += 1
|
||||
@@ -345,36 +345,40 @@ class IrAttachment(models.Model):
|
||||
# this write will read the datas from the Object Storage and
|
||||
# write them back in the DB (the logic for location to write is
|
||||
# in the 'datas' inverse computed field)
|
||||
attachment.write({'datas': attachment.datas})
|
||||
attachment.write({"datas": attachment.datas})
|
||||
# as the file will potentially be dropped on the bucket,
|
||||
# we should commit the changes here
|
||||
new_env.cr.commit()
|
||||
if current % 100 == 0 or total - current == 0:
|
||||
_logger.info(
|
||||
'attachment %s/%s after %.2fs',
|
||||
current, total,
|
||||
time.time() - start_time
|
||||
"attachment %s/%s after %.2fs",
|
||||
current,
|
||||
total,
|
||||
time.time() - start_time,
|
||||
)
|
||||
|
||||
@api.model
|
||||
def _force_storage_to_object_storage(self, new_cr=False):
|
||||
_logger.info('migrating files to the object storage')
|
||||
storage = self.env.context.get('storage_location') or self._storage()
|
||||
_logger.info("migrating files to the object storage")
|
||||
storage = self.env.context.get("storage_location") or self._storage()
|
||||
# The weird "res_field = False OR res_field != False" domain
|
||||
# is required! It's because of an override of _search in ir.attachment
|
||||
# which adds ('res_field', '=', False) when the domain does not
|
||||
# contain 'res_field'.
|
||||
# https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347
|
||||
domain = ['!', ('store_fname', '=like', '{}://%'.format(storage)),
|
||||
'|',
|
||||
('res_field', '=', False),
|
||||
('res_field', '!=', False)]
|
||||
# https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347 # noqa: B950
|
||||
domain = [
|
||||
"!",
|
||||
("store_fname", "=like", "{}://%".format(storage)),
|
||||
"|",
|
||||
("res_field", "=", False),
|
||||
("res_field", "!=", False),
|
||||
]
|
||||
# We do a copy of the environment so we can workaround the cache issue
|
||||
# below. We do not create a new cursor by default because it causes
|
||||
# serialization issues due to concurrent updates on attachments during
|
||||
# the installation
|
||||
with self.do_in_new_env(new_cr=new_cr) as new_env:
|
||||
model_env = new_env['ir.attachment']
|
||||
model_env = new_env["ir.attachment"]
|
||||
ids = model_env.search(domain).ids
|
||||
files_to_clean = []
|
||||
for attachment_id in ids:
|
||||
@@ -383,12 +387,14 @@ class IrAttachment(models.Model):
|
||||
# check that no other transaction has
|
||||
# locked the row, don't send a file to storage
|
||||
# in that case
|
||||
self.env.cr.execute("SELECT id "
|
||||
self.env.cr.execute(
|
||||
"SELECT id "
|
||||
"FROM ir_attachment "
|
||||
"WHERE id = %s "
|
||||
"FOR UPDATE NOWAIT",
|
||||
(attachment_id,),
|
||||
log_exceptions=False)
|
||||
log_exceptions=False,
|
||||
)
|
||||
|
||||
# This is a trick to avoid having the 'datas'
|
||||
# function fields computed for every attachment on
|
||||
@@ -401,8 +407,9 @@ class IrAttachment(models.Model):
|
||||
if path:
|
||||
files_to_clean.append(path)
|
||||
except psycopg2.OperationalError:
|
||||
_logger.error('Could not migrate attachment %s to S3',
|
||||
attachment_id)
|
||||
_logger.error(
|
||||
"Could not migrate attachment %s to S3", attachment_id
|
||||
)
|
||||
|
||||
def clean():
|
||||
clean_fs(files_to_clean)
|
||||
@@ -410,7 +417,7 @@ class IrAttachment(models.Model):
|
||||
# delete the files from the filesystem once we know the changes
|
||||
# have been committed in ir.attachment
|
||||
if files_to_clean:
|
||||
new_env.cr.after('commit', clean)
|
||||
new_env.cr.after("commit", clean)
|
||||
|
||||
def _get_stores(self):
|
||||
""" To get the list of stores activated in the system """
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
from . import fields
|
||||
|
||||
|
||||
@@ -5,11 +5,10 @@
|
||||
"summary": "Implementation of FileURL type fields",
|
||||
"version": "13.0.1.0.0",
|
||||
"category": "Technical Settings",
|
||||
'author': 'Camptocamp, Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
"depends": [
|
||||
"base_attachment_object_storage",
|
||||
],
|
||||
"author": "Camptocamp, Odoo Community Association (OCA)",
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"license": "AGPL-3",
|
||||
"depends": ["base_attachment_object_storage"],
|
||||
"auto_install": False,
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -29,10 +29,10 @@ fields.Field.__doc__ += """
|
||||
class FileURL(fields.Binary):
|
||||
|
||||
_slots = {
|
||||
'attachment': True, # Override default with True
|
||||
'storage_location': '', # External storage activated on the system (cf base_attachment_storage) # noqa
|
||||
'storage_path': '', # Path to be used as storage key (prefix of filename) # noqa
|
||||
'filename': '', # Field to use to store the filename on ir.attachment
|
||||
"attachment": True, # Override default with True
|
||||
"storage_location": "", # External storage activated on the system (cf base_attachment_storage) # noqa
|
||||
"storage_path": "", # Path to be used as storage key (prefix of filename) # noqa
|
||||
"filename": "", # Field to use to store the filename on ir.attachment
|
||||
}
|
||||
|
||||
def create(self, record_values):
|
||||
@@ -46,26 +46,27 @@ class FileURL(fields.Binary):
|
||||
if not value:
|
||||
continue
|
||||
vals = {
|
||||
'name': self.name,
|
||||
'res_model': self.model_name,
|
||||
'res_field': self.name,
|
||||
'res_id': record.id,
|
||||
'type': 'binary',
|
||||
'datas': value,
|
||||
"name": self.name,
|
||||
"res_model": self.model_name,
|
||||
"res_field": self.name,
|
||||
"res_id": record.id,
|
||||
"type": "binary",
|
||||
"datas": value,
|
||||
}
|
||||
fname = False
|
||||
if self.filename:
|
||||
fname = record[self.filename]
|
||||
vals['datas_fname'] = fname
|
||||
vals["datas_fname"] = fname
|
||||
if fname and self.storage_path:
|
||||
storage_key = self._build_storage_key(fname)
|
||||
if not fname:
|
||||
storage_key = False
|
||||
env['ir.attachment'].sudo().with_context(
|
||||
env["ir.attachment"].sudo().with_context(
|
||||
binary_field_real_user=env.user,
|
||||
storage_location=self.storage_location,
|
||||
force_storage_key=storage_key,
|
||||
).create(vals)
|
||||
return super().create(record_values)
|
||||
|
||||
def write(self, records, value):
|
||||
for record in records:
|
||||
@@ -79,21 +80,21 @@ class FileURL(fields.Binary):
|
||||
storage_location=self.storage_location,
|
||||
force_storage_key=storage_key,
|
||||
),
|
||||
value
|
||||
value,
|
||||
)
|
||||
return True
|
||||
|
||||
def _setup_regular_base(self, model):
|
||||
super()._setup_regular_base(model)
|
||||
if self.storage_path:
|
||||
assert self.filename is not None, \
|
||||
assert self.filename is not None, (
|
||||
"Field %s defines storage_path without filename" % self
|
||||
)
|
||||
|
||||
def _build_storage_key(self, filename):
|
||||
return '/'.join([
|
||||
self.storage_path.rstrip('/'),
|
||||
unicodedata.normalize('NFKC', filename)
|
||||
])
|
||||
return "/".join(
|
||||
[self.storage_path.rstrip("/"), unicodedata.normalize("NFKC", filename)]
|
||||
)
|
||||
|
||||
|
||||
fields.FileURL = FileURL
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import models
|
||||
|
||||
@@ -2,19 +2,20 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Cloud Platform',
|
||||
'summary': 'Addons required for the Camptocamp Cloud Platform',
|
||||
'version': '13.0.2.0.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'Extra Tools',
|
||||
'depends': [
|
||||
'session_redis',
|
||||
'monitoring_status',
|
||||
'logging_json',
|
||||
'server_environment', # OCA/server-tools
|
||||
{
|
||||
"name": "Cloud Platform",
|
||||
"summary": "Addons required for the Camptocamp Cloud Platform",
|
||||
"version": "13.0.2.0.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Extra Tools",
|
||||
"depends": [
|
||||
"session_redis",
|
||||
"monitoring_status",
|
||||
"logging_json",
|
||||
"server_environment", # OCA/server-tools
|
||||
],
|
||||
'website': 'https://www.camptocamp.com',
|
||||
'data': [],
|
||||
'installable': True,
|
||||
}
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import cloud_platform
|
||||
|
||||
@@ -4,46 +4,38 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
from collections import namedtuple
|
||||
from distutils.util import strtobool
|
||||
|
||||
from odoo import api, models
|
||||
from odoo.tools.config import config
|
||||
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_true(strval):
|
||||
return bool(strtobool(strval or '0'))
|
||||
return bool(strtobool(strval or "0"))
|
||||
|
||||
|
||||
PlatformConfig = namedtuple(
|
||||
'PlatformConfig',
|
||||
'filestore'
|
||||
)
|
||||
PlatformConfig = namedtuple("PlatformConfig", "filestore")
|
||||
|
||||
|
||||
FilestoreKind = namedtuple(
|
||||
'FilestoreKind',
|
||||
['name', 'location']
|
||||
)
|
||||
FilestoreKind = namedtuple("FilestoreKind", ["name", "location"])
|
||||
|
||||
|
||||
class CloudPlatform(models.AbstractModel):
|
||||
_name = 'cloud.platform'
|
||||
_description = 'cloud.platform'
|
||||
_name = "cloud.platform"
|
||||
_description = "cloud.platform"
|
||||
|
||||
@api.model
|
||||
def _default_config(self):
|
||||
return PlatformConfig(self._filestore_kinds()['db'])
|
||||
return PlatformConfig(self._filestore_kinds()["db"])
|
||||
|
||||
@api.model
|
||||
def _filestore_kinds(self):
|
||||
return {
|
||||
'db': FilestoreKind('db', 'local'),
|
||||
'file': FilestoreKind('file', 'local'),
|
||||
"db": FilestoreKind("db", "local"),
|
||||
"file": FilestoreKind("file", "local"),
|
||||
}
|
||||
|
||||
@api.model
|
||||
@@ -53,33 +45,31 @@ class CloudPlatform(models.AbstractModel):
|
||||
@api.model
|
||||
def _config_by_server_env(self, platform_kind, environment):
|
||||
configs_getter = getattr(
|
||||
self,
|
||||
'_config_by_server_env_for_%s' % platform_kind,
|
||||
None
|
||||
self, "_config_by_server_env_for_%s" % platform_kind, None
|
||||
)
|
||||
configs = configs_getter() if configs_getter else {}
|
||||
return configs.get(environment) or self._default_config()
|
||||
|
||||
def _get_running_env(self):
|
||||
environment_name = config['running_env']
|
||||
if environment_name.startswith('labs'):
|
||||
environment_name = config["running_env"]
|
||||
if environment_name.startswith("labs"):
|
||||
# We allow to have environments such as 'labs-logistics'
|
||||
# or 'labs-finance', in order to have the matching ribbon.
|
||||
environment_name = 'labs'
|
||||
environment_name = "labs"
|
||||
return environment_name
|
||||
|
||||
@api.model
|
||||
def _install(self, platform_kind):
|
||||
assert platform_kind in self._platform_kinds()
|
||||
params = self.env['ir.config_parameter'].sudo()
|
||||
params.set_param('cloud.platform.kind', platform_kind)
|
||||
params = self.env["ir.config_parameter"].sudo()
|
||||
params.set_param("cloud.platform.kind", platform_kind)
|
||||
environment_name = self._get_running_env()
|
||||
configs = self._config_by_server_env(platform_kind, environment_name)
|
||||
params.set_param('ir_attachment.location', configs.filestore.name)
|
||||
params.set_param("ir_attachment.location", configs.filestore.name)
|
||||
self.check()
|
||||
if configs.filestore.location == 'remote':
|
||||
self.env['ir.attachment'].sudo().force_storage()
|
||||
_logger.info('cloud platform configured for {}'.format(platform_kind))
|
||||
if configs.filestore.location == "remote":
|
||||
self.env["ir.attachment"].sudo().force_storage()
|
||||
_logger.info("cloud platform configured for {}".format(platform_kind))
|
||||
|
||||
@api.model
|
||||
def install(self):
|
||||
@@ -91,39 +81,39 @@ class CloudPlatform(models.AbstractModel):
|
||||
|
||||
@api.model
|
||||
def _check_redis(self, environment_name):
|
||||
if environment_name in ('prod', 'integration', 'labs', 'test'):
|
||||
assert is_true(os.environ.get('ODOO_SESSION_REDIS')), (
|
||||
if environment_name in ("prod", "integration", "labs", "test"):
|
||||
assert is_true(os.environ.get("ODOO_SESSION_REDIS")), (
|
||||
"Redis must be activated on prod, integration, labs,"
|
||||
" test instances. This is done by setting ODOO_SESSION_REDIS=1."
|
||||
)
|
||||
assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or
|
||||
os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST') or
|
||||
os.environ.get('ODOO_SESSION_REDIS_URL')), (
|
||||
assert (
|
||||
os.environ.get("ODOO_SESSION_REDIS_HOST")
|
||||
or os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST")
|
||||
or os.environ.get("ODOO_SESSION_REDIS_URL")
|
||||
), (
|
||||
"ODOO_SESSION_REDIS_HOST or "
|
||||
"ODOO_SESSION_REDIS_SENTINEL_HOST or "
|
||||
"ODOO_SESSION_REDIS_URL "
|
||||
"environment variable is required to connect on Redis"
|
||||
)
|
||||
assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), (
|
||||
assert os.environ.get("ODOO_SESSION_REDIS_PREFIX"), (
|
||||
"ODOO_SESSION_REDIS_PREFIX environment variable is required "
|
||||
"to store sessions on Redis"
|
||||
)
|
||||
|
||||
prefix = os.environ['ODOO_SESSION_REDIS_PREFIX']
|
||||
assert re.match(r'^[a-z-0-9]+-odoo-[a-z-0-9]+$', prefix), (
|
||||
prefix = os.environ["ODOO_SESSION_REDIS_PREFIX"]
|
||||
assert re.match(r"^[a-z-0-9]+-odoo-[a-z-0-9]+$", prefix), (
|
||||
"ODOO_SESSION_REDIS_PREFIX must match '<client>-odoo-<env>'"
|
||||
", we got: '%s'" % (prefix,)
|
||||
)
|
||||
|
||||
@api.model
|
||||
def check(self):
|
||||
if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')):
|
||||
_logger.warning(
|
||||
"cloud platform checks disabled, this is not safe"
|
||||
)
|
||||
if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")):
|
||||
_logger.warning("cloud platform checks disabled, this is not safe")
|
||||
return
|
||||
params = self.env['ir.config_parameter'].sudo()
|
||||
kind = params.get_param('cloud.platform.kind')
|
||||
params = self.env["ir.config_parameter"].sudo()
|
||||
kind = params.get_param("cloud.platform.kind")
|
||||
if not kind:
|
||||
_logger.warning(
|
||||
"cloud platform not configured, you should "
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
|
||||
def install(ctx):
|
||||
ctx.env['cloud.platform'].install()
|
||||
ctx.env["cloud.platform"].install()
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
Cloud Platform Azure
|
||||
====================
|
||||
# Cloud Platform Azure
|
||||
|
||||
Install addons specific to the Azure setup.
|
||||
|
||||
|
||||
@@ -9,16 +9,9 @@
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Extra Tools",
|
||||
"depends": [
|
||||
"cloud_platform",
|
||||
"attachment_azure",
|
||||
"monitoring_prometheus",
|
||||
],
|
||||
"excludes": [
|
||||
"cloud_platform_ovh",
|
||||
"cloud_platform_exoscale",
|
||||
],
|
||||
"website": "https://www.camptocamp.com",
|
||||
"depends": ["cloud_platform", "attachment_azure", "monitoring_prometheus"],
|
||||
"excludes": ["cloud_platform_ovh", "cloud_platform_exoscale"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
# Copyright 2016-2021 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
import re
|
||||
import os
|
||||
import re
|
||||
|
||||
from odoo import models, api
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig
|
||||
from odoo import api, models
|
||||
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import (
|
||||
FilestoreKind,
|
||||
PlatformConfig,
|
||||
)
|
||||
|
||||
AZURE_STORE_KIND = FilestoreKind("azure", "remote")
|
||||
|
||||
@@ -42,8 +44,7 @@ class CloudPlatform(models.AbstractModel):
|
||||
@api.model
|
||||
def _check_filestore(self, environment_name):
|
||||
params = self.env["ir.config_parameter"].sudo()
|
||||
use_azure = (params.get_param("ir_attachment.location") ==
|
||||
AZURE_STORE_KIND.name)
|
||||
use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name
|
||||
if environment_name in ("prod", "integration"):
|
||||
# Labs instances use azure by default, but we don't want
|
||||
# to enforce it in case we want to test something with a different
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
Cloud Platform Exoscale
|
||||
=======================
|
||||
# Cloud Platform Exoscale
|
||||
|
||||
Install addons specific to the Exoscale setup.
|
||||
|
||||
|
||||
@@ -9,16 +9,9 @@
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Extra Tools",
|
||||
"depends": [
|
||||
"cloud_platform",
|
||||
"attachment_s3",
|
||||
"monitoring_statsd",
|
||||
],
|
||||
"excludes": [
|
||||
"cloud_platform_ovh",
|
||||
"cloud_platform_azure",
|
||||
],
|
||||
"website": "https://www.camptocamp.com",
|
||||
"depends": ["cloud_platform", "attachment_s3", "monitoring_statsd"],
|
||||
"excludes": ["cloud_platform_ovh", "cloud_platform_azure"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,50 +1,51 @@
|
||||
# Copyright 2016-2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
import re
|
||||
import os
|
||||
import re
|
||||
|
||||
from odoo import models, api
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig
|
||||
from odoo import api, models
|
||||
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import (
|
||||
FilestoreKind,
|
||||
PlatformConfig,
|
||||
)
|
||||
|
||||
S3_STORE_KIND = FilestoreKind('s3', 'remote')
|
||||
S3_STORE_KIND = FilestoreKind("s3", "remote")
|
||||
|
||||
|
||||
class CloudPlatform(models.AbstractModel):
|
||||
_inherit = 'cloud.platform'
|
||||
_inherit = "cloud.platform"
|
||||
|
||||
@api.model
|
||||
def _filestore_kinds(self):
|
||||
kinds = super(CloudPlatform, self)._filestore_kinds()
|
||||
kinds['s3'] = S3_STORE_KIND
|
||||
kinds["s3"] = S3_STORE_KIND
|
||||
return kinds
|
||||
|
||||
@api.model
|
||||
def _platform_kinds(self):
|
||||
kinds = super(CloudPlatform, self)._platform_kinds()
|
||||
kinds.append('exoscale')
|
||||
kinds.append("exoscale")
|
||||
return kinds
|
||||
|
||||
@api.model
|
||||
def _config_by_server_env_for_exoscale(self):
|
||||
fs_kinds = self._filestore_kinds()
|
||||
configs = {
|
||||
'prod': PlatformConfig(filestore=fs_kinds['s3']),
|
||||
'integration': PlatformConfig(filestore=fs_kinds['s3']),
|
||||
'labs': PlatformConfig(filestore=fs_kinds['s3']),
|
||||
'test': PlatformConfig(filestore=fs_kinds['db']),
|
||||
'dev': PlatformConfig(filestore=fs_kinds['db']),
|
||||
"prod": PlatformConfig(filestore=fs_kinds["s3"]),
|
||||
"integration": PlatformConfig(filestore=fs_kinds["s3"]),
|
||||
"labs": PlatformConfig(filestore=fs_kinds["s3"]),
|
||||
"test": PlatformConfig(filestore=fs_kinds["db"]),
|
||||
"dev": PlatformConfig(filestore=fs_kinds["db"]),
|
||||
}
|
||||
return configs
|
||||
|
||||
@api.model
|
||||
def _check_filestore(self, environment_name):
|
||||
params = self.env['ir.config_parameter'].sudo()
|
||||
use_s3 = (params.get_param('ir_attachment.location') ==
|
||||
S3_STORE_KIND.name)
|
||||
if environment_name in ('prod', 'integration'):
|
||||
params = self.env["ir.config_parameter"].sudo()
|
||||
use_s3 = params.get_param("ir_attachment.location") == S3_STORE_KIND.name
|
||||
if environment_name in ("prod", "integration"):
|
||||
# Labs instances use s3 by default, but we don't want
|
||||
# to enforce it in case we want to test something with a different
|
||||
# storage. At your own risks!
|
||||
@@ -55,16 +56,16 @@ class CloudPlatform(models.AbstractModel):
|
||||
"automatically."
|
||||
)
|
||||
if use_s3:
|
||||
assert os.environ.get('AWS_ACCESS_KEY_ID'), (
|
||||
assert os.environ.get("AWS_ACCESS_KEY_ID"), (
|
||||
"AWS_ACCESS_KEY_ID environment variable is required when "
|
||||
"ir_attachment.location is 's3'."
|
||||
)
|
||||
assert os.environ.get('AWS_SECRET_ACCESS_KEY'), (
|
||||
assert os.environ.get("AWS_SECRET_ACCESS_KEY"), (
|
||||
"AWS_SECRET_ACCESS_KEY environment variable is required when "
|
||||
"ir_attachment.location is 's3'."
|
||||
)
|
||||
bucket_name = os.environ.get('AWS_BUCKETNAME', '')
|
||||
if environment_name in ('prod', 'integration', 'labs'):
|
||||
bucket_name = os.environ.get("AWS_BUCKETNAME", "")
|
||||
if environment_name in ("prod", "integration", "labs"):
|
||||
assert bucket_name, (
|
||||
"AWS_BUCKETNAME environment variable is required when "
|
||||
"ir_attachment.location is 's3'.\n"
|
||||
@@ -80,10 +81,10 @@ class CloudPlatform(models.AbstractModel):
|
||||
#
|
||||
# Use AWS_BUCKETNAME_UNSTRUCTURED to by-pass check on bucket name
|
||||
# structure
|
||||
if os.environ.get('AWS_BUCKETNAME_UNSTRUCTURED'):
|
||||
if os.environ.get("AWS_BUCKETNAME_UNSTRUCTURED"):
|
||||
return
|
||||
prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name))
|
||||
if environment_name == 'prod':
|
||||
prod_bucket = bool(re.match(r"[a-z-0-9]+-odoo-prod", bucket_name))
|
||||
if environment_name == "prod":
|
||||
assert prod_bucket, (
|
||||
"AWS_BUCKETNAME should match '<client>-odoo-prod', "
|
||||
"we got: '%s'" % (bucket_name,)
|
||||
@@ -96,9 +97,9 @@ class CloudPlatform(models.AbstractModel):
|
||||
"we got: '%s'" % (bucket_name,)
|
||||
)
|
||||
|
||||
elif environment_name == 'test':
|
||||
elif environment_name == "test":
|
||||
# store in DB so we don't have files local to the host
|
||||
assert params.get_param('ir_attachment.location') == 'db', (
|
||||
assert params.get_param("ir_attachment.location") == "db", (
|
||||
"In test instances, files must be stored in the database with "
|
||||
"'ir_attachment.location' set to 'db'. This is "
|
||||
"automatically set by the function 'install()'."
|
||||
@@ -106,4 +107,4 @@ class CloudPlatform(models.AbstractModel):
|
||||
|
||||
@api.model
|
||||
def install(self):
|
||||
self._install('exoscale')
|
||||
self._install("exoscale")
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
Cloud Platform OVH
|
||||
==================
|
||||
# Cloud Platform OVH
|
||||
|
||||
Install addons specific to the OVH setup.
|
||||
|
||||
* The object storage is Swift
|
||||
|
||||
|
||||
@@ -9,16 +9,9 @@
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Extra Tools",
|
||||
"depends": [
|
||||
"cloud_platform",
|
||||
"attachment_swift",
|
||||
"monitoring_statsd",
|
||||
],
|
||||
"excludes": [
|
||||
"cloud_platform_exoscale",
|
||||
"cloud_platform_azure",
|
||||
],
|
||||
"website": "https://www.camptocamp.com",
|
||||
"depends": ["cloud_platform", "attachment_swift", "monitoring_statsd"],
|
||||
"excludes": ["cloud_platform_exoscale", "cloud_platform_azure"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,51 +1,51 @@
|
||||
# Copyright 2017-2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
import re
|
||||
import os
|
||||
import re
|
||||
|
||||
from odoo import api, models
|
||||
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig
|
||||
from odoo.addons.cloud_platform.models.cloud_platform import (
|
||||
FilestoreKind,
|
||||
PlatformConfig,
|
||||
)
|
||||
|
||||
|
||||
SWIFT_STORE_KIND = FilestoreKind('swift', 'remote')
|
||||
SWIFT_STORE_KIND = FilestoreKind("swift", "remote")
|
||||
|
||||
|
||||
class CloudPlatform(models.AbstractModel):
|
||||
_inherit = 'cloud.platform'
|
||||
_inherit = "cloud.platform"
|
||||
|
||||
@api.model
|
||||
def _filestore_kinds(self):
|
||||
kinds = super(CloudPlatform, self)._filestore_kinds()
|
||||
kinds['swift'] = SWIFT_STORE_KIND
|
||||
kinds["swift"] = SWIFT_STORE_KIND
|
||||
return kinds
|
||||
|
||||
@api.model
|
||||
def _platform_kinds(self):
|
||||
kinds = super()._platform_kinds()
|
||||
kinds.append('ovh')
|
||||
kinds.append("ovh")
|
||||
return kinds
|
||||
|
||||
@api.model
|
||||
def _config_by_server_env_for_ovh(self):
|
||||
fs_kinds = self._filestore_kinds()
|
||||
configs = {
|
||||
'prod': PlatformConfig(filestore=fs_kinds['swift']),
|
||||
'integration': PlatformConfig(filestore=fs_kinds['swift']),
|
||||
'labs': PlatformConfig(filestore=fs_kinds['swift']),
|
||||
'test': PlatformConfig(filestore=fs_kinds['db']),
|
||||
'dev': PlatformConfig(filestore=fs_kinds['db']),
|
||||
"prod": PlatformConfig(filestore=fs_kinds["swift"]),
|
||||
"integration": PlatformConfig(filestore=fs_kinds["swift"]),
|
||||
"labs": PlatformConfig(filestore=fs_kinds["swift"]),
|
||||
"test": PlatformConfig(filestore=fs_kinds["db"]),
|
||||
"dev": PlatformConfig(filestore=fs_kinds["db"]),
|
||||
}
|
||||
return configs
|
||||
|
||||
@api.model
|
||||
def _check_filestore(self, environment_name):
|
||||
params = self.env['ir.config_parameter'].sudo()
|
||||
use_swift = (params.get_param('ir_attachment.location') ==
|
||||
SWIFT_STORE_KIND.name)
|
||||
if environment_name in ('prod', 'integration'):
|
||||
params = self.env["ir.config_parameter"].sudo()
|
||||
use_swift = params.get_param("ir_attachment.location") == SWIFT_STORE_KIND.name
|
||||
if environment_name in ("prod", "integration"):
|
||||
# Labs instances use swift by default, but we don't want
|
||||
# to enforce it in case we want to test something with a different
|
||||
# storage. At your own risks!
|
||||
@@ -56,20 +56,20 @@ class CloudPlatform(models.AbstractModel):
|
||||
"automatically."
|
||||
)
|
||||
if use_swift:
|
||||
assert os.environ.get('SWIFT_AUTH_URL'), (
|
||||
assert os.environ.get("SWIFT_AUTH_URL"), (
|
||||
"SWIFT_AUTH_URL environment variable is required when "
|
||||
"ir_attachment.location is 'swift'."
|
||||
)
|
||||
assert os.environ.get('SWIFT_ACCOUNT'), (
|
||||
assert os.environ.get("SWIFT_ACCOUNT"), (
|
||||
"SWIFT_ACCOUNT environment variable is required when "
|
||||
"ir_attachment.location is 'swift'."
|
||||
)
|
||||
assert os.environ.get('SWIFT_PASSWORD'), (
|
||||
assert os.environ.get("SWIFT_PASSWORD"), (
|
||||
"SWIFT_PASSWORD environment variable is required when "
|
||||
"ir_attachment.location is 'swift'."
|
||||
)
|
||||
container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '')
|
||||
if environment_name in ('prod', 'integration', 'labs'):
|
||||
container_name = os.environ.get("SWIFT_WRITE_CONTAINER", "")
|
||||
if environment_name in ("prod", "integration", "labs"):
|
||||
assert container_name, (
|
||||
"SWIFT_WRITE_CONTAINER environment variable is required when "
|
||||
"ir_attachment.location is 'swift'.\n"
|
||||
@@ -80,16 +80,15 @@ class CloudPlatform(models.AbstractModel):
|
||||
"If you don't actually need a bucket, change the"
|
||||
" 'ir_attachment.location' parameter."
|
||||
)
|
||||
prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod',
|
||||
container_name))
|
||||
prod_container = bool(re.match(r"[a-z0-9-]+-odoo-prod", container_name))
|
||||
# A bucket name is defined under the following format
|
||||
# <client>-odoo-<env>
|
||||
#
|
||||
# Use SWIFT_WRITE_CONTAINER_UNSTRUCTURED to by-pass check on bucket name
|
||||
# structure
|
||||
if os.environ.get('SWIFT_WRITE_CONTAINER_UNSTRUCTURED'):
|
||||
if os.environ.get("SWIFT_WRITE_CONTAINER_UNSTRUCTURED"):
|
||||
return
|
||||
if environment_name == 'prod':
|
||||
if environment_name == "prod":
|
||||
assert prod_container, (
|
||||
"SWIFT_WRITE_CONTAINER should match '<client>-odoo-prod', "
|
||||
"we got: '%s'" % (container_name,)
|
||||
@@ -101,9 +100,9 @@ class CloudPlatform(models.AbstractModel):
|
||||
"SWIFT_WRITE_CONTAINER should not match "
|
||||
"'<client>-odoo-prod', we got: '%s'" % (container_name,)
|
||||
)
|
||||
elif environment_name == 'test':
|
||||
elif environment_name == "test":
|
||||
# store in DB so we don't have files local to the host
|
||||
assert params.get_param('ir_attachment.location') == 'db', (
|
||||
assert params.get_param("ir_attachment.location") == "db", (
|
||||
"In test instances, files must be stored in the database with "
|
||||
"'ir_attachment.location' set to 'db'. This is "
|
||||
"automatically set by the function 'install()'."
|
||||
@@ -111,4 +110,4 @@ class CloudPlatform(models.AbstractModel):
|
||||
|
||||
@api.model
|
||||
def install(self):
|
||||
self._install('ovh')
|
||||
self._install("ovh")
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import json_log
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
# Copyright 2016-2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
{'name': 'JSON Logging',
|
||||
'version': '13.0.1.0.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'Extra Tools',
|
||||
'depends': ['base',
|
||||
],
|
||||
'external_dependencies': {
|
||||
'python': ['python-json-logger'],
|
||||
},
|
||||
'website': 'http://www.camptocamp.com',
|
||||
'data': [],
|
||||
'installable': True,
|
||||
}
|
||||
{
|
||||
"name": "JSON Logging",
|
||||
"version": "13.0.1.0.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Extra Tools",
|
||||
"depends": ["base"],
|
||||
"external_dependencies": {"python": ["python-json-logger"]},
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from distutils.util import strtobool
|
||||
|
||||
from odoo import http
|
||||
@@ -20,24 +19,22 @@ except ImportError:
|
||||
|
||||
|
||||
def is_true(strval):
|
||||
return bool(strtobool(strval or '0'.lower()))
|
||||
return bool(strtobool(strval or "0".lower()))
|
||||
|
||||
|
||||
class OdooJsonFormatter(jsonlogger.JsonFormatter):
|
||||
|
||||
def add_fields(self, log_record, record, message_dict):
|
||||
record.pid = os.getpid()
|
||||
record.dbname = getattr(threading.currentThread(), 'dbname', '?')
|
||||
record.request_id = getattr(threading.current_thread(), 'request_uuid', None)
|
||||
record.uid = getattr(threading.current_thread(), 'uid', None)
|
||||
record.dbname = getattr(threading.currentThread(), "dbname", "?")
|
||||
record.request_id = getattr(threading.current_thread(), "request_uuid", None)
|
||||
record.uid = getattr(threading.current_thread(), "uid", None)
|
||||
_super = super(OdooJsonFormatter, self)
|
||||
return _super.add_fields(log_record, record, message_dict)
|
||||
|
||||
|
||||
if is_true(os.environ.get('ODOO_LOGGING_JSON')):
|
||||
format = ('%(asctime)s %(pid)s %(levelname)s'
|
||||
'%(dbname)s %(name)s: %(message)s')
|
||||
formatter = OdooJsonFormatter(format)
|
||||
if is_true(os.environ.get("ODOO_LOGGING_JSON")):
|
||||
format_str = "%(asctime)s %(pid)s %(levelname)s" "%(dbname)s %(name)s: %(message)s"
|
||||
formatter = OdooJsonFormatter(format_str)
|
||||
logging.getLogger().handlers[0].formatter = formatter
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
|
||||
from . import utils
|
||||
from . import models
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Monitoring: Requests Logging',
|
||||
'version': '13.0.1.0.0',
|
||||
'author': 'Camptocamp,Numigi,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'category',
|
||||
'depends': ['base', 'web'],
|
||||
'website': 'http://www.camptocamp.com',
|
||||
'data': [],
|
||||
'installable': True,
|
||||
}
|
||||
{
|
||||
"name": "Monitoring: Requests Logging",
|
||||
"version": "13.0.1.0.0",
|
||||
"author": "Camptocamp,Numigi,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "category",
|
||||
"depends": ["base", "web"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
|
||||
from ..utils import is_enabled
|
||||
|
||||
if is_enabled():
|
||||
|
||||
@@ -3,15 +3,16 @@
|
||||
|
||||
import json
|
||||
import time
|
||||
from os import environ
|
||||
from collections import MutableMapping
|
||||
from contextlib import suppress
|
||||
from os import environ
|
||||
|
||||
from werkzeug.exceptions import HTTPException
|
||||
|
||||
from odoo import models
|
||||
from odoo.http import request as http_request
|
||||
from odoo.tools.config import config
|
||||
|
||||
|
||||
udp_dest = environ.get("ODOO_REQUESTS_LOGGING_UDP")
|
||||
if udp_dest:
|
||||
import socket
|
||||
@@ -25,6 +26,8 @@ if udp_dest:
|
||||
def output_method(data):
|
||||
data += "\n"
|
||||
sock.sendto(data.encode("utf-8"), (ip, port))
|
||||
|
||||
|
||||
else:
|
||||
import logging
|
||||
|
||||
|
||||
@@ -6,5 +6,5 @@ from os import environ
|
||||
|
||||
|
||||
def is_enabled():
|
||||
env_val = environ.get('ODOO_REQUESTS_LOGGING')
|
||||
return bool(strtobool(env_val or '0'.lower()))
|
||||
env_val = environ.get("ODOO_REQUESTS_LOGGING")
|
||||
return bool(strtobool(env_val or "0".lower()))
|
||||
|
||||
@@ -8,15 +8,9 @@
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "category",
|
||||
"depends": [
|
||||
"base",
|
||||
"web",
|
||||
"server_environment",
|
||||
],
|
||||
"website": "http://www.camptocamp.com",
|
||||
"depends": ["base", "web", "server_environment"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"external_dependencies": {
|
||||
"python": ["prometheus_client"],
|
||||
},
|
||||
"external_dependencies": {"python": ["prometheus_client"]},
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
# Copyright 2016-2021 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
from odoo.http import Controller, route
|
||||
from prometheus_client import generate_latest
|
||||
|
||||
from odoo.http import Controller, route
|
||||
|
||||
|
||||
class PrometheusController(Controller):
|
||||
@route('/metrics', auth='public')
|
||||
@route("/metrics", auth="public")
|
||||
def metrics(self):
|
||||
return generate_latest()
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Copyright 2016-2021 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
from prometheus_client import Counter, Summary
|
||||
|
||||
from odoo import models
|
||||
from odoo.http import request
|
||||
from prometheus_client import Summary, Counter
|
||||
|
||||
|
||||
REQUEST_TIME = Summary(
|
||||
"request_latency_sec", "Request response time in sec", ["query_type"]
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import models
|
||||
|
||||
@@ -2,19 +2,15 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Monitoring: Statsd Metrics',
|
||||
'version': '13.0.1.0.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'category',
|
||||
'depends': ['base',
|
||||
'web',
|
||||
'server_environment',
|
||||
],
|
||||
'website': 'http://www.camptocamp.com',
|
||||
'data': [],
|
||||
'external_dependencies': {
|
||||
'python': ['statsd'],
|
||||
},
|
||||
'installable': True,
|
||||
}
|
||||
{
|
||||
"name": "Monitoring: Statsd Metrics",
|
||||
"version": "13.0.1.0.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "category",
|
||||
"depends": ["base", "web", "server_environment"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"external_dependencies": {"python": ["statsd"]},
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
|
||||
from . import ir_http
|
||||
|
||||
@@ -4,38 +4,46 @@
|
||||
from odoo import models
|
||||
from odoo.http import request
|
||||
|
||||
from ..statsd_client import statsd, customer, environment
|
||||
from ..statsd_client import customer, environment, statsd
|
||||
|
||||
|
||||
class IrHttp(models.AbstractModel):
|
||||
_inherit = 'ir.http'
|
||||
_inherit = "ir.http"
|
||||
|
||||
@classmethod
|
||||
def _dispatch(cls):
|
||||
if not statsd:
|
||||
return super()._dispatch()
|
||||
|
||||
path_info = request.httprequest.environ.get('PATH_INFO')
|
||||
if path_info.startswith('/longpolling/'):
|
||||
path_info = request.httprequest.environ.get("PATH_INFO")
|
||||
if path_info.startswith("/longpolling/"):
|
||||
return super()._dispatch()
|
||||
|
||||
parts = ['http', ]
|
||||
if path_info.startswith('/web/dataset/call_button'):
|
||||
parts += ['button',
|
||||
customer, environment,
|
||||
request.params['model'].replace('.', '_'),
|
||||
request.params['method'],
|
||||
parts = [
|
||||
"http",
|
||||
]
|
||||
elif path_info.startswith('/web/dataset/exec_workflow'):
|
||||
parts += ['workflow',
|
||||
customer, environment,
|
||||
request.params['model'].replace('.', '_'),
|
||||
request.params['signal'],
|
||||
if path_info.startswith("/web/dataset/call_button"):
|
||||
parts += [
|
||||
"button",
|
||||
customer,
|
||||
environment,
|
||||
request.params["model"].replace(".", "_"),
|
||||
request.params["method"],
|
||||
]
|
||||
elif path_info.startswith("/web/dataset/exec_workflow"):
|
||||
parts += [
|
||||
"workflow",
|
||||
customer,
|
||||
environment,
|
||||
request.params["model"].replace(".", "_"),
|
||||
request.params["signal"],
|
||||
]
|
||||
else:
|
||||
parts += ['request',
|
||||
customer, environment,
|
||||
parts += [
|
||||
"request",
|
||||
customer,
|
||||
environment,
|
||||
]
|
||||
|
||||
with statsd.timer('.'.join(parts)):
|
||||
with statsd.timer(".".join(parts)):
|
||||
return super()._dispatch()
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from distutils.util import strtobool
|
||||
|
||||
from odoo.tools.config import config
|
||||
@@ -14,40 +13,39 @@ try:
|
||||
from statsd import defaults
|
||||
from statsd.client import StatsClient
|
||||
except ImportError:
|
||||
_logger.warning('statds must be installed')
|
||||
_logger.warning("statds must be installed")
|
||||
defaults = None # noqa
|
||||
StatsClient = None # noqa
|
||||
|
||||
|
||||
def is_true(strval):
|
||||
return bool(strtobool(strval or '0'.lower()))
|
||||
return bool(strtobool(strval or "0".lower()))
|
||||
|
||||
|
||||
statsd_active = is_true(os.environ.get('ODOO_STATSD'))
|
||||
statsd_active = is_true(os.environ.get("ODOO_STATSD"))
|
||||
|
||||
statsd = None
|
||||
customer = None
|
||||
environment = None
|
||||
if statsd_active and statsd is None and StatsClient is not None:
|
||||
if not os.environ.get('STATSD_CUSTOMER'):
|
||||
raise Exception(
|
||||
'STATSD_CUSTOMER must contain the name of the customer'
|
||||
)
|
||||
customer = os.environ.get('STATSD_CUSTOMER')
|
||||
if os.environ.get('STATSD_ENVIRONMENT'):
|
||||
environment = os.environ['STATSD_ENVIRONMENT']
|
||||
elif config.get('running_env'):
|
||||
environment = config['running_env']
|
||||
if not os.environ.get("STATSD_CUSTOMER"):
|
||||
raise Exception("STATSD_CUSTOMER must contain the name of the customer")
|
||||
customer = os.environ.get("STATSD_CUSTOMER")
|
||||
if os.environ.get("STATSD_ENVIRONMENT"):
|
||||
environment = os.environ["STATSD_ENVIRONMENT"]
|
||||
elif config.get("running_env"):
|
||||
environment = config["running_env"]
|
||||
else:
|
||||
raise Exception(
|
||||
'Either STATSD_ENVIRONMENT or configuration option running_env '
|
||||
'must contain the environment (prod, integration, ...)'
|
||||
"Either STATSD_ENVIRONMENT or configuration option running_env "
|
||||
"must contain the environment (prod, integration, ...)"
|
||||
)
|
||||
|
||||
host = os.getenv('STATSD_HOST', defaults.HOST)
|
||||
port = int(os.getenv('STATSD_PORT', defaults.PORT))
|
||||
prefix = os.getenv('STATSD_PREFIX', defaults.PREFIX)
|
||||
maxudpsize = int(os.getenv('STATSD_MAXUDPSIZE', defaults.MAXUDPSIZE))
|
||||
ipv6 = bool(int(os.getenv('STATSD_IPV6', defaults.IPV6)))
|
||||
statsd = StatsClient(host=host, port=port, prefix='odoo',
|
||||
maxudpsize=maxudpsize, ipv6=ipv6)
|
||||
host = os.getenv("STATSD_HOST", defaults.HOST)
|
||||
port = int(os.getenv("STATSD_PORT", defaults.PORT))
|
||||
prefix = os.getenv("STATSD_PREFIX", defaults.PREFIX)
|
||||
maxudpsize = int(os.getenv("STATSD_MAXUDPSIZE", defaults.MAXUDPSIZE))
|
||||
ipv6 = bool(int(os.getenv("STATSD_IPV6", defaults.IPV6)))
|
||||
statsd = StatsClient(
|
||||
host=host, port=port, prefix="odoo", maxudpsize=maxudpsize, ipv6=ipv6
|
||||
)
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Monitoring: Status',
|
||||
'version': '13.0.1.0.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'category',
|
||||
'depends': ['base', 'web'],
|
||||
'website': 'http://www.camptocamp.com',
|
||||
'data': [],
|
||||
'installable': True,
|
||||
}
|
||||
{
|
||||
"name": "Monitoring: Status",
|
||||
"version": "13.0.1.0.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "category",
|
||||
"depends": ["base", "web"],
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
# Copyright 2016-2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
import logging
|
||||
import json
|
||||
import logging
|
||||
|
||||
import werkzeug
|
||||
|
||||
from odoo import http
|
||||
|
||||
from odoo.addons.web.controllers.main import ensure_db
|
||||
|
||||
|
||||
class HealthCheckFilter(logging.Filter):
|
||||
|
||||
def __init__(self, path, name=''):
|
||||
def __init__(self, path, name=""):
|
||||
super().__init__(name)
|
||||
self.path = path
|
||||
|
||||
@@ -20,20 +20,19 @@ class HealthCheckFilter(logging.Filter):
|
||||
return self.path not in record.getMessage()
|
||||
|
||||
|
||||
logging.getLogger('werkzeug').addFilter(
|
||||
HealthCheckFilter('GET /monitoring/status HTTP')
|
||||
logging.getLogger("werkzeug").addFilter(
|
||||
HealthCheckFilter("GET /monitoring/status HTTP")
|
||||
)
|
||||
|
||||
|
||||
class Monitoring(http.Controller):
|
||||
|
||||
@http.route('/monitoring/status', type='http', auth='none')
|
||||
@http.route("/monitoring/status", type="http", auth="none")
|
||||
def status(self):
|
||||
ensure_db()
|
||||
# TODO: add 'sub-systems' status and infos:
|
||||
# queue job, cron, database, ...
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
info = {'status': 1}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
info = {"status": 1}
|
||||
session = http.request.session
|
||||
# We set a custom expiration of 1 second for this request, as we do a
|
||||
# lot of health checks, we don't want those anonymous sessions to be
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
|
||||
from . import http
|
||||
from . import session
|
||||
|
||||
@@ -2,17 +2,16 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
|
||||
{'name': 'Sessions in Redis',
|
||||
'summary': 'Store web sessions in Redis',
|
||||
'version': '13.0.1.0.0',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'category': 'Extra Tools',
|
||||
'depends': ['base'],
|
||||
'external_dependencies': {
|
||||
'python': ['redis'],
|
||||
},
|
||||
'website': 'http://www.camptocamp.com',
|
||||
'data': [],
|
||||
'installable': True,
|
||||
}
|
||||
{
|
||||
"name": "Sessions in Redis",
|
||||
"summary": "Store web sessions in Redis",
|
||||
"version": "13.0.1.0.0",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"license": "AGPL-3",
|
||||
"category": "Extra Tools",
|
||||
"depends": ["base"],
|
||||
"external_dependencies": {"python": ["redis"]},
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"data": [],
|
||||
"installable": True,
|
||||
}
|
||||
|
||||
+30
-23
@@ -3,7 +3,6 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from distutils.util import strtobool
|
||||
|
||||
import odoo
|
||||
@@ -23,42 +22,42 @@ except ImportError:
|
||||
|
||||
|
||||
def is_true(strval):
|
||||
return bool(strtobool(strval or '0'.lower()))
|
||||
return bool(strtobool(strval or "0".lower()))
|
||||
|
||||
|
||||
sentinel_host = os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')
|
||||
sentinel_master_name = os.environ.get(
|
||||
'ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME'
|
||||
)
|
||||
sentinel_host = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST")
|
||||
sentinel_master_name = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME")
|
||||
if sentinel_host and not sentinel_master_name:
|
||||
raise Exception(
|
||||
"ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME must be defined "
|
||||
"when using session_redis"
|
||||
)
|
||||
sentinel_port = int(os.environ.get('ODOO_SESSION_REDIS_SENTINEL_PORT', 26379))
|
||||
host = os.environ.get('ODOO_SESSION_REDIS_HOST', 'localhost')
|
||||
port = int(os.environ.get('ODOO_SESSION_REDIS_PORT', 6379))
|
||||
prefix = os.environ.get('ODOO_SESSION_REDIS_PREFIX')
|
||||
url = os.environ.get('ODOO_SESSION_REDIS_URL')
|
||||
password = os.environ.get('ODOO_SESSION_REDIS_PASSWORD')
|
||||
expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION')
|
||||
anon_expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS')
|
||||
sentinel_port = int(os.environ.get("ODOO_SESSION_REDIS_SENTINEL_PORT", 26379))
|
||||
host = os.environ.get("ODOO_SESSION_REDIS_HOST", "localhost")
|
||||
port = int(os.environ.get("ODOO_SESSION_REDIS_PORT", 6379))
|
||||
prefix = os.environ.get("ODOO_SESSION_REDIS_PREFIX")
|
||||
url = os.environ.get("ODOO_SESSION_REDIS_URL")
|
||||
password = os.environ.get("ODOO_SESSION_REDIS_PASSWORD")
|
||||
expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION")
|
||||
anon_expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS")
|
||||
|
||||
|
||||
@lazy_property
|
||||
def session_store(self):
|
||||
if sentinel_host:
|
||||
sentinel = Sentinel([(sentinel_host, sentinel_port)],
|
||||
password=password)
|
||||
sentinel = Sentinel([(sentinel_host, sentinel_port)], password=password)
|
||||
redis_client = sentinel.master_for(sentinel_master_name)
|
||||
elif url:
|
||||
redis_client = redis.from_url(url)
|
||||
else:
|
||||
redis_client = redis.Redis(host=host, port=port, password=password)
|
||||
return RedisSessionStore(redis=redis_client, prefix=prefix,
|
||||
return RedisSessionStore(
|
||||
redis=redis_client,
|
||||
prefix=prefix,
|
||||
expiration=expiration,
|
||||
anon_expiration=anon_expiration,
|
||||
session_class=http.OpenERPSession)
|
||||
session_class=http.OpenERPSession,
|
||||
)
|
||||
|
||||
|
||||
def session_gc(session_store):
|
||||
@@ -79,14 +78,22 @@ def purge_fs_sessions(path):
|
||||
pass
|
||||
|
||||
|
||||
if is_true(os.environ.get('ODOO_SESSION_REDIS')):
|
||||
if is_true(os.environ.get("ODOO_SESSION_REDIS")):
|
||||
if sentinel_host:
|
||||
_logger.debug("HTTP sessions stored in Redis with prefix '%s'. "
|
||||
_logger.debug(
|
||||
"HTTP sessions stored in Redis with prefix '%s'. "
|
||||
"Using Sentinel on %s:%s",
|
||||
prefix or '', sentinel_host, sentinel_port)
|
||||
prefix or "",
|
||||
sentinel_host,
|
||||
sentinel_port,
|
||||
)
|
||||
else:
|
||||
_logger.debug("HTTP sessions stored in Redis with prefix '%s' on "
|
||||
"%s:%s", prefix or '', host, port)
|
||||
_logger.debug(
|
||||
"HTTP sessions stored in Redis with prefix '%s' on " "%s:%s",
|
||||
prefix or "",
|
||||
host,
|
||||
port,
|
||||
)
|
||||
|
||||
http.Root.session_store = session_store
|
||||
http.session_gc = session_gc
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||
|
||||
import json
|
||||
|
||||
from datetime import date, datetime
|
||||
|
||||
import dateutil
|
||||
|
||||
+39
-27
@@ -19,8 +19,14 @@ _logger = logging.getLogger(__name__)
|
||||
class RedisSessionStore(SessionStore):
|
||||
""" SessionStore that saves session to redis """
|
||||
|
||||
def __init__(self, redis, session_class=None,
|
||||
prefix='', expiration=None, anon_expiration=None):
|
||||
def __init__(
|
||||
self,
|
||||
redis,
|
||||
session_class=None,
|
||||
prefix="",
|
||||
expiration=None,
|
||||
anon_expiration=None,
|
||||
):
|
||||
super().__init__(session_class=session_class)
|
||||
self.redis = redis
|
||||
if expiration is None:
|
||||
@@ -31,14 +37,12 @@ class RedisSessionStore(SessionStore):
|
||||
self.anon_expiration = DEFAULT_SESSION_TIMEOUT_ANONYMOUS
|
||||
else:
|
||||
self.anon_expiration = anon_expiration
|
||||
self.prefix = 'session:'
|
||||
self.prefix = "session:"
|
||||
if prefix:
|
||||
self.prefix = '%s:%s:' % (
|
||||
self.prefix, prefix
|
||||
)
|
||||
self.prefix = "%s:%s:" % (self.prefix, prefix)
|
||||
|
||||
def build_key(self, sid):
|
||||
return '%s%s' % (self.prefix, sid)
|
||||
return "%s%s" % (self.prefix, sid)
|
||||
|
||||
def save(self, session):
|
||||
key = self.build_key(session.sid)
|
||||
@@ -51,48 +55,56 @@ class RedisSessionStore(SessionStore):
|
||||
expiration = session.expiration or self.anon_expiration
|
||||
if _logger.isEnabledFor(logging.DEBUG):
|
||||
if session.uid:
|
||||
user_msg = "user '%s' (id: %s)" % (
|
||||
session.login, session.uid)
|
||||
user_msg = "user '%s' (id: %s)" % (session.login, session.uid)
|
||||
else:
|
||||
user_msg = "anonymous user"
|
||||
_logger.debug("saving session with key '%s' and "
|
||||
"expiration of %s seconds for %s",
|
||||
key, expiration, user_msg)
|
||||
_logger.debug(
|
||||
"saving session with key '%s' and " "expiration of %s seconds for %s",
|
||||
key,
|
||||
expiration,
|
||||
user_msg,
|
||||
)
|
||||
|
||||
data = json.dumps(
|
||||
dict(session), cls=json_encoding.SessionEncoder
|
||||
).encode('utf-8')
|
||||
data = json.dumps(dict(session), cls=json_encoding.SessionEncoder).encode(
|
||||
"utf-8"
|
||||
)
|
||||
if self.redis.set(key, data):
|
||||
return self.redis.expire(key, expiration)
|
||||
|
||||
def delete(self, session):
|
||||
key = self.build_key(session.sid)
|
||||
_logger.debug('deleting session with key %s', key)
|
||||
_logger.debug("deleting session with key %s", key)
|
||||
return self.redis.delete(key)
|
||||
|
||||
def get(self, sid):
|
||||
if not self.is_valid_key(sid):
|
||||
_logger.debug("session with invalid sid '%s' has been asked, "
|
||||
"returning a new one", sid)
|
||||
_logger.debug(
|
||||
"session with invalid sid '%s' has been asked, " "returning a new one",
|
||||
sid,
|
||||
)
|
||||
return self.new()
|
||||
|
||||
key = self.build_key(sid)
|
||||
saved = self.redis.get(key)
|
||||
if not saved:
|
||||
_logger.debug("session with non-existent key '%s' has been asked, "
|
||||
"returning a new one", key)
|
||||
_logger.debug(
|
||||
"session with non-existent key '%s' has been asked, "
|
||||
"returning a new one",
|
||||
key,
|
||||
)
|
||||
return self.new()
|
||||
try:
|
||||
data = json.loads(
|
||||
saved.decode('utf-8'), cls=json_encoding.SessionDecoder
|
||||
)
|
||||
data = json.loads(saved.decode("utf-8"), cls=json_encoding.SessionDecoder)
|
||||
except ValueError:
|
||||
_logger.debug("session for key '%s' has been asked but its json "
|
||||
"content could not be read, it has been reset", key)
|
||||
_logger.debug(
|
||||
"session for key '%s' has been asked but its json "
|
||||
"content could not be read, it has been reset",
|
||||
key,
|
||||
)
|
||||
data = {}
|
||||
return self.session_class(data, sid, False)
|
||||
|
||||
def list(self):
|
||||
keys = self.redis.keys('%s*' % self.prefix)
|
||||
keys = self.redis.keys("%s*" % self.prefix)
|
||||
_logger.debug("a listing redis keys has been called")
|
||||
return [key[len(self.prefix):] for key in keys]
|
||||
return [key[len(self.prefix) :] for key in keys]
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
# Copyright 2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
|
||||
{
|
||||
'name': 'test base fileurl fields',
|
||||
'summary': """A module to verify fileurl field.""",
|
||||
'version': '12.0.1.0.0',
|
||||
'category': 'Tests',
|
||||
'author': 'Camptocamp,Odoo Community Association (OCA)',
|
||||
'license': 'AGPL-3',
|
||||
'depends': [
|
||||
'base_fileurl_field'
|
||||
],
|
||||
'data': [
|
||||
"views/res_partner.xml",
|
||||
"views/res_users.xml",
|
||||
],
|
||||
'installable': True,
|
||||
'auto_install': False,
|
||||
"name": "test base fileurl fields",
|
||||
"summary": """A module to verify fileurl field.""",
|
||||
"version": "13.0.1.0.0",
|
||||
"category": "Tests",
|
||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||
"license": "AGPL-3",
|
||||
"depends": ["base_fileurl_field"],
|
||||
"data": ["views/res_partner.xml", "views/res_users.xml"],
|
||||
"installable": True,
|
||||
"auto_install": False,
|
||||
}
|
||||
|
||||
@@ -1,44 +1,57 @@
|
||||
# Copyright 2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
|
||||
from odoo import models, fields, api, _
|
||||
from odoo import _, api, fields, models
|
||||
from odoo.exceptions import ValidationError
|
||||
|
||||
# make pytest happy
|
||||
# in pytest context module dependencies are not loaded
|
||||
# thus FileURL is unknown
|
||||
from odoo.addons import base_fileurl_field # noqa
|
||||
|
||||
|
||||
class ResPartner(models.Model):
|
||||
|
||||
_inherit = 'res.partner'
|
||||
_inherit = "res.partner"
|
||||
|
||||
name = fields.Char()
|
||||
url_file = fields.FileURL(
|
||||
storage_location='s3',
|
||||
filename='url_file_fname',
|
||||
storage_path='partner'
|
||||
storage_location="s3", filename="url_file_fname", storage_path="partner"
|
||||
)
|
||||
url_file_fname = fields.Char()
|
||||
|
||||
url_image = fields.FileURL(
|
||||
storage_location='s3',
|
||||
filename='url_image_fname',
|
||||
storage_path='partner_image',
|
||||
storage_location="s3", filename="url_image_fname", storage_path="partner_image",
|
||||
)
|
||||
url_image_fname = fields.Char()
|
||||
|
||||
@api.constrains('url_file', 'url_file_fname')
|
||||
@api.constrains("url_file", "url_file_fname")
|
||||
def _check_url_file_fname(self):
|
||||
rec = self.search([('url_file_fname', '=', self.url_file_fname)])
|
||||
if len(rec) > 1:
|
||||
raise ValidationError(_(
|
||||
for rec in self:
|
||||
match = self.search(
|
||||
[("url_file_fname", "=", rec.url_file_fname), ("id", "!=", rec.id)],
|
||||
limit=1,
|
||||
)
|
||||
if match:
|
||||
raise ValidationError(
|
||||
_(
|
||||
"This file name is already used on an existing record. "
|
||||
"Please use another file name or delete the url_file on :\n"
|
||||
"Model: %s Id: %s" % (self._name, rec.id)
|
||||
))
|
||||
"Model: %s Id: %s" % (self._name, match.id)
|
||||
)
|
||||
)
|
||||
|
||||
@api.constrains('url_image', 'url_image_fname')
|
||||
@api.constrains("url_image", "url_image_fname")
|
||||
def _check_url_image_fname(self):
|
||||
rec = self.search([('url_image_fname', '=', self.url_image_fname)])
|
||||
if len(rec) > 1:
|
||||
raise ValidationError(_(
|
||||
for rec in self:
|
||||
match = self.search(
|
||||
[("url_image_fname", "=", rec.url_image_fname), ("id", "!=", rec.id)],
|
||||
limit=1,
|
||||
)
|
||||
if match:
|
||||
raise ValidationError(
|
||||
_(
|
||||
"This file name is already used on an existing record. "
|
||||
"Please use another file name or delete the url_image on :\n"
|
||||
"Model: %s Id: %s" % (self._name, rec.id)
|
||||
))
|
||||
"Model: %s Id: %s" % (self._name, match.id)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# Copyright 2019 Camptocamp SA
|
||||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
|
||||
from odoo import models, fields
|
||||
from odoo import fields, models
|
||||
|
||||
|
||||
class ResUsers(models.Model):
|
||||
|
||||
_inherit = 'res.users'
|
||||
_inherit = "res.users"
|
||||
|
||||
partner_url_file = fields.FileURL(related='partner_id.url_file')
|
||||
partner_url_file_fname = fields.Char(related='partner_id.url_file_fname')
|
||||
partner_url_file = fields.FileURL(related="partner_id.url_file")
|
||||
partner_url_file_fname = fields.Char(related="partner_id.url_file_fname")
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import logging
|
||||
|
||||
from odoo import models, api
|
||||
from odoo import api, models
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -14,23 +14,23 @@ class IrAttachment(models.Model):
|
||||
_inherit = "ir.attachment"
|
||||
|
||||
def _get_stores(self):
|
||||
l = ['s3']
|
||||
l += super(IrAttachment, self)._get_stores()
|
||||
return l
|
||||
stores = ["s3"]
|
||||
stores += super()._get_stores()
|
||||
return stores
|
||||
|
||||
@api.model
|
||||
def _store_file_read(self, fname, bin_size=False):
|
||||
if fname.startswith('s3://'):
|
||||
if fname.startswith("s3://"):
|
||||
return FAKE_S3_BUCKET.get(fname)
|
||||
else:
|
||||
return super(IrAttachment, self)._store_file_read(fname, bin_size)
|
||||
|
||||
@api.model
|
||||
def _store_file_write(self, key, bin_data):
|
||||
location = self.env.context.get('storage_location') or self._storage()
|
||||
if location == 's3':
|
||||
location = self.env.context.get("storage_location") or self._storage()
|
||||
if location == "s3":
|
||||
FAKE_S3_BUCKET[key] = bin_data
|
||||
filename = 's3://fake_bucket/%s' % key
|
||||
filename = "s3://fake_bucket/%s" % key
|
||||
else:
|
||||
_super = super(IrAttachment, self)
|
||||
filename = _super._store_file_write(key, bin_data)
|
||||
@@ -38,7 +38,7 @@ class IrAttachment(models.Model):
|
||||
|
||||
@api.model
|
||||
def _store_file_delete(self, fname):
|
||||
if fname.startswith('s3://'):
|
||||
if fname.startswith("s3://"):
|
||||
FAKE_S3_BUCKET.pop(fname)
|
||||
else:
|
||||
super(IrAttachment, self)._store_file_delete(fname)
|
||||
|
||||
@@ -2,38 +2,41 @@
|
||||
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
|
||||
import base64
|
||||
|
||||
from odoo.tests import TransactionCase
|
||||
from odoo.modules.module import get_module_resource
|
||||
from odoo.exceptions import ValidationError
|
||||
from odoo.modules.module import get_module_resource
|
||||
from odoo.tests import TransactionCase
|
||||
|
||||
|
||||
class TestFileUrlFields(TransactionCase):
|
||||
|
||||
def test_fileurl_fields(self):
|
||||
file_path = get_module_resource('test_base_fileurl_field', 'data',
|
||||
'sample.txt')
|
||||
image_path = get_module_resource('test_base_fileurl_field', 'data',
|
||||
'pattern.png')
|
||||
partner = self.env.ref('base.main_partner')
|
||||
with open(file_path, 'rb') as f:
|
||||
with open(image_path, 'rb') as i:
|
||||
partner.write({
|
||||
'url_file': base64.b64encode(f.read()),
|
||||
'url_file_fname': 'sample.txt',
|
||||
'url_image': base64.b64encode(i.read()),
|
||||
'url_image_fname': 'pattern.png',
|
||||
})
|
||||
file_path = get_module_resource("test_base_fileurl_field", "data", "sample.txt")
|
||||
image_path = get_module_resource(
|
||||
"test_base_fileurl_field", "data", "pattern.png"
|
||||
)
|
||||
partner = self.env.ref("base.main_partner")
|
||||
with open(file_path, "rb") as f:
|
||||
with open(image_path, "rb") as i:
|
||||
partner.write(
|
||||
{
|
||||
"url_file": base64.b64encode(f.read()),
|
||||
"url_file_fname": "sample.txt",
|
||||
"url_image": base64.b64encode(i.read()),
|
||||
"url_image_fname": "pattern.png",
|
||||
}
|
||||
)
|
||||
|
||||
with open(file_path, 'rb') as f:
|
||||
with open(file_path, "rb") as f:
|
||||
self.assertEqual(base64.decodebytes(partner.url_file), f.read())
|
||||
|
||||
with open(image_path, 'rb') as i:
|
||||
with open(image_path, "rb") as i:
|
||||
self.assertEqual(base64.decodebytes(partner.url_image), i.read())
|
||||
|
||||
partner2 = self.env.ref('base.partner_admin')
|
||||
with open(file_path, 'rb') as f:
|
||||
partner2 = self.env.ref("base.partner_admin")
|
||||
with open(file_path, "rb") as f:
|
||||
with self.assertRaises(ValidationError):
|
||||
partner2.write({
|
||||
'url_file': base64.b64encode(f.read()),
|
||||
'url_file_fname': 'sample.txt',
|
||||
})
|
||||
partner2.write(
|
||||
{
|
||||
"url_file": base64.b64encode(f.read()),
|
||||
"url_file_fname": "sample.txt",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<odoo>
|
||||
<record id="view_partner_form_inherit" model="ir.ui.view">
|
||||
<field name="name">res.partner.form.inherit</field>
|
||||
@@ -9,11 +9,15 @@
|
||||
<page name="fileurl_test" string="FileURL Test fields">
|
||||
<group string="Default widget">
|
||||
<field name="url_file" filename="url_file_fname" />
|
||||
<field name="url_file_fname" invisible="1"/>
|
||||
<field name="url_file_fname" invisible="1" />
|
||||
</group>
|
||||
<group string="Image widget">
|
||||
<field name="url_image" widget="image" filename="url_image_fname" />
|
||||
<field name="url_image_fname" invisible="1"/>
|
||||
<field
|
||||
name="url_image"
|
||||
widget="image"
|
||||
filename="url_image_fname"
|
||||
/>
|
||||
<field name="url_image_fname" invisible="1" />
|
||||
</group>
|
||||
</page>
|
||||
</xpath>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<odoo>
|
||||
<record id="view_users_form_inherit" model="ir.ui.view">
|
||||
<field name="name">res.users.form.inherit</field>
|
||||
@@ -9,7 +9,7 @@
|
||||
<page name="fileurl_test" string="FileURL Test fields">
|
||||
<group string="Default widget">
|
||||
<field name="url_file" filename="url_file_fname" />
|
||||
<field name="url_file_fname" invisible="1"/>
|
||||
<field name="url_file_fname" invisible="1" />
|
||||
</group>
|
||||
</page>
|
||||
</xpath>
|
||||
|
||||
Reference in New Issue
Block a user