From 8cc9daa4a8c9b055756e85b0ff639078ba62c8a3 Mon Sep 17 00:00:00 2001 From: Anna Janiszewska Date: Mon, 30 May 2022 14:37:57 +0200 Subject: [PATCH 1/3] Add s3_bucket_exists on storage backend --- storage_backend_s3/components/s3_adapter.py | 18 +++++++----------- storage_backend_s3/models/storage_backend.py | 5 +++++ .../views/backend_storage_view.xml | 10 ++++++++++ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/storage_backend_s3/components/s3_adapter.py b/storage_backend_s3/components/s3_adapter.py index 703741cd9a..a02c72ffc3 100644 --- a/storage_backend_s3/components/s3_adapter.py +++ b/storage_backend_s3/components/s3_adapter.py @@ -21,10 +21,6 @@ _logger.debug(err) -# Keep track of existing buckets and avoid checking for them every time. -EXISTING_BUCKETS = [] - - class S3StorageAdapter(Component): _name = "s3.adapter" _inherit = "base.storage.adapter" @@ -60,19 +56,19 @@ def _get_or_create_bucket(self, s3, bucket_name, **params): else: create_params = self._get_create_bucket_params(bucket_name, params) bucket = s3.create_bucket(**create_params) + self.collection.s3_bucket_exists = True return bucket - def _check_bucket_exists(self, s3, bucket_name): - if bucket_name in EXISTING_BUCKETS: - # If the bucket is not available later - # we'll have an error on each call of course - # but that does not happen often - # and in any case you will get what's wrong soon. + def _check_bucket_exists(self, s3, bucket_name, force=False): + if self.env.context.get("force_s3_check"): + force = True + if force is False and self.collection.s3_bucket_exists: return True + # if test is ok, set the flag try: s3.meta.client.head_bucket(Bucket=bucket_name) # The call above is expensive, avoid it when possible. - EXISTING_BUCKETS.append(bucket_name) + self.collection.s3_bucket_exists = True return True except ClientError as e: # If a client error is thrown, then check that it was a 404 error. diff --git a/storage_backend_s3/models/storage_backend.py b/storage_backend_s3/models/storage_backend.py index 6a0d04a706..87efb9f00c 100644 --- a/storage_backend_s3/models/storage_backend.py +++ b/storage_backend_s3/models/storage_backend.py @@ -43,6 +43,7 @@ class StorageBackend(models.Model): "eg: Exoscale", ) aws_bucket = fields.Char(string="Bucket") + s3_bucket_exists = fields.Boolean(string="Bucket exists") aws_access_key_id = fields.Char(string="Access Key ID") aws_secret_access_key = fields.Char(string="Secret Access Key") aws_region = fields.Selection(selection="_selection_aws_region", string="Region") @@ -84,3 +85,7 @@ def _selection_aws_region(self): + AWS_REGIONS + [("other", "Empty or Other (Manually specify below)")] ) + + def action_ensure_bucket_exists(self): + adapter = self._get_adapter() + adapter._get_bucket() diff --git a/storage_backend_s3/views/backend_storage_view.xml b/storage_backend_s3/views/backend_storage_view.xml index 80a0df38d2..a1cbf3c1d6 100644 --- a/storage_backend_s3/views/backend_storage_view.xml +++ b/storage_backend_s3/views/backend_storage_view.xml @@ -4,6 +4,15 @@ storage.backend + + From 3e0f85146fef01dcbee81dad397706becdb16f7f Mon Sep 17 00:00:00 2001 From: Anna Janiszewska Date: Tue, 31 May 2022 15:05:35 +0200 Subject: [PATCH 2/3] Add validation for s3 --- storage_backend_s3/components/s3_adapter.py | 10 ++++-- storage_backend_s3/models/storage_backend.py | 34 ++++++++++++++++--- .../views/backend_storage_view.xml | 11 +----- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/storage_backend_s3/components/s3_adapter.py b/storage_backend_s3/components/s3_adapter.py index a02c72ffc3..2bc99e2054 100644 --- a/storage_backend_s3/components/s3_adapter.py +++ b/storage_backend_s3/components/s3_adapter.py @@ -46,6 +46,7 @@ def _get_bucket(self): s3 = boto3.resource("s3", **params) bucket_name = self.collection.aws_bucket bucket = self._get_or_create_bucket(s3, bucket_name, **params) + self.collection.validated = True return bucket def _get_or_create_bucket(self, s3, bucket_name, **params): @@ -60,9 +61,7 @@ def _get_or_create_bucket(self, s3, bucket_name, **params): return bucket def _check_bucket_exists(self, s3, bucket_name, force=False): - if self.env.context.get("force_s3_check"): - force = True - if force is False and self.collection.s3_bucket_exists: + if force is False and self.collection.validated: return True # if test is ok, set the flag try: @@ -142,3 +141,8 @@ def list(self, relative_path): def delete(self, relative_path): s3object = self._get_object(relative_path) s3object.delete() + + def validate_config(self, **params): + s3 = boto3.resource("s3", **params) + bucket_name = self.collection.aws_bucket + self.validated = self._check_bucket_exists(s3, bucket_name) diff --git a/storage_backend_s3/models/storage_backend.py b/storage_backend_s3/models/storage_backend.py index 87efb9f00c..e19535fcf1 100644 --- a/storage_backend_s3/models/storage_backend.py +++ b/storage_backend_s3/models/storage_backend.py @@ -6,7 +6,7 @@ import logging -from odoo import fields, models +from odoo import _, fields, models _logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ class StorageBackend(models.Model): "eg: Exoscale", ) aws_bucket = fields.Char(string="Bucket") - s3_bucket_exists = fields.Boolean(string="Bucket exists") + validated = fields.Boolean(string="Bucket exists", compute="_compute_s3_validated") aws_access_key_id = fields.Char(string="Access Key ID") aws_secret_access_key = fields.Char(string="Secret Access Key") aws_region = fields.Selection(selection="_selection_aws_region", string="Region") @@ -62,6 +62,12 @@ class StorageBackend(models.Model): ] ) + def _compute_s3_validated(self): + for rec in self: + if rec.backend_type == "amazon_s3": + adapter = self._get_adapter() + rec.validated = hasattr(adapter, "validate_config") + @property def _server_env_fields(self): env_fields = super()._server_env_fields @@ -86,6 +92,26 @@ def _selection_aws_region(self): + [("other", "Empty or Other (Manually specify below)")] ) - def action_ensure_bucket_exists(self): + def action_test_config(self): + if not self.validated: + raise AttributeError("Validation not supported!") adapter = self._get_adapter() - adapter._get_bucket() + try: + adapter.validate_config() + title = _("Connection Test Succeeded!") + message = _("Everything seems properly set up!") + msg_type = "success" + except Exception as err: + title = _("Connection Test Failed!") + message = str(err) + msg_type = "danger" + return { + "type": "ir.actions.client", + "tag": "display_notification", + "params": { + "title": title, + "message": message, + "type": msg_type, + "sticky": False, + }, + } diff --git a/storage_backend_s3/views/backend_storage_view.xml b/storage_backend_s3/views/backend_storage_view.xml index a1cbf3c1d6..dec2d162e7 100644 --- a/storage_backend_s3/views/backend_storage_view.xml +++ b/storage_backend_s3/views/backend_storage_view.xml @@ -4,15 +4,6 @@ storage.backend - - + From 122d40021f599732d2960a7c96c22ceb6770e381 Mon Sep 17 00:00:00 2001 From: Anna Janiszewska Date: Tue, 31 May 2022 16:02:02 +0200 Subject: [PATCH 3/3] fixup! Add validation for s3 --- storage_backend/models/storage_backend.py | 5 +-- storage_backend_s3/components/s3_adapter.py | 4 +-- storage_backend_s3/models/storage_backend.py | 33 +------------------- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/storage_backend/models/storage_backend.py b/storage_backend/models/storage_backend.py index 0cabba4c28..1ab6987983 100644 --- a/storage_backend/models/storage_backend.py +++ b/storage_backend/models/storage_backend.py @@ -68,11 +68,12 @@ class StorageBackend(models.Model): help="Relative path to the directory to store the file" ) has_validation = fields.Boolean(compute="_compute_has_validation") - + validated = fields.Boolean(string="Validated",compute="_compute_has_validation") + def _compute_has_validation(self): for rec in self: adapter = self._get_adapter() - rec.has_validation = hasattr(adapter, "validate_config") + rec.validated = rec.has_validation = hasattr(adapter, "validate_config") @property def _server_env_fields(self): diff --git a/storage_backend_s3/components/s3_adapter.py b/storage_backend_s3/components/s3_adapter.py index 2bc99e2054..12b6020235 100644 --- a/storage_backend_s3/components/s3_adapter.py +++ b/storage_backend_s3/components/s3_adapter.py @@ -57,7 +57,7 @@ def _get_or_create_bucket(self, s3, bucket_name, **params): else: create_params = self._get_create_bucket_params(bucket_name, params) bucket = s3.create_bucket(**create_params) - self.collection.s3_bucket_exists = True + self.collection.validated = True return bucket def _check_bucket_exists(self, s3, bucket_name, force=False): @@ -67,7 +67,7 @@ def _check_bucket_exists(self, s3, bucket_name, force=False): try: s3.meta.client.head_bucket(Bucket=bucket_name) # The call above is expensive, avoid it when possible. - self.collection.s3_bucket_exists = True + self.collection.validated = True return True except ClientError as e: # If a client error is thrown, then check that it was a 404 error. diff --git a/storage_backend_s3/models/storage_backend.py b/storage_backend_s3/models/storage_backend.py index e19535fcf1..6a0d04a706 100644 --- a/storage_backend_s3/models/storage_backend.py +++ b/storage_backend_s3/models/storage_backend.py @@ -6,7 +6,7 @@ import logging -from odoo import _, fields, models +from odoo import fields, models _logger = logging.getLogger(__name__) @@ -43,7 +43,6 @@ class StorageBackend(models.Model): "eg: Exoscale", ) aws_bucket = fields.Char(string="Bucket") - validated = fields.Boolean(string="Bucket exists", compute="_compute_s3_validated") aws_access_key_id = fields.Char(string="Access Key ID") aws_secret_access_key = fields.Char(string="Secret Access Key") aws_region = fields.Selection(selection="_selection_aws_region", string="Region") @@ -62,12 +61,6 @@ class StorageBackend(models.Model): ] ) - def _compute_s3_validated(self): - for rec in self: - if rec.backend_type == "amazon_s3": - adapter = self._get_adapter() - rec.validated = hasattr(adapter, "validate_config") - @property def _server_env_fields(self): env_fields = super()._server_env_fields @@ -91,27 +84,3 @@ def _selection_aws_region(self): + AWS_REGIONS + [("other", "Empty or Other (Manually specify below)")] ) - - def action_test_config(self): - if not self.validated: - raise AttributeError("Validation not supported!") - adapter = self._get_adapter() - try: - adapter.validate_config() - title = _("Connection Test Succeeded!") - message = _("Everything seems properly set up!") - msg_type = "success" - except Exception as err: - title = _("Connection Test Failed!") - message = str(err) - msg_type = "danger" - return { - "type": "ir.actions.client", - "tag": "display_notification", - "params": { - "title": title, - "message": message, - "type": msg_type, - "sticky": False, - }, - }