[PATCH s3storage] [storages] Make S3 key pattern configurable

Nicola Spanti nicola.spanti at logilab.fr
Fri Feb 14 11:20:24 CET 2020


Le 12/02/2020 à 15:19, Elouan Martinet a écrit :
> # HG changeset patch
> # User Elouan Martinet <elouan.martinet at logilab.fr>
> # Date 1581516845 -3600
> #      Wed Feb 12 15:14:05 2020 +0100
> # Node ID e25f30a845a2b2d2c00229cc65748003cc4602b0
> # Parent  c35e1a12698152addf3dec053db579af0e3bf4c0
> # EXP-Topic s3-key-pattern
> [storages] Make S3 key pattern configurable
> 
> diff -r c35e1a126981 -r e25f30a845a2 cubicweb_s3storage/site_cubicweb.py
> --- a/cubicweb_s3storage/site_cubicweb.py	Wed Jan 29 16:53:01 2020 +0100
> +++ b/cubicweb_s3storage/site_cubicweb.py	Wed Feb 12 15:14:05 2020 +0100
> @@ -10,6 +10,12 @@
>           'help': 'Delete S3 objects on entity deletion',
>           'group': 's3',
>           'level': 2}),
> +    ('s3-key-pattern', {
> +        'type': 'string',
> +        'default': '$uuid',
> +        'help': 'Define how S3 keys are generated',
> +        'group': 's3',
> +        'level': 2}),
>       )
>   
>   
> diff -r c35e1a126981 -r e25f30a845a2 cubicweb_s3storage/storages.py
> --- a/cubicweb_s3storage/storages.py	Wed Jan 29 16:53:01 2020 +0100
> +++ b/cubicweb_s3storage/storages.py	Wed Feb 12 15:14:05 2020 +0100
> @@ -21,9 +21,10 @@
>   import uuid
>   from logging import getLogger
>   
> -from six import PY3
> +from six import PY3, text_type
>   import os
>   import boto3
> +from string import Template
>   
>   from cubicweb import Binary, set_log_methods
>   from cubicweb.server.sources.storages import Storage
> @@ -133,10 +134,26 @@
>           return self.new_s3_key(entity, attr)
>   
>       def new_s3_key(self, entity, attr):
> -        """Generate a new key for given entity attr.
> +        """Generate a new key for given entity attr."""
> +        key_pattern = entity._cw.repo.config.get('s3-key-pattern')
> +        if key_pattern == '' or key_pattern is None:
> +            key_pattern = '$uuid'
> +
> +        if '$' not in key_pattern:  # FIXME incomplete check

C'est normal d'avoir laissé un FIXME ? Si oui et que ce serait vraiment 
long à faire, ce serait bien d'indiquer dans le code en quoi c'est 
incomplet.

> +            msg = 'Invalid s3-key-pattern: %s does not contain variables'
> +            raise InvariableKeyPatternException(msg.format(key_pattern))
>   
> -        This implemenation just return a random UUID"""
> -        return str(uuid.uuid1())
> +        template = Template(key_pattern)
> +        return template.substitute({
> +            'attr': attr,
> +            'eid': text_type(entity.eid),
> +            'etype': entity.cw_etype,
> +            'uuid': uuid.uuid1(),  # FIXME generates uuid even if unused
> +        })
> +
> +
> +class InvariableKeyPatternException(Exception):
> +    pass
>   
>   
>   set_log_methods(S3Storage,
> diff -r c35e1a126981 -r e25f30a845a2 test/test_s3storage.py
> --- a/test/test_s3storage.py	Wed Jan 29 16:53:01 2020 +0100
> +++ b/test/test_s3storage.py	Wed Feb 12 15:14:05 2020 +0100
> @@ -8,6 +8,7 @@
>   from cubicweb import Binary
>   from cubicweb.server.migractions import ServerMigrationHelper
>   from cubicweb_s3storage import testing
> +from cubicweb_s3storage.storages import InvariableKeyPatternException
>   
>   
>   def create_image(cnx, data=b'the-data', **kwargs):
> @@ -28,6 +29,19 @@
>               k2 = s3storage.get_s3_key(fobj, 'data')
>           self.assertEqual(k1, k2)
>   
> +    def test_new_s3key_entity_variables(self):
> +        self.repo.vreg.config['s3-key-pattern'] = \
> +            'cubicweb-data/$etype/$eid/$attr'
> +        self.test_s3key_gen()
> +        self.repo.vreg.config['s3-key-pattern'] = None
> +
> +    def test_new_s3key_invariable_key_pattern(self):
> +        self.repo.vreg.config['s3-key-pattern'] = \
> +            'invariable-key-pattern'
> +        with self.assertRaises(InvariableKeyPatternException):
> +            self.test_s3key_gen()
> +        self.repo.vreg.config['s3-key-pattern'] = None
> +
>       def test_entity_create(self):
>           with self.admin_access.client_cnx() as cnx:
>               eid = create_image(cnx, b'some content').eid
> 



More information about the cubicweb-devel mailing list