[PATCH s3storage] [storages] Make S3 key pattern configurable

Elouan Martinet elouan.martinet at logilab.fr
Wed Feb 12 15:19:18 CET 2020


# HG changeset patch
# User Elouan Martinet <elouan.martinet at logilab.fr>
# Date 1581516845 -3600
#      Wed Feb 12 15:14:05 2020 +0100
# Node ID e25f30a845a2b2d2c00229cc65748003cc4602b0
# Parent  c35e1a12698152addf3dec053db579af0e3bf4c0
# EXP-Topic s3-key-pattern
[storages] Make S3 key pattern configurable

diff -r c35e1a126981 -r e25f30a845a2 cubicweb_s3storage/site_cubicweb.py
--- a/cubicweb_s3storage/site_cubicweb.py	Wed Jan 29 16:53:01 2020 +0100
+++ b/cubicweb_s3storage/site_cubicweb.py	Wed Feb 12 15:14:05 2020 +0100
@@ -10,6 +10,12 @@
         'help': 'Delete S3 objects on entity deletion',
         'group': 's3',
         'level': 2}),
+    ('s3-key-pattern', {
+        'type': 'string',
+        'default': '$uuid',
+        'help': 'Define how S3 keys are generated',
+        'group': 's3',
+        'level': 2}),
     )
 
 
diff -r c35e1a126981 -r e25f30a845a2 cubicweb_s3storage/storages.py
--- a/cubicweb_s3storage/storages.py	Wed Jan 29 16:53:01 2020 +0100
+++ b/cubicweb_s3storage/storages.py	Wed Feb 12 15:14:05 2020 +0100
@@ -21,9 +21,10 @@
 import uuid
 from logging import getLogger
 
-from six import PY3
+from six import PY3, text_type
 import os
 import boto3
+from string import Template
 
 from cubicweb import Binary, set_log_methods
 from cubicweb.server.sources.storages import Storage
@@ -133,10 +134,26 @@
         return self.new_s3_key(entity, attr)
 
     def new_s3_key(self, entity, attr):
-        """Generate a new key for given entity attr.
+        """Generate a new key for given entity attr."""
+        key_pattern = entity._cw.repo.config.get('s3-key-pattern')
+        if key_pattern == '' or key_pattern is None:
+            key_pattern = '$uuid'
+
+        if '$' not in key_pattern:  # FIXME incomplete check
+            msg = 'Invalid s3-key-pattern: %s does not contain variables'
+            raise InvariableKeyPatternException(msg.format(key_pattern))
 
-        This implemenation just return a random UUID"""
-        return str(uuid.uuid1())
+        template = Template(key_pattern)
+        return template.substitute({
+            'attr': attr,
+            'eid': text_type(entity.eid),
+            'etype': entity.cw_etype,
+            'uuid': uuid.uuid1(),  # FIXME generates uuid even if unused
+        })
+
+
+class InvariableKeyPatternException(Exception):
+    pass
 
 
 set_log_methods(S3Storage,
diff -r c35e1a126981 -r e25f30a845a2 test/test_s3storage.py
--- a/test/test_s3storage.py	Wed Jan 29 16:53:01 2020 +0100
+++ b/test/test_s3storage.py	Wed Feb 12 15:14:05 2020 +0100
@@ -8,6 +8,7 @@
 from cubicweb import Binary
 from cubicweb.server.migractions import ServerMigrationHelper
 from cubicweb_s3storage import testing
+from cubicweb_s3storage.storages import InvariableKeyPatternException
 
 
 def create_image(cnx, data=b'the-data', **kwargs):
@@ -28,6 +29,19 @@
             k2 = s3storage.get_s3_key(fobj, 'data')
         self.assertEqual(k1, k2)
 
+    def test_new_s3key_entity_variables(self):
+        self.repo.vreg.config['s3-key-pattern'] = \
+            'cubicweb-data/$etype/$eid/$attr'
+        self.test_s3key_gen()
+        self.repo.vreg.config['s3-key-pattern'] = None
+
+    def test_new_s3key_invariable_key_pattern(self):
+        self.repo.vreg.config['s3-key-pattern'] = \
+            'invariable-key-pattern'
+        with self.assertRaises(InvariableKeyPatternException):
+            self.test_s3key_gen()
+        self.repo.vreg.config['s3-key-pattern'] = None
+
     def test_entity_create(self):
         with self.admin_access.client_cnx() as cnx:
             eid = create_image(cnx, b'some content').eid



More information about the cubicweb-devel mailing list