[PATCH 3 of 4 celerytask] Add a new celerytask-log-dir option

Philippe Pepiot philippe.pepiot at logilab.fr
Fri Jun 22 14:59:57 CEST 2018


# HG changeset patch
# User Philippe Pepiot <philippe.pepiot at logilab.fr>
# Date 1529510234 -7200
#      Wed Jun 20 17:57:14 2018 +0200
# Node ID 82130ff7c6d5d8e733c34881bc08a6b066c6a962
# Parent  8079245626055b42cee6be8de1fb41fbc15a4fb6
# Available At https://hg.logilab.org/review/cubes/celerytask
#              hg pull https://hg.logilab.org/review/cubes/celerytask -r 82130ff7c6d5
Add a new celerytask-log-dir option

We're going to introduce a new file based logger for celery task logs.
Add a new 'celerytask-log-dir' option that is used to store those logs files.

The CUBICWEB_CELERYTASK_LOGDIR option has been introduced in
cw-celerytask-helpers for non-cubicweb workers (i.e. celery workers that
doesn't require a cubicweb connexion).

We must ensure that both options are consistent and set the the appropriate
value, so a startup hook ensure the configuration is correct.

diff --git a/hooks.py b/hooks.py
--- a/hooks.py
+++ b/hooks.py
@@ -18,6 +18,9 @@
 """cubicweb-celerytask specific hooks and operations"""
 from __future__ import print_function
 
+import errno
+import os.path
+
 from celery import current_app
 import celery.task.control
 
@@ -71,3 +74,36 @@ class CeleryTaskDeletedHook(Hook):
         op = DeleteCeleryTaskOp.get_instance(self._cw)
         for task in self.entity.child_tasks():
             op.add_data(task.task_id)
+
+
+class CeleryTaskStartupHook(Hook):
+    """Initialize CUBICWEB_CELERYTASK_LOGDIR"""
+    __regid__ = 'celerytask.server-startup-hook'
+    events = ('server_startup', 'server_maintenance')
+
+    def __call__(self):
+        self.setup_celerytask_logdir(self.repo.vreg.config)
+
+    @staticmethod
+    def setup_celerytask_logdir(config):
+        celery_logdir = current_app.conf.get('CUBICWEB_CELERYTASK_LOGDIR')
+        logdir = config['celerytask-log-dir']
+        if celery_logdir and logdir and celery_logdir != logdir:
+            raise RuntimeError((
+                "You misconfigured your application by setting two different "
+                "celerytask log directories: "
+                "{} in celeryconfig and {} in cubicweb all-in-one.conf"
+            ).format(celery_logdir, logdir))
+        elif celery_logdir and not logdir:
+            logdir = config['celerytask-log-dir'] = celery_logdir
+        elif not celery_logdir:
+            if not logdir:
+                logdir = os.path.join(config.appdatahome, 'logs')
+            current_app.conf['CUBICWEB_CELERYTASK_LOGDIR'] = logdir
+            config['celerytask-log-dir'] = logdir
+        # ensure directory exist
+        try:
+            os.makedirs(logdir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST:
+                raise
diff --git a/migration/0.6.0_Any.py b/migration/0.6.0_Any.py
new file mode 100644
--- /dev/null
+++ b/migration/0.6.0_Any.py
@@ -0,0 +1,1 @@
+option_added('celerytask-log-dir')
diff --git a/site_cubicweb.py b/site_cubicweb.py
new file mode 100644
--- /dev/null
+++ b/site_cubicweb.py
@@ -0,0 +1,10 @@
+options = (
+    ('celerytask-log-dir', {
+        'type': 'string',
+        'default': None,
+        'help': ('celery task log directory; if unset, defaults to '
+                 '<config.appdatahome>/logs'),
+        'group': 'celerytask',
+        'level': 2,
+    }),
+)
diff --git a/test/test_hooks.py b/test/test_hooks.py
new file mode 100644
--- /dev/null
+++ b/test/test_hooks.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# copyright 2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr -- mailto:contact at logilab.fr
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+import shutil
+
+import celery
+from cubicweb.devtools import testlib
+from cubes.celerytask.hooks import CeleryTaskStartupHook
+
+
+class CeleryTaskStartupHookTC(testlib.CubicWebTC):
+
+    def test_create_log_dir(self):
+        """hook should set a celerytask-log-dir by default and create it"""
+        logdir = self.config['celerytask-log-dir']
+        self.assertEqual(
+            self.config['celerytask-log-dir'],
+            os.path.join(self.config.appdatahome, 'logs'))
+        shutil.rmtree(logdir, ignore_errors=True)
+        CeleryTaskStartupHook.setup_celerytask_logdir(self.config)
+        self.assertTrue(os.path.isdir(logdir))
+
+    def test_celeryconfig_log_dir(self):
+        """when CUBICWEB_CELERYTASK_LOGDIR is set in celeryconfig and no
+        celerytask-log-dir is set we must use value from
+        CUBICWEB_CELERYTASK_LOGDIR"""
+        conf = celery.current_app.conf
+        logdir = os.path.join(self.config.appdatahome, 'testhook')
+        old = (self.config['celerytask-log-dir'],
+               conf['CUBICWEB_CELERYTASK_LOGDIR'])
+        (self.config['celerytask-log-dir'],
+         conf['CUBICWEB_CELERYTASK_LOGDIR']) = None, logdir
+        try:
+            CeleryTaskStartupHook.setup_celerytask_logdir(self.config)
+            self.assertEqual(self.config['celerytask-log-dir'], logdir)
+        finally:
+            (self.config['celerytask-log-dir'],
+             conf['CUBICWEB_CELERYTASK_LOGDIR']) = old
+            os.rmdir(logdir)
+
+    def test_mismatching_dirs(self):
+        """hook should raise in case os mistmaching configuration"""
+        conf = celery.current_app.conf
+        old = conf['CUBICWEB_CELERYTASK_LOGDIR']
+        conf['CUBICWEB_CELERYTASK_LOGDIR'] = '/some/dir'
+        try:
+            with self.assertRaises(RuntimeError) as cm:
+                CeleryTaskStartupHook.setup_celerytask_logdir(self.config)
+            self.assertIn('You misconfigured your application',
+                          str(cm.exception))
+        finally:
+            conf['CUBICWEB_CELERYTASK_LOGDIR'] = old


More information about the cubicweb-devel mailing list