[PATCH 5 of 6 cw-celerytask-helpers] Move common celery signals and helpers to __init__

Philippe Pepiot philippe.pepiot at logilab.fr
Wed Jun 20 17:59:54 CEST 2018


# HG changeset patch
# User Philippe Pepiot <philippe.pepiot at logilab.fr>
# Date 1529501701 -7200
#      Wed Jun 20 15:35:01 2018 +0200
# Node ID 442a4b108acaa0c19c000615fa338366a19888d0
# Parent  9f5a9b939be34088169f808fb1459ac3161df9d8
# Available At https://hg.logilab.org/review/cw-celerytask-helpers
#              hg pull https://hg.logilab.org/review/cw-celerytask-helpers -r 442a4b108aca
Move common celery signals and helpers to __init__

Some celery signals and helpers functions from redislogger where not related
redis specially. Since we will add other backends than redis in next csets,
move them to __init__.py so we ensure they are activated when there are
'cw_celerytask_helpers.*' in CELERY_IMPORTS.

diff --git a/cw_celerytask_helpers/__init__.py b/cw_celerytask_helpers/__init__.py
--- a/cw_celerytask_helpers/__init__.py
+++ b/cw_celerytask_helpers/__init__.py
@@ -0,0 +1,53 @@
+import contextlib
+import logging
+import sys
+
+import celery
+from celery.utils.log import get_task_logger
+from celery import signals
+
+PY2 = sys.version_info[0] == 2
+
+
+ at signals.celeryd_init.connect
+def configure_worker(conf=None, **kwargs):
+    conf.setdefault('CELERY_TASK_SERIALIZER', 'json')
+    conf.setdefault('CELERY_RESULT_SERIALIZER', 'json')
+    conf.setdefault('CELERY_ACCEPT_CONTENT', ['json', 'msgpack', 'yaml'])
+    conf.setdefault('CUBICWEB_CELERYTASK_REDIS_URL',
+                    'redis://localhost:6379/0')
+
+
+ at signals.task_failure.connect
+def log_exception(**kwargs):
+    logger = logging.getLogger("celery.task")
+    if PY2:
+        einfo = str(kwargs['einfo']).decode('utf8')
+    else:
+        einfo = str(kwargs['einfo'])
+    logger.critical(u"unhandled exception:\n%s", einfo)
+
+
+ at signals.celeryd_after_setup.connect
+def setup_cubicweb_logging(conf=None, **kwargs):
+    """
+    Set parent to "celery.task" for all instantiated logger names starting with
+    "cube" or "cubicweb"
+    """
+    logall = conf.get('CUBICWEB_CELERYTASK_LOG_ALL', False)
+    for logname in logging.root.manager.loggerDict.keys():
+        if (logall or
+           logname.startswith('cubes') or logname.startswith('cubicweb')):
+            get_task_logger(logname)
+
+
+ at contextlib.contextmanager
+def redirect_stdouts(logger):
+    old_outs = sys.stdout, sys.stderr
+    try:
+        app = celery.current_app
+        rlevel = app.conf.CELERY_REDIRECT_STDOUTS_LEVEL
+        app.log.redirect_stdouts_to_logger(logger, rlevel)
+        yield
+    finally:
+        sys.stdout, sys.stderr = old_outs
diff --git a/cw_celerytask_helpers/redislogger.py b/cw_celerytask_helpers/redislogger.py
--- a/cw_celerytask_helpers/redislogger.py
+++ b/cw_celerytask_helpers/redislogger.py
@@ -4,20 +4,15 @@ Add this module 'cw_celerytask_helpers.r
 """
 from __future__ import absolute_import
 
-import contextlib
-import logging
 import json
 import datetime
-import sys
+import logging
 
-import celery
 from celery._state import get_current_task
-from celery.utils.log import get_task_logger
 from celery import signals
 
 from .utils import get_redis_client
 
-PY2 = sys.version_info[0] == 2
 LOG_KEY_PREFIX = "cw:celerytask:log"
 
 
@@ -46,19 +41,6 @@ def setup_redis_logging(conf=None, **kwa
     logger.addHandler(store_handler)
 
 
- at signals.celeryd_after_setup.connect
-def setup_cubicweb_logging(conf=None, **kwargs):
-    """
-    Set parent to "celery.task" for all instantiated logger names starting with
-    "cube" or "cubicweb"
-    """
-    logall = conf.get('CUBICWEB_CELERYTASK_LOG_ALL', False)
-    for logname in logging.root.manager.loggerDict.keys():
-        if (logall or
-           logname.startswith('cubes') or logname.startswith('cubicweb')):
-            get_task_logger(logname)
-
-
 def get_task_logs(task_id):
     """
     Get task logs by id
@@ -73,37 +55,6 @@ def flush_task_logs(task_id):
     return redis_client.delete(get_log_key(task_id))
 
 
- at contextlib.contextmanager
-def redirect_stdouts(logger):
-    old_outs = sys.stdout, sys.stderr
-    try:
-        app = celery.current_app
-        rlevel = app.conf.CELERY_REDIRECT_STDOUTS_LEVEL
-        app.log.redirect_stdouts_to_logger(logger, rlevel)
-        yield
-    finally:
-        sys.stdout, sys.stderr = old_outs
-
-
- at signals.celeryd_init.connect
-def configure_worker(conf=None, **kwargs):
-    conf.setdefault('CELERY_TASK_SERIALIZER', 'json')
-    conf.setdefault('CELERY_RESULT_SERIALIZER', 'json')
-    conf.setdefault('CELERY_ACCEPT_CONTENT', ['json', 'msgpack', 'yaml'])
-    conf.setdefault('CUBICWEB_CELERYTASK_REDIS_URL',
-                    'redis://localhost:6379/0')
-
-
- at signals.task_failure.connect
-def log_exception(**kwargs):
-    logger = logging.getLogger("celery.task")
-    if PY2:
-        einfo = str(kwargs['einfo']).decode('utf8')
-    else:
-        einfo = str(kwargs['einfo'])
-    logger.critical(u"unhandled exception:\n%s", einfo)
-
-
 class RedisFormatter(logging.Formatter):
     def format(self, record):
         """


More information about the cubicweb-devel mailing list