关于python：如何创建Celery Windows Service？


How to create Celery Windows Service?

我正在尝试创建Windows服务来启动Celery。我碰到过一篇使用Task Scheduler来做的文章。但是，它似乎启动了许多芹菜实例，并不断消耗内存，直到机器死机为止。有什么方法可以将其作为Windows服务启动吗？

我从另一个网站得到了答案。 Celeryd(Celery的守护程序服务)作为粘贴程序应用程序运行，在这里搜索" Paster Windows Service"会引导我。它描述了如何将Pylons应用程序作为Windows服务运行。作为Paster框架的新成员并托管python Web服务，一开始我就没想到要检查它。但是该解决方案适用于Celery，脚本中到处都有微小的变化。

我已经修改了脚本，使修改Celery设置更加容易。基本更改为：

  • 使用Celery服务的设置创建一个INI文件(如下所示)
  • 创建一个python脚本来创建Windows服务。

    • INI文件设置(celeryd.ini)：

    1
    2
    3
    4
    5
    [celery:service]
    service_name = CeleryService
    service_display_name = Celery Service
    service_description = WSCGI Windows Celery Service
    service_logfile = celeryd.log

    用于创建Windows服务的Python脚本(CeleryService.py)：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
    100
    101
    102
    103
    104
    105
    106
    107
    108
    109
    110
    111
    112
    113
    """
    The most basic (working) Windows service possible.
    Requires Mark Hammond's pywin32 package.  
    Most of the code was taken from a  CherryPy 2.2 example of how to set up a service
    """
    import pkg_resources
    import win32serviceutil
    from paste.script.serve import ServeCommand as Server
    import os, sys
    import ConfigParser

    import win32service
    import win32event

    SCRIPT_DIR          = os.path.abspath(os.path.dirname(__file__))
    INI_FILE            = 'celeryd.ini'
    SERV_SECTION        = 'celery:service'
    SERV_NAME           = 'service_name'
    SERV_DISPLAY_NAME   = 'service_display_name'
    SERV_DESC           = 'service_description'
    SERV_LOG_FILE       = 'service_logfile'
    SERV_APPLICATION    = 'celeryd'
    SERV_LOG_FILE_VAR   = 'CELERYD_LOG_FILE'

    # Default Values
    SERV_NAME_DEFAULT           = 'CeleryService'
    SERV_DISPLAY_NAME_DEFAULT   = 'Celery Service'
    SERV_DESC_DEFAULT           = 'WSCGI Windows Celery Service'
    SERV_LOG_FILE_DEFAULT       = r'D:\\logs\\celery.log'

    class DefaultSettings(object):
        def __init__(self):
            if SCRIPT_DIR:
                os.chdir(SCRIPT_DIR)
            # find the ini file
            self.ini = os.path.join(SCRIPT_DIR,INI_FILE)
            # create a config parser opject and populate it with the ini file
            c = ConfigParser.SafeConfigParser()
            c.read(self.ini)
            self.c = c

        def getDefaults(self):
            '''
            Check for and get the default settings
            '''
            if (
                (not self.c.has_section(SERV_SECTION)) or
                (not self.c.has_option(SERV_SECTION, SERV_NAME)) or
                (not self.c.has_option(SERV_SECTION, SERV_DISPLAY_NAME)) or
                (not self.c.has_option(SERV_SECTION, SERV_DESC)) or
                (not self.c.has_option(SERV_SECTION, SERV_LOG_FILE))
                ):
                print 'setting defaults'
                self.setDefaults()
            service_name = self.c.get(SERV_SECTION, SERV_NAME)
            service_display_name = self.c.get(SERV_SECTION, SERV_DISPLAY_NAME)
            service_description = self.c.get(SERV_SECTION, SERV_DESC)
            iniFile = self.ini
            service_logfile = self.c.get(SERV_SECTION, SERV_LOG_FILE)
            return service_name, service_display_name, service_description, iniFile, service_logfile

        def setDefaults(self):
            '''
            set and add the default setting to the ini file
            '''
            if not self.c.has_section(SERV_SECTION):
                self.c.add_section(SERV_SECTION)
            self.c.set(SERV_SECTION, SERV_NAME, SERV_NAME_DEFAULT)
            self.c.set(SERV_SECTION, SERV_DISPLAY_NAME, SERV_DISPLAY_NAME_DEFAULT)
            self.c.set(SERV_SECTION, SERV_DESC, SERV_DESC_DEFAULT)
            self.c.set(SERV_SECTION, SERV_LOG_FILE, SERV_LOG_FILE_DEFAULT)
            cfg = file(self.ini, 'wr')
            self.c.write(cfg)
            cfg.close()
            print '''
    you must set the celery:service section service_name, service_display_name,
    and service_description options to define the service
    in the %s file
    '''     % self.ini
            sys.exit()


    class CeleryService(win32serviceutil.ServiceFramework):
       """NT Service."""

        d = DefaultSettings()
        service_name, service_display_name, service_description, iniFile, logFile = d.getDefaults()

        _svc_name_ = service_name
        _svc_display_name_ = service_display_name
        _svc_description_ = service_description

        def __init__(self, args):
            win32serviceutil.ServiceFramework.__init__(self, args)
            # create an event that SvcDoRun can wait on and SvcStop
            # can set.
            self.stop_event = win32event.CreateEvent(None, 0, 0, None)

        def SvcDoRun(self):
            os.chdir(SCRIPT_DIR)
            s = Server(SERV_APPLICATION)
            os.environ[SERV_LOG_FILE_VAR] = self.logFile
            s.run([self.iniFile])
            win32event.WaitForSingleObject(self.stop_event, win32event.INFINITE)

        def SvcStop(self):
            self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
            #win32event.SetEvent(self.stop_event)
            self.ReportServiceStatus(win32service.SERVICE_STOPPED)
            sys.exit()

    if __name__ == '__main__':
        win32serviceutil.HandleCommandLine(CeleryService)

    要安装该服务，请运行python CeleryService.py install，然后运行python CeleryService.py start以启动该服务。注意：这些命令应在具有管理员权限的命令行中运行。

    如果需要删除服务，请运行python CeleryService.py remove。

    我正在尝试托管Celery，作为增强RhodeCode安装的一部分。此解决方案似乎有效。希望这会对某人有所帮助。

    相关讨论

    可接受的答案不适用于使用Django应用程序运行celery。但这启发了我想出一个解决方案，用于将celery作为Django的Windows服务运行。请注意，以下内容仅适用于Django项目。经过一些修改，它可以与其他应用程序一起使用。

    以下讨论假定已安装Python> = 3.6和RabbitMQ，并且rabbitmq-server在localhost上运行。

    在Django项目的顶级文件夹中(与manage.py处于同一级别)创建一个文件celery_service.py(或您喜欢的任何文件)，其内容如下：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    '''Usage : python celery_service.py install (start / stop / remove)
    Run celery as a Windows service
    '''
    import win32service
    import win32serviceutil
    import win32api
    import win32con
    import win32event
    import subprocess
    import sys
    import os
    from pathlib import Path
    import shlex
    import logging
    import time

    # The directory for celery.log and celery_service.log
    # Default: the directory of this script
    INSTDIR = Path(__file__).parent
    # The path of python Scripts
    # Usually it is in path_to/venv/Scripts.
    # If it is already in system PATH, then it can be set as ''
    PYTHONSCRIPTPATH = INSTDIR / 'venvcelery/Scripts'
    # The directory name of django project
    # Note: it is the directory at the same level of manage.py
    # not the parent directory
    PROJECTDIR = 'proj'

    logging.basicConfig(
        filename = INSTDIR / 'celery_service.log',
        level = logging.DEBUG,
        format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
    )

    class CeleryService(win32serviceutil.ServiceFramework):

        _svc_name_ ="Celery"
        _svc_display_name_ ="Celery Distributed Task Queue Service"

        def __init__(self, args):
            win32serviceutil.ServiceFramework.__init__(self, args)
            self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)          

        def SvcStop(self):
            logging.info('Stopping {name} service ...'.format(name=self._svc_name_))        
            self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
            win32event.SetEvent(self.hWaitStop)
            self.ReportServiceStatus(win32service.SERVICE_STOPPED)
            sys.exit()          

        def SvcDoRun(self):
            logging.info('Starting {name} service ...'.format(name=self._svc_name_))
            os.chdir(INSTDIR) # so that proj worker can be found
            logging.info('cwd: ' + os.getcwd())
            self.ReportServiceStatus(win32service.SERVICE_RUNNING)
            command = '"{celery_path}" -A {proj_dir} worker -f"{log_path}" -l info -P eventlet'.format(
                celery_path=PYTHONSCRIPTPATH / 'celery.exe',
                proj_dir=PROJECTDIR,
                log_path=INSTDIR / 'celery.log')
            logging.info('command: ' + command)
            args = shlex.split(command)
            proc = subprocess.Popen(args)
            logging.info('pid: {pid}'.format(pid=proc.pid))
            self.timeout = 3000
            while True:
                rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
                if rc == win32event.WAIT_OBJECT_0:
                    # stop signal encountered
                    # terminate process 'proc'
                    PROCESS_TERMINATE = 1
                    handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
                    win32api.TerminateProcess(handle, -1)
                    win32api.CloseHandle(handle)                
                    break

    if __name__ == '__main__':
       win32serviceutil.HandleCommandLine(CeleryService)

    在脚本可以运行之前，您需要

  • (可选)创建python虚拟环境，例如" venvcelery"。

  • 安装以下要求：

    django> = 2.0.0
    sqlalchemy> = 1.0.14
    芹菜> = 4.3.0，<5.0 pywin32> = 227
    eventlet> = 0.25

  • 修复pywin32 pywintypes36.dll位置。参考

  • 在celery_service.py

    中正确设置PYTHONSCRIPTPATH和PROJECTDIR

    • PYTHONSCRIPTPATH通常是python安装路径或当前虚拟环境下的"脚本"文件夹。

    PROJECTDIR是Django项目的目录名称。

    它是与manage.py处于同一级别的目录，而不是父目录。

    现在您可以使用以下方式安装/启动/停止/删除服务：

    1
    2
    3
    4
    python celery_service.py install
    python celery_service.py start
    python celery_service.py stop
    python celery_service.py remove

    我创建了一个演示Django项目，其中celery作为Windows服务运行：

    https://github.com/azalea/django_celery_windows_service

    如果您对正在运行的示例感兴趣。

    注意：这是更新的版本，假定Python> = 3.6，Django 2.2和Celery 4。

    可以在编辑历史记录中查看具有Python 2.7，Django 1.6和Celery 3的旧版本。

    相关讨论

    @azalea的答案对我有很大帮助，但我想在这里强调的一件事是，该服务(celery_service.py)需要与您的用户名/密码一起安装，否则，当您运行subprocess.Popen(args) in SvcDoRun()函数时，什么都不会发生，因为会出现权限问题。要设置用户名/密码，可以选择以下两种方法之一：

  • 使用命令行：

    1
    python33 .\\celeryService1.py --username .\\USERNAME --password PASSWORD

  • 转到"计算机管理(本地)>服务和应用程序>服务"，找到您的服务器(在@azalea的示例中，它是" Celery Distributed Task Queue Service")，然后右键单击以打开"属性"页面，输入"此帐户"在登录选项卡中

    • 感谢Azalea，因为这使我进入了能够在Windows上使用Celery 4创建2个Windows服务的道路。

    一个可以启动/停止多个工作人员的T
    其次，可以启动/停止节拍服务并使用Celery 4整理pid。

    对此，我没有解决的唯一警告是，您无法重新启动工作进程，因为您需要确保在启动备份之前已停止多个复活的生成进程。

    Workers.py：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    '''Usage : python celery_service.py install (start / stop / remove)
    Run celery as a Windows service
    '''
    import win32service
    import win32serviceutil
    import win32api
    import win32con
    import win32event
    import subprocess
    import sys
    import os
    import shlex
    import logging
    import time

    # The directory for celery_worker.log and celery_worker_service.log
    # Default: the directory of this script
    INSTDIR = 'X:\\Application\\Project'
    LOGDIR = 'X:\\Application\\LogFiles'
    # The path of python Scripts
    # Usually it is in PYTHON_INSTALL_DIR/Scripts. e.g.
    # r'C:\\Python27\\Scripts'
    # If it is already in system PATH, then it can be set as ''
    PYTHONSCRIPTPATH = 'C:\\Python36\\Scripts'
    # The directory name of django project
    # Note: it is the directory at the same level of manage.py
    # not the parent directory
    PROJECTDIR = 'Project'

    logging.basicConfig(
        filename = os.path.join(LOGDIR, 'celery_worker_service.log'),
        level = logging.DEBUG,
        format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
    )

    class CeleryService(win32serviceutil.ServiceFramework):

        _svc_name_ ="CeleryWorkers"
        _svc_display_name_ ="CeleryWorkers"

        def __init__(self, args):
            win32serviceutil.ServiceFramework.__init__(self, args)
            self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)

        def SvcStop(self):
            logging.info('Stopping {name} service ...'.format(name=self._svc_name_))        
            self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
            win32event.SetEvent(self.hWaitStop)
            logging.info('Stopped1 {name} service ...'.format(name=self._svc_name_))  
            logging.info('Stopped3 {name} service ...'.format(name=self._svc_name_))
            command = '"{celery_path}" -A {proj_dir} --workdir=X:/Application/Project control shutdown --timeout=10'.format(
            celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
            proj_dir=PROJECTDIR,
            log_path=os.path.join(LOGDIR,'celery_worker.log'))
            logging.info('command: ' + command)
            args = shlex.split(command)
            proc = subprocess.Popen(args)
            logging.info('Stopped celery shutdown  ...')
            self.ReportServiceStatus(win32service.SERVICE_STOPPED)
            logging.info('Stopped2 {name} service ...'.format(name=self._svc_name_))  
            sys.exit()          

        def SvcDoRun(self):
            logging.info('Starting {name} service ...'.format(name=self._svc_name_))
            os.chdir(INSTDIR) # so that proj worker can be found
            logging.info('cwd: ' + os.getcwd())
            self.ReportServiceStatus(win32service.SERVICE_RUNNING)
            command = '"{celery_path}" -A {proj_dir} -c 8 worker --workdir=X:/Application/Project --pidfile=celeryservice.pid  -f"{log_path}" -l info'.format(
                celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
                proj_dir=PROJECTDIR,
                log_path=os.path.join(LOGDIR,'celery_worker.log'))
            logging.info('command: ' + command)
            args = shlex.split(command)
            proc = subprocess.Popen(args)
            logging.info('pid: {pid}'.format(pid=proc.pid))
            self.timeout = 3000
            while True:
                rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
                if rc == win32event.WAIT_OBJECT_0:
                    # stop signal encountered
                    # terminate process 'proc'
                    PROCESS_TERMINATE = 1
                    handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
                    win32api.TerminateProcess(handle, -1)
                    win32api.CloseHandle(handle)                
                    break

    if __name__ == '__main__':
       win32serviceutil.HandleCommandLine(CeleryService)

    Beatservice.py：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    '''Usage : python celery_service.py install (start / stop / remove)
    Run celery as a Windows service
    '''
    import win32service
    import win32serviceutil
    import win32api
    import win32con
    import win32event
    import subprocess
    import sys
    import os
    import shlex
    import logging
    import time
    import signal

    # The directory for celery_beat.log and celery_beat_service.log
    # Default: the directory of this script
    INSTDIR = os.path.dirname(os.path.realpath(__file__))
    LOGPATH = 'X:\\Application\\Logs'
    # The path of python Scripts
    # Usually it is in PYTHON_INSTALL_DIR/Scripts. e.g.
    # r'C:\\Python27\\Scripts'
    # If it is already in system PATH, then it can be set as ''
    PYTHONSCRIPTPATH = 'C:\\Python36\\Scripts'
    # The directory name of django project
    # Note: it is the directory at the same level of manage.py
    # not the parent directory
    PROJECTDIR = 'PROJECT'

    logging.basicConfig(
        filename = os.path.join(LOGPATH, 'celery_beat_service.log'),
        level = logging.DEBUG,
        format = '[%(asctime)-15s: %(levelname)-7.7s] %(message)s'
    )

    class CeleryService(win32serviceutil.ServiceFramework):

        _svc_name_ ="CeleryBeat"
        _svc_display_name_ ="CeleryBeat"

        def __init__(self, args):
            win32serviceutil.ServiceFramework.__init__(self, args)
            self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)          

        def SvcStop(self):
            logging.info('Stopping 1 {name} service ...'.format(name=self._svc_name_))        
            self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
            win32event.SetEvent(self.hWaitStop)
            pidno = open("X:\\Aplication\\Project\\celerybeat.pid","r")
            _pid_id_ = pidid=pidno.read()
            pidno.close()
            logging.info(_pid_id_)
            logging.info('taskkill /F /PID {pidid} ..'.format(pidid=_pid_id_))
            cmdcom = 'taskkill /F /PID {pidid}'.format(pidid=_pid_id_)
            logging.info(cmdcom)
            killargs = shlex.split(cmdcom)
            process = subprocess.Popen(killargs)
            output, error = process.communicate()
            logging.info(output)
            logging.info('Stopping 2 {name} service ...'.format(name=self._svc_name_))
            os.remove("X:\\Application\\PROJECT\\celerybeat.pid")
            logging.info('X:\\Application\\PROJECT\\celerybeat.pid  file removed')
            self.ReportServiceStatus(win32service.SERVICE_STOPPED)
            sys.exit()

        def SvcDoRun(self):
            logging.info('Starting {name} service ...'.format(name=self._svc_name_))
            os.chdir(INSTDIR) # so that proj worker can be found
            logging.info('cwd: ' + os.getcwd())
            self.ReportServiceStatus(win32service.SERVICE_RUNNING)
            command = '"{celery_path}" -A {proj_dir} beat --workdir=X:/Application/Project -f X:/Application/logs/beat.log -l info'.format(
                celery_path=os.path.join(PYTHONSCRIPTPATH, 'celery.exe'),
                proj_dir=PROJECTDIR,
                log_path=os.path.join(LOGPATH,'celery_beat.log'))
            logging.info('command: ' + command)
            args = shlex.split(command)
            proc = subprocess.Popen(args)
            logging.info('pid: {pid}'.format(pid=proc.pid))
            self.timeout = 3000
            while True:
                rc = win32event.WaitForSingleObject(self.hWaitStop, self.timeout)
                if rc == win32event.WAIT_OBJECT_0:
                    # stop signal encountered
                    # terminate process 'proc'
                    PROCESS_TERMINATE = 1
                    handle = win32api.OpenProcess(PROCESS_TERMINATE, False, proc.pid)
                    win32api.TerminateProcess(handle, -1)
                    win32api.CloseHandle(handle)                
                    break

    if __name__ == '__main__':
       win32serviceutil.HandleCommandLine(CeleryService)

    这里是一个很好的项目，但是没有成功使用它：
    链接到django-windows-tools的GitHub。
    它在最后一个命令行给了我超时。没有足够的时间来搜索原因。

    该软件包允许在IIS上设置Django项目的FastCGI，Celery和Static文件。