Celery & Redis 설치 및 예제

EC2(ubuntu)를 기준으로 작성하였습니다.

celery(redis)설치

# (venv)
pip install "celery[redis]" redis
pip install django-celery-beat
pip install django-celery-results
pip freeze > requirements.txt

 

Supervisor 등록

# /etc/supervisor/conf.d/
vim /etc/supervisor/conf.d/myproject-celery.conf

[program:myproject_celery]
user=root
directory=/var/www/myproject/src/
command=/var/www/myproject/bin/celery -A myproject worker -l info
 
autostart=true
autorestart=true
stdout_logfile=/var/log/myproject/celery.log
stderr_logfile=/var/log/myproject/celery.err.log"

[program:myproject_celery]: supervisor에 사용될 프로세스 이름

user=root: 이 프로세스에 접근할 수 있는 권한을 가진 유저

command ...: 프로세스가 실행될 때 함께 실행될 명령어

 

# 초기화
supervisorctl reread
supervisorctl update
# 상태 확인
sudo supervisorctl status myproject_celery

 

Django - Celery default setting(Celery with Django)

# Project 구조
- myproject
    - src
        - config
            - __init__.py
            - settings.py
            - celery.py
        - red
            - tasks.py
        - manage.py


# __init__.py
# myproject/src/config/__init__.py
from __future__ import absolute_import

# settings.py
from .celery import app as celery_app  

# settings.py
INSTALLED_APPS = [
    ...
    'django.contrib.staticfiles',
    # celery setting
    'django_celery_beat',
    'django_celery_results',
]

CELERY_BROKER_URL = 'redis://localhost:6379'
CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = TIME_ZONE

 

Celery Setting

# celery.py
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery

# set the default Django settings module for the 'celery' program.
# os.environ.setdefault('DJANGO_SETTINGS_MODULE', '<config_name>.settings')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')

app = Celery('proj')  # proj - prefix

# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')

# Load task modules from all registered Django app configs.
app.autodiscover_tasks()

@app.task(bind=True)
def debug_task(self):
    print('Request: {0!r}'.format(self.request))

 

task - 비동기 작업 생성

# tasks.py
from __future__ import absolute_import, unicode_literals
import random
from celery.decorators import task

@task(name="sum_two_numbers")
def add(x, y):
    return x + y

@task(name="multiply_two_numbers")
def mul(x, y):
    total = x * (y * random.randint(3, 100))
    return total

@task(name="sum_list_numbers")
def xsum(numbers):
    return sum(numbers)

 

Celery Test

# (서버를 실행시킬 터미널 - main terminal)
# myproject/src$ celery -A <config_folder_name> worker -l info -> manage.py가 위치한 폴더에서 실행
celery -A config worker -l info

# (새로운 터미널)python shell
>>> from red.tasks import *
>>> add(1,3)
4
>>> add.delay(1,3)
<AsyncResult: 409e5e6d-1f3c-437c-8a67-6139fece62c9>

# task 동작 완료 후 main terminal
[2020-01-08 13:07:27,181: INFO/ForkPoolWorker-1] Task sum_two_numbers[409e5e6d-1f3c-437
c-8a67-6139fece62c9] succeeded in 0.010715518001234159s: 4

 

Crontab - Scheduler 등록(Contab Scheduler)

# celery.py
from celery.schedules import crontab
app.conf.beat_schedule = {
    'add-every-minute-contrab': {
        'task': 'multiply_two_numbers',
        'schedule': crontab(),
        'args': (16, 16),
    },
    'add-every-5-seconds': {
        'task': 'multiply_two_numbers',
        'schedule': 5.0,
        'args': (16, 16)
    },
    'add-every-30-seconds': {
        'task': 'tasks.add',
        'schedule': 30.0,
        'args': (16, 16)
    },
}


# terminal에 정보 출력
celery -A config beat -l info
# django db에 schedule 설정 및 result 저장
celery -A config beat -l info -S django