celery integration & basic markup task

markup/tasks.py

@@ -0,0 +1,95 @@
+from __future__ import absolute_import, unicode_literals
+from celery import task, shared_task
+import sys
+import datetime
+import fileinput
+import smtplib
+#from mailbox import Maildir
+from email.feedparser import FeedParser
+from email.message import EmailMessage
+
+#from demoapp.models import Widget
+#from celery import Celery
+#app = Celery('procat2', broker='amqp://localhost')
+
+#@app.task
+
+@shared_task
+def add(x, y):
+    return x + y
+
+body_ok = """Hi,
+
+Attached is your marked up catalog.  In the near future you'll also
+get a spreadsheet and a new catalog with just the marked up items.
+
+Enjoy,
+ProCatalog Markup Bot"""
+
+body_missing = """Hi,
+
+I couldn't find a pdf attached to your message.  I can't do much
+without a marked up catalog pdf, so please include that when you try
+again.
+
+Thanks,
+ProCatalog Markup Bot"""
+
+
+# def log_message(msg):
+#     with open('/var/log/markup_submit.log', 'a') as f:
+#         f.write("{} - {} / {}\n".format(datetime.datetime.now().isoformat(),
+#                                         msg['From'], msg['Subject']))
+
+def reply(frm, subj, attach):
+    print('ok: {} - {}'.format(frm, subj))
+    # msg = EmailMessage()
+    # msg.set_content(body_ok)
+    # msg.add_attachment(attach.get_payload(decode=True),
+    #                    maintype=attach.get_content_maintype(),
+    #                    subtype=attach.get_content_subtype(),
+    #                    filename=attach.get_filename())
+    # send(frm, subj, msg)
+
+
+def reply_missing(frm, subj):
+    print('missing: {} - {}'.format(frm, subj))
+    # msg = EmailMessage()
+    # msg.set_content(body_missing)
+    # send(frm, subj, msg)
+
+
+def send(frm, subj, msg):
+    msg['From'] = 'Keen ProCatalog Markup Bot <support@procatalog.io>'
+    msg['To'] = frm
+    msg['Bcc'] = 'alx-markup@procatalog.io'
+    msg['Subject'] = 'Re: {}'.format(subj)
+
+    maildir = Maildir('/tmp/markup_submit_mail')
+    maildir.add(msg)
+
+    with smtplib.SMTP('localhost') as s:
+        s.starttls()
+        s.login('remotevm', '5gW311IOs')
+        s.send_message(msg)
+
+
+@shared_task
+def process_message(path):
+    parser = FeedParser()
+    with open(path) as f:
+        for line in f:
+            parser.feed(line)
+    msg = parser.close()
+
+    #print('read message: {}'.format(msg))
+    #store_message(msg)
+    #log_message(msg)
+
+    if msg.is_multipart():
+        for attach in msg.walk():
+            if attach.get_content_type() == 'application/pdf':
+                reply(msg['From'], msg['Subject'], attach)
+                break
+            else:
+                reply_missing(msg['From'], msg['Subject'])

markup/views.py

@@ -1,24 +1,42 @@
+import os
+import logging
+import humanize
+from tempfile import mkstemp
+from shutil import copyfile
+
 from django.core import serializers
 from django.http import HttpResponseRedirect, HttpResponse, JsonResponse
 from django.shortcuts import render, get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
-#from django.urls import reverse
+from django.core.files.uploadhandler import TemporaryFileUploadHandler
-#from django.utils.translation import gettext as _
 
-#from lazysignup.decorators import allow_lazy_user
+from .tasks import process_message
-#from account.decorators import login_required
 
-#from procat2.models import Catalog
+log = logging.getLogger(__name__)
 
 
 @csrf_exempt
 @require_http_methods(["POST"])
 def submit(request):
+    # always upload into a file
+    request.upload_handlers = [TemporaryFileUploadHandler(request)]
+
     body = request.body
     if not body or len(body) < 1:
         return HttpResponse('Bad request: no data', status=400)
 
-    #data = json.loads(body.decode('utf-8'))
+    msg_file = request.FILES['file']
+    if not msg_file:
+        return HttpResponse('Bad request: no file', status=400)
+
+    msg_size = humanize.naturalsize(msg_file.size, gnu=True)
+    log.debug('message file size: {}'.format(msg_size))
+
+    _, tmpfile = mkstemp(suffix='.eml', prefix='markup_', dir=None, text=False)
+    log.debug('copy message file from {} to {}'.format(msg_file.temporary_file_path(), tmpfile))
+    copyfile(msg_file.temporary_file_path(), tmpfile)
+
+    process_message.delay(tmpfile)
 
     return JsonResponse({'success': True}, safe=False)

procat2/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import absolute_import, unicode_literals
+
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

procat2/celery.py

@@ -0,0 +1,15 @@
+from __future__ import absolute_import, unicode_literals
+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'procat2.settings')
+
+app = Celery('procat2')
+
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+app.autodiscover_tasks()
+
+# @app.task(bind=True)
+# def debug_task(self):
+#     print('Request: {0!r}'.format(self.request))

procat2/settings.py

@@ -9,8 +9,15 @@ https://docs.djangoproject.com/en/2.1/topics/settings/
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/2.1/ref/settings/
 """
+from __future__ import absolute_import, unicode_literals
 
 import os
+
+# ^^^ The above is required if you want to import from the celery
+# library.  If you don't have this then `from celery.schedules import`
+# becomes `proj.celery.schedules` in Python 2.x since it allows
+# for relative imports by default.
+
 from django.urls import reverse_lazy
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -77,6 +84,12 @@ LOGGING = {
             #'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
             'propagate': True,
         },
+        'markup': {
+            'handlers': ['console', 'file'],
+            'level': 'DEBUG',
+            #'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
+            'propagate': True,
+        },
         'django': {
             'handlers': ['console', 'file'],
             #'level': 'DEBUG',
@@ -101,6 +114,7 @@ INSTALLED_APPS = [
     'django.contrib.staticfiles',
     'django.contrib.sites',
     'django_extensions',
+    'django_celery_results',
     'account',
     'lazysignup',
     'webpack_loader',
@@ -108,6 +122,7 @@ INSTALLED_APPS = [
     'dashboard',
     'products',
     'quickinfo',
+    'markup',
 ]
 
 MIDDLEWARE = [
@@ -227,3 +242,13 @@ WEBPACK_LOADER = {
         'IGNORE': [r'.+\.hot-update.js', r'.+\.map']
     }
 }
+
+# allow large file uploads
+# for example the markup tool receiving emails
+DATA_UPLOAD_MAX_MEMORY_SIZE = FILE_UPLOAD_MAX_MEMORY_SIZE = 200 * 1024 * 1024  # 200MB
+
+# celery settings
+CELERY_BROKER_URL = 'amqp://guest:guest@localhost//'
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_RESULT_BACKEND = 'django-db'
+CELERY_TASK_SERIALIZER = 'json'

requirements.txt

@@ -5,12 +5,14 @@ celery==4.3.0
 decorator==4.4.0
 Django==2.2.4
 django-appconf==1.0.3
+django-celery-results==1.1.2
 django-debug-toolbar==1.11
 django-extensions==2.1.6
 django-lazysignup==2.0.0
 django-settings-export==1.2.1
 django-user-accounts==2.1.0
 django-webpack-loader==0.6.0
+humanize==0.5.1
 importlib-metadata==0.23
 ipdb==0.11
 ipython==7.3.0