markup: first email code

markup/email.py

@@ -0,0 +1,72 @@
+import sys
+import string
+import random
+import smtplib
+from pathlib import Path
+from mailbox import Maildir
+from email.message import EmailMessage
+
+import logging
+log = logging.getLogger(__name__)
+
+
+body_ok = """Hi,
+
+Attached is a spreadsheet with the articles you marked up in the
+catalog.
+
+Enjoy,
+ProCatalog Markup Bot
+"""
+
+body_missing = """Hi,
+
+I couldn't find a pdf attached to your message.  I can't do much
+without a marked up catalog pdf, so please include that when you try
+again.
+
+Thanks,
+ProCatalog Markup Bot
+"""
+
+
+def reply(frm, subj, xls_path):
+    msg = EmailMessage()
+    msg.set_content(body_ok)
+
+    with open(xls_path, 'rb') as fp:
+        msg.add_attachment(fp.read(),
+                           maintype='application',
+                           subtype='vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+                           filename=Path(xls_path).name)
+
+    send(frm, subj, msg)
+
+
+def reply_missing(frm, subj):
+    msg = EmailMessage()
+    msg.set_content(body_missing)
+    send(frm, subj, msg)
+
+
+def send(frm, subj, msg):
+    msg['From'] = 'Keen ProCatalog Markup Bot <support@procatalog.io>'
+    msg['To'] = frm
+    msg['Bcc'] = 'alx-markup@procatalog.io'
+    msg['Subject'] = f'Re: {subj}'
+    msg['Message-ID'] = msgid()
+
+    maildir = Maildir('/tmp/markup_submit_mail')
+    maildir.add(msg)
+
+    log.info(f'sending email to "{frm}": {subj}')
+
+    with smtplib.SMTP('mail.mk.int') as s:
+        s.starttls()
+        s.login('remotevm', '5gW311IOs')
+        s.send_message(msg)
+
+
+def msgid():
+    rand = ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=16))
+    return f'<{rand}@markup.procatalog.io>'

markup/tasks.py

@@ -1,77 +1,28 @@
 from __future__ import absolute_import, unicode_literals
 from celery import task, shared_task
+from celery.utils.log import get_task_logger
+
+import os
 import sys
 import datetime
 import fileinput
 import smtplib
-#from mailbox import Maildir
+from pathlib import Path
+
 from email.feedparser import FeedParser
 from email.message import EmailMessage
 
-#from demoapp.models import Widget
+import django
-#from celery import Celery
+from django.conf import settings
-#app = Celery('procat2', broker='amqp://localhost')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'procat2.settings')
+django.setup()
 
-#@app.task
+from .utils import clean_path, ensure_dir, set_file_perms, WORKDIR
+from .email import reply, reply_missing
+from .matching import find_marked_products
+from .spreadsheet import write_spreadsheet
 
-@shared_task
+logger = get_task_logger(__name__)
-def add(x, y):
-    return x + y
-
-body_ok = """Hi,
-
-Attached is your marked up catalog.  In the near future you'll also
-get a spreadsheet and a new catalog with just the marked up items.
-
-Enjoy,
-ProCatalog Markup Bot"""
-
-body_missing = """Hi,
-
-I couldn't find a pdf attached to your message.  I can't do much
-without a marked up catalog pdf, so please include that when you try
-again.
-
-Thanks,
-ProCatalog Markup Bot"""
-
-
-# def log_message(msg):
-#     with open('/var/log/markup_submit.log', 'a') as f:
-#         f.write("{} - {} / {}\n".format(datetime.datetime.now().isoformat(),
-#                                         msg['From'], msg['Subject']))
-
-def reply(frm, subj, attach):
-    print('ok: {} - {}'.format(frm, subj))
-    # msg = EmailMessage()
-    # msg.set_content(body_ok)
-    # msg.add_attachment(attach.get_payload(decode=True),
-    #                    maintype=attach.get_content_maintype(),
-    #                    subtype=attach.get_content_subtype(),
-    #                    filename=attach.get_filename())
-    # send(frm, subj, msg)
-
-
-def reply_missing(frm, subj):
-    print('missing: {} - {}'.format(frm, subj))
-    # msg = EmailMessage()
-    # msg.set_content(body_missing)
-    # send(frm, subj, msg)
-
-
-def send(frm, subj, msg):
-    msg['From'] = 'Keen ProCatalog Markup Bot <support@procatalog.io>'
-    msg['To'] = frm
-    msg['Bcc'] = 'alx-markup@procatalog.io'
-    msg['Subject'] = 'Re: {}'.format(subj)
-
-    maildir = Maildir('/tmp/markup_submit_mail')
-    maildir.add(msg)
-
-    with smtplib.SMTP('localhost') as s:
-        s.starttls()
-        s.login('remotevm', '5gW311IOs')
-        s.send_message(msg)
 
 
 @shared_task
@@ -82,14 +33,41 @@ def process_message(path):
             parser.feed(line)
     msg = parser.close()
 
-    #print('read message: {}'.format(msg))
+    found_pdf = False
-    #store_message(msg)
-    #log_message(msg)
-
     if msg.is_multipart():
         for attach in msg.walk():
             if attach.get_content_type() == 'application/pdf':
-                reply(msg['From'], msg['Subject'], attach)
+                process_attachment(msg['From'], msg['Subject'], attach)
-                break
+                found_pdf = True
-            else:
+
-                reply_missing(msg['From'], msg['Subject'])
+    if not found_pdf:
+        reply_missing(msg['From'], msg['Subject'])
+
+
+def process_attachment(from_address, subject, attachment):
+    # write out pdf
+    pdf_name = attachment.get_filename()
+    pdf_base = Path(pdf_name).stem
+
+    workdir = os.path.join(WORKDIR, clean_path(from_address), pdf_base)
+    ensure_dir(workdir)
+    pdf_path = os.path.join(workdir, pdf_name)
+    print(f'saving pdf to {pdf_path}')
+    with open(pdf_path, 'wb') as att:
+        att.write(attachment.get_payload(decode=True))
+    set_file_perms(pdf_path)
+
+    # find matches
+    matches = find_marked_products(pdf_path, workdir, debug=0)
+    print(f'{len(matches)} product matches')
+
+    # write spreadsheet
+    xls_path = write_spreadsheet(matches, workdir, pdf_base)
+
+    if xls_path:
+        # send reply
+        print(f'wrote spreadsheet: {xls_path}')
+        reply(from_address, subject, xls_path)
+    else:
+        # send error
+        print(f'error creating spreadsheet')

markup/work/test_email.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+#
+# process an .eml file through the whole markup process
+
+import sys
+import os
+import inspect
+
+currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentdir = os.path.dirname(currentdir)
+parentparentdir = os.path.dirname(parentdir)
+sys.path.insert(0, parentparentdir)
+
+import getopt
+import django
+from django.conf import settings
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'procat2.settings')
+django.setup()
+
+from markup.tasks import process_message
+
+
+def main(argv):
+    def usage():
+        print('usage: %s file.eml' % argv[0])
+        return 100
+    try:
+        (opts, args) = getopt.getopt(argv[1:], '')
+    except getopt.GetoptError:
+        return usage()
+    if not args: return usage()
+
+    process_message(args[0])
+
+
+if __name__ == '__main__': sys.exit(main(sys.argv))