From 606f3fd5da0a793d904667706b5e7f8d60da6969 Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Sat, 19 Oct 2019 22:11:21 -0700 Subject: [PATCH] markup: first email code --- markup/email.py | 72 +++++++++++++++++++++++ markup/tasks.py | 120 ++++++++++++++++---------------------- markup/work/test_email.py | 37 ++++++++++++ 3 files changed, 158 insertions(+), 71 deletions(-) create mode 100644 markup/email.py create mode 100755 markup/work/test_email.py diff --git a/markup/email.py b/markup/email.py new file mode 100644 index 0000000..0eb0036 --- /dev/null +++ b/markup/email.py @@ -0,0 +1,72 @@ +import sys +import string +import random +import smtplib +from pathlib import Path +from mailbox import Maildir +from email.message import EmailMessage + +import logging +log = logging.getLogger(__name__) + + +body_ok = """Hi, + +Attached is a spreadsheet with the articles you marked up in the +catalog. + +Enjoy, +ProCatalog Markup Bot +""" + +body_missing = """Hi, + +I couldn't find a pdf attached to your message. I can't do much +without a marked up catalog pdf, so please include that when you try +again. + +Thanks, +ProCatalog Markup Bot +""" + + +def reply(frm, subj, xls_path): + msg = EmailMessage() + msg.set_content(body_ok) + + with open(xls_path, 'rb') as fp: + msg.add_attachment(fp.read(), + maintype='application', + subtype='vnd.openxmlformats-officedocument.spreadsheetml.sheet', + filename=Path(xls_path).name) + + send(frm, subj, msg) + + +def reply_missing(frm, subj): + msg = EmailMessage() + msg.set_content(body_missing) + send(frm, subj, msg) + + +def send(frm, subj, msg): + msg['From'] = 'Keen ProCatalog Markup Bot ' + msg['To'] = frm + msg['Bcc'] = 'alx-markup@procatalog.io' + msg['Subject'] = f'Re: {subj}' + msg['Message-ID'] = msgid() + + maildir = Maildir('/tmp/markup_submit_mail') + maildir.add(msg) + + log.info(f'sending email to "{frm}": {subj}') + + with smtplib.SMTP('mail.mk.int') as s: + s.starttls() + s.login('remotevm', '5gW311IOs') + s.send_message(msg) + + +def msgid(): + rand = ''.join(random.choices(string.ascii_uppercase + string.ascii_lowercase + string.digits, k=16)) + return f'<{rand}@markup.procatalog.io>' diff --git a/markup/tasks.py b/markup/tasks.py index dcade6a..fc87eba 100644 --- a/markup/tasks.py +++ b/markup/tasks.py @@ -1,77 +1,28 @@ from __future__ import absolute_import, unicode_literals from celery import task, shared_task +from celery.utils.log import get_task_logger + +import os import sys import datetime import fileinput import smtplib -#from mailbox import Maildir +from pathlib import Path + from email.feedparser import FeedParser from email.message import EmailMessage -#from demoapp.models import Widget -#from celery import Celery -#app = Celery('procat2', broker='amqp://localhost') +import django +from django.conf import settings +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'procat2.settings') +django.setup() -#@app.task +from .utils import clean_path, ensure_dir, set_file_perms, WORKDIR +from .email import reply, reply_missing +from .matching import find_marked_products +from .spreadsheet import write_spreadsheet -@shared_task -def add(x, y): - return x + y - -body_ok = """Hi, - -Attached is your marked up catalog. In the near future you'll also -get a spreadsheet and a new catalog with just the marked up items. - -Enjoy, -ProCatalog Markup Bot""" - -body_missing = """Hi, - -I couldn't find a pdf attached to your message. I can't do much -without a marked up catalog pdf, so please include that when you try -again. - -Thanks, -ProCatalog Markup Bot""" - - -# def log_message(msg): -# with open('/var/log/markup_submit.log', 'a') as f: -# f.write("{} - {} / {}\n".format(datetime.datetime.now().isoformat(), -# msg['From'], msg['Subject'])) - -def reply(frm, subj, attach): - print('ok: {} - {}'.format(frm, subj)) - # msg = EmailMessage() - # msg.set_content(body_ok) - # msg.add_attachment(attach.get_payload(decode=True), - # maintype=attach.get_content_maintype(), - # subtype=attach.get_content_subtype(), - # filename=attach.get_filename()) - # send(frm, subj, msg) - - -def reply_missing(frm, subj): - print('missing: {} - {}'.format(frm, subj)) - # msg = EmailMessage() - # msg.set_content(body_missing) - # send(frm, subj, msg) - - -def send(frm, subj, msg): - msg['From'] = 'Keen ProCatalog Markup Bot ' - msg['To'] = frm - msg['Bcc'] = 'alx-markup@procatalog.io' - msg['Subject'] = 'Re: {}'.format(subj) - - maildir = Maildir('/tmp/markup_submit_mail') - maildir.add(msg) - - with smtplib.SMTP('localhost') as s: - s.starttls() - s.login('remotevm', '5gW311IOs') - s.send_message(msg) +logger = get_task_logger(__name__) @shared_task @@ -82,14 +33,41 @@ def process_message(path): parser.feed(line) msg = parser.close() - #print('read message: {}'.format(msg)) - #store_message(msg) - #log_message(msg) - + found_pdf = False if msg.is_multipart(): for attach in msg.walk(): if attach.get_content_type() == 'application/pdf': - reply(msg['From'], msg['Subject'], attach) - break - else: - reply_missing(msg['From'], msg['Subject']) + process_attachment(msg['From'], msg['Subject'], attach) + found_pdf = True + + if not found_pdf: + reply_missing(msg['From'], msg['Subject']) + + +def process_attachment(from_address, subject, attachment): + # write out pdf + pdf_name = attachment.get_filename() + pdf_base = Path(pdf_name).stem + + workdir = os.path.join(WORKDIR, clean_path(from_address), pdf_base) + ensure_dir(workdir) + pdf_path = os.path.join(workdir, pdf_name) + print(f'saving pdf to {pdf_path}') + with open(pdf_path, 'wb') as att: + att.write(attachment.get_payload(decode=True)) + set_file_perms(pdf_path) + + # find matches + matches = find_marked_products(pdf_path, workdir, debug=0) + print(f'{len(matches)} product matches') + + # write spreadsheet + xls_path = write_spreadsheet(matches, workdir, pdf_base) + + if xls_path: + # send reply + print(f'wrote spreadsheet: {xls_path}') + reply(from_address, subject, xls_path) + else: + # send error + print(f'error creating spreadsheet') diff --git a/markup/work/test_email.py b/markup/work/test_email.py new file mode 100755 index 0000000..72e761c --- /dev/null +++ b/markup/work/test_email.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# +# process an .eml file through the whole markup process + +import sys +import os +import inspect + +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parentdir = os.path.dirname(currentdir) +parentparentdir = os.path.dirname(parentdir) +sys.path.insert(0, parentparentdir) + +import getopt +import django +from django.conf import settings + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'procat2.settings') +django.setup() + +from markup.tasks import process_message + + +def main(argv): + def usage(): + print('usage: %s file.eml' % argv[0]) + return 100 + try: + (opts, args) = getopt.getopt(argv[1:], '') + except getopt.GetoptError: + return usage() + if not args: return usage() + + process_message(args[0]) + + +if __name__ == '__main__': sys.exit(main(sys.argv))