From 5282f7cb2fead8d2cfe4faf477641b797ea53479 Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Fri, 18 Oct 2019 13:11:53 -0700 Subject: [PATCH] markup: matching works --- markup/img.py | 42 +++++++++++++++++++++++++---- markup/matching.py | 43 ++++++++++++++++++++++++++++++ markup/pdf.py | 50 ++++++++++++++++------------------- markup/utils.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 32 deletions(-) create mode 100644 markup/matching.py create mode 100644 markup/utils.py diff --git a/markup/img.py b/markup/img.py index 8b0901c..3506a5f 100644 --- a/markup/img.py +++ b/markup/img.py @@ -1,7 +1,7 @@ import os import shutil -from PIL import Image, ImageFilter +from PIL import Image, ImageFilter, ImageDraw, ImageFont import numpy import imutils import cv2 @@ -11,6 +11,8 @@ from pathlib import Path from django.conf import settings +from .utils import cv2_rect + WORKDIR = os.path.join(settings.ASSET_DIR, 'markup', 'work') # https://www.pyimagesearch.com/2014/10/20/finding-shapes-images-using-python-opencv/ @@ -56,7 +58,8 @@ def find_shapes(image_path): # if M["m00"] == 0: M["m00"] = 0.00001 # cX = int(M["m10"] / M["m00"]) # cY = int(M["m01"] / M["m00"]) - bboxes.append({'x': x, 'y': y, 'w': w, 'h': h}) + #print('add contour rect: {}'.format(cv2_rect(x, y, w, h))) + bboxes.append(cv2_rect(x, y, w, h)) # draw contours contour_image = numpy.zeros((threshold.shape[0], threshold.shape[1], 3), dtype=numpy.uint8) @@ -64,8 +67,8 @@ def find_shapes(image_path): # compute the center of the contour color = (rng.randint(0,512), rng.randint(0,512), rng.randint(0,512)) cv2.drawContours(contour_image, contours, i, color) - box = bboxes[i] - cv2.rectangle(contour_image, (box['x'],box['y']), (box['x']+box['w'],box['y']+box['h']), color, 1) + rect = bboxes[i] + cv2.rectangle(contour_image, (rect.left, rect.top), (rect.right, rect.bottom), color, 1) # cv2.circle(contour_image, (cX, cY), 2, color, -1) # cv2.putText(contour_image, "center", (cX - 20, cY - 15), # cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 1) @@ -76,4 +79,33 @@ def find_shapes(image_path): os.chmod(contour_path, 0o664) shutil.chown(contour_path, group='procat') - return bboxes + return img.width, img.height, bboxes + + +def write_debug_image(cat_name, page_num, prods, scribbles): + path = os.path.join(WORKDIR, "debug-{}-{}.png".format(cat_name, page_num)) + + pagew = int(11*72) + pageh = int(8.5*72) + + img = Image.new('RGBA', (pagew, pageh), 'white') + draw = ImageDraw.Draw(img, 'RGBA') + fnt = ImageFont.truetype('/usr/share/fonts/truetype/lato/Lato-Regular.ttf', 10) + + for prod in filter(lambda p: p['page'] == page_num, prods): + rect = prod['rect'] + fill_color = "hsv(120, 22%, 100%)" if 'matched' in prod else None + outline_color = "hsv(120, 50%, 100%)" + draw.rectangle((rect.p1(pageh), rect.p2(pageh)), + fill=fill_color, outline=outline_color, width=2) + bl = rect.p1(pageh) + draw.text((bl[0] + 3, bl[1] + 3), prod['material'], + font=fnt, fill="hsv(120, 22%, 50%)") + + for scribble in filter(lambda s: s['page'] == page_num, scribbles): + rect = scribble['rect'] + draw.rectangle((rect.p1(pageh), rect.p2(pageh)), outline="hsv(210, 22%, 100%)", width=2) + for box in scribble['bboxes']: + draw.rectangle((box.p1(pageh), box.p2(pageh)), outline="hsv(0, 22%, 100%)", width=2) + + img.save(path) diff --git a/markup/matching.py b/markup/matching.py new file mode 100644 index 0000000..908b02e --- /dev/null +++ b/markup/matching.py @@ -0,0 +1,43 @@ +from markup.img import find_shapes, write_debug_image +from markup.utils import overlaps + + +def find_scribbles_shapes(scribbles): + for scribble in scribbles: + imgw, imgh, shapes = find_shapes(scribble['image']) + rects = [transform(scribble['rect'], imgw, imgh, s) for s in shapes] + scribble['bboxes'] = rects + + +def transform(pdf_rect, imgw, imgh, shape): + """Convert scribble from image coords to pdf coords""" + # get scale factor for image coords + # to convert to pdf coordinates + pdfw = pdf_rect.right - pdf_rect.left + pdfh = pdf_rect.bottom - pdf_rect.top + scalew = pdfw / imgw + scaleh = pdfh / imgh + return shape.scale(scalew, scaleh).translate(pdf_rect.left, pdf_rect.top) + + +def find_matches(all_prods, scribbles, overlap_threshold): + # segment by page + page_prods = {} + for p in all_prods: + pagenum = p['page'] + if pagenum in page_prods: + page_prods[pagenum].append(p) + else: + page_prods[pagenum] = [p] + + matches = [] + for s in scribbles: + pagenum = s['page'] + prods = page_prods[pagenum] + for p in prods: + for box in s['bboxes']: + if overlaps(p['rect'], box, overlap_threshold): + p['matched'] = s + matches.append(p) + + return matches diff --git a/markup/pdf.py b/markup/pdf.py index ec8da13..1eaf6b9 100644 --- a/markup/pdf.py +++ b/markup/pdf.py @@ -1,28 +1,20 @@ -#from __future__ import absolute_import, unicode_literals - -import sys, os.path, re, json, pickle, subprocess +import os +import sys +import subprocess import shutil -#from pprint import pprint -#import dumper - -#from pdfminer.psparser import PSKeyword, PSLiteral, LIT from pdfminer.pdfparser import PDFParser -from pdfminer.pdfdocument import PDFDocument #, PDFNoOutlines -#from pdfminer.pdftypes import PDFObjectNotFound, PDFValueError, PDFNotImplementedError -#from pdfminer.pdftypes import dict_value, num_value, list_value -#from pdfminer.pdftypes import PDFStream, PDFObjRef, resolve1, resolve_all, stream_value +from pdfminer.pdfdocument import PDFDocument from pdfminer.pdftypes import PDFObjRef, resolve1 -#from pdfminer.pdfpage import PDFPage -#from pdfminer.utils import isnumber -#from pdfminer.image import ImageWriter from django.conf import settings +from .utils import pdf_rect + WORKDIR = os.path.join(settings.ASSET_DIR, 'markup', 'work') -def make_product_box(obj, pagenum): +def make_product_box(obj, pagenum, mediabox): name = obj['ProCatName'].decode() material = obj['ProCatMaterialNumber'].decode() color = obj['ProCatColor'].decode() @@ -34,7 +26,7 @@ def make_product_box(obj, pagenum): 'name': name, 'color': color, 'gender': gender, - 'rect': rect, + 'rect': pdf_rect(rect, mediabox[3]), 'page': pagenum } else: print('Annotation without rect:') @@ -42,30 +34,28 @@ def make_product_box(obj, pagenum): return None -def make_scribble(obj, pagenum): - rect = obj['Rect'] # position - #print(obj) +def make_scribble(obj, pagenum, mediabox): + rect = obj['Rect'] # position on page # walk the object tree down to the image appearance = resolve1(obj['AP']) - #print('app', appearance) normal_appearance = appearance['N'] if not normal_appearance or normal_appearance.objid <= 0: print('skipping scribble - no normal appearance') return normal_appearance = resolve1(normal_appearance) - #print('norm app', normal_appearance) resources = resolve1(normal_appearance['Resources']) xobj = resolve1(resources['XObject']) im1 = resolve1(xobj['Im1']) # PDFStream of the image flter = im1['Filter'] if flter.name == 'JPXDecode': - export_jp2(im1) + path = export_jp2(im1) return { 'page': pagenum, - 'rect': rect, - 'objid': im1.objid } + 'rect': pdf_rect(rect, mediabox[3]), + 'objid': im1.objid, + 'image': path } else: print('skipping non-jp2 image') return None @@ -94,9 +84,10 @@ def export_jp2(obj): os.chmod(png_path, 0o664) shutil.chown(png_path, group='procat') + return png_path + def parse_pdf(fname, debug=0): - PDFDocument.debug = debug PDFParser.debug = debug @@ -115,6 +106,11 @@ def parse_pdf(fname, debug=0): page = resolve1(page) if not 'Annots' in page: continue + mediabox = page['MediaBox'] + # if 'CropBox' in page: + # cropbox = page['CropBox'] + # print('crop',cropbox) + annots = page['Annots'] if isinstance(annots, PDFObjRef): annots = resolve1(annots) @@ -122,9 +118,9 @@ def parse_pdf(fname, debug=0): for anno in annots: anno = resolve1(anno) if 'AAPL:AKExtras' in anno: - scribbles.append(make_scribble(anno, pagenum)) + scribbles.append(make_scribble(anno, pagenum, mediabox)) elif 'ProCatName' in anno: - prod_boxes.append(make_product_box(anno, pagenum)) + prod_boxes.append(make_product_box(anno, pagenum, mediabox)) else: print('ignoring other annotation') diff --git a/markup/utils.py b/markup/utils.py new file mode 100644 index 0000000..47ab266 --- /dev/null +++ b/markup/utils.py @@ -0,0 +1,66 @@ +def pdf_rect(rect, container_height): + x1 = min(rect[0], rect[2]) + y1 = max(rect[1], rect[3]) + x2 = max(rect[0], rect[2]) + y2 = min(rect[1], rect[3]) + # and convert from pdf to image coords + return Rect(x1, container_height - y1, x2, container_height - y2) + + +def cv2_rect(l, t, w, h): + return Rect(l, t, l + w, t + h) + + +def overlaps(r1, r2, threshold): + A = r1.to_dict() + B = r2.to_dict() + + # https://stackoverflow.com/questions/9324339/how-much-do-two-rectangles-overlap + SA = A['w'] * A['h'] + SB = B['w'] * B['h'] + SI = max([0, 1 + min([A['x2'], B['x2']]) - max([A['x1'], B['x1']])]) * max([0, 1 + min([A['y2'], B['y2']]) - max([A['y1'], B['y1']])]) + SU = SA + SB - SI + overlap = float(SI) / float(SU) + + print('overlap: {}'.format(int(overlap * 100))) + return overlap > threshold + + +class Rect(object): + + def __init__(self, l, t, r, b): + self.left = l + self.top = t + self.right = r + self.bottom = b + + def translate(self, x, y): + self.left += x + self.top += y + self.right += x + self.bottom += y + return self + + def scale(self, x, y): + self.left *= x + self.top *= y + self.right *= x + self.bottom *= y + return self + + def p1(self, page_height): + return (self.left, self.top) + + def p2(self, page_height): + return (self.right, self.bottom) + + def to_dict(self): + return {'x1': self.left, + 'y1': self.top, + 'x2': self.right, + 'y2': self.bottom, + 'w': self.right - self.left, + 'h': self.bottom - self.top } + + def __repr__(self): + return 'Rect[l={}, t={}, r={}, b={}]'.format(int(self.left), int(self.top), int(self.right), int(self.bottom))