markup: matching works

markup/img.py

@@ -1,7 +1,7 @@
 import os
 import shutil
 
-from PIL import Image, ImageFilter
+from PIL import Image, ImageFilter, ImageDraw, ImageFont
 import numpy
 import imutils
 import cv2
@@ -11,6 +11,8 @@ from pathlib import Path
 
 from django.conf import settings
 
+from .utils import cv2_rect
+
 WORKDIR = os.path.join(settings.ASSET_DIR, 'markup', 'work')
 
 # https://www.pyimagesearch.com/2014/10/20/finding-shapes-images-using-python-opencv/
@@ -56,7 +58,8 @@ def find_shapes(image_path):
         # if M["m00"] == 0: M["m00"] = 0.00001
         # cX = int(M["m10"] / M["m00"])
         # cY = int(M["m01"] / M["m00"])
-        bboxes.append({'x': x, 'y': y, 'w': w, 'h': h})
+        #print('add contour rect: {}'.format(cv2_rect(x, y, w, h)))
+        bboxes.append(cv2_rect(x, y, w, h))
 
     # draw contours
     contour_image = numpy.zeros((threshold.shape[0], threshold.shape[1], 3), dtype=numpy.uint8)
@@ -64,8 +67,8 @@ def find_shapes(image_path):
         # compute the center of the contour
         color = (rng.randint(0,512), rng.randint(0,512), rng.randint(0,512))
         cv2.drawContours(contour_image, contours, i, color)
-        box = bboxes[i]
+        rect = bboxes[i]
-        cv2.rectangle(contour_image, (box['x'],box['y']), (box['x']+box['w'],box['y']+box['h']), color, 1)
+        cv2.rectangle(contour_image, (rect.left, rect.top), (rect.right, rect.bottom), color, 1)
         # cv2.circle(contour_image, (cX, cY), 2, color, -1)
         # cv2.putText(contour_image, "center", (cX - 20, cY - 15),
         #             cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 1)
@@ -76,4 +79,33 @@ def find_shapes(image_path):
     os.chmod(contour_path, 0o664)
     shutil.chown(contour_path, group='procat')
 
-    return bboxes
+    return img.width, img.height, bboxes
+
+
+def write_debug_image(cat_name, page_num, prods, scribbles):
+    path = os.path.join(WORKDIR, "debug-{}-{}.png".format(cat_name, page_num))
+
+    pagew = int(11*72)
+    pageh = int(8.5*72)
+
+    img = Image.new('RGBA', (pagew, pageh), 'white')
+    draw = ImageDraw.Draw(img, 'RGBA')
+    fnt = ImageFont.truetype('/usr/share/fonts/truetype/lato/Lato-Regular.ttf', 10)
+
+    for prod in filter(lambda p: p['page'] == page_num, prods):
+        rect = prod['rect']
+        fill_color = "hsv(120, 22%, 100%)" if 'matched' in prod else None
+        outline_color = "hsv(120, 50%, 100%)"
+        draw.rectangle((rect.p1(pageh), rect.p2(pageh)),
+                       fill=fill_color, outline=outline_color, width=2)
+        bl = rect.p1(pageh)
+        draw.text((bl[0] + 3, bl[1] + 3), prod['material'],
+                  font=fnt, fill="hsv(120, 22%, 50%)")
+
+    for scribble in filter(lambda s: s['page'] == page_num, scribbles):
+        rect = scribble['rect']
+        draw.rectangle((rect.p1(pageh), rect.p2(pageh)), outline="hsv(210, 22%, 100%)", width=2)
+        for box in scribble['bboxes']:
+            draw.rectangle((box.p1(pageh), box.p2(pageh)), outline="hsv(0, 22%, 100%)", width=2)
+
+    img.save(path)

markup/matching.py

@@ -0,0 +1,43 @@
+from markup.img import find_shapes, write_debug_image
+from markup.utils import overlaps
+
+
+def find_scribbles_shapes(scribbles):
+    for scribble in scribbles:
+        imgw, imgh, shapes = find_shapes(scribble['image'])
+        rects = [transform(scribble['rect'], imgw, imgh, s) for s in shapes]
+        scribble['bboxes'] = rects
+
+
+def transform(pdf_rect, imgw, imgh, shape):
+    """Convert scribble from image coords to pdf coords"""
+    # get scale factor for image coords
+    # to convert to pdf coordinates
+    pdfw = pdf_rect.right - pdf_rect.left
+    pdfh = pdf_rect.bottom - pdf_rect.top
+    scalew = pdfw / imgw
+    scaleh = pdfh / imgh
+    return shape.scale(scalew, scaleh).translate(pdf_rect.left, pdf_rect.top)
+
+
+def find_matches(all_prods, scribbles, overlap_threshold):
+    # segment by page
+    page_prods = {}
+    for p in all_prods:
+        pagenum = p['page']
+        if pagenum in page_prods:
+            page_prods[pagenum].append(p)
+        else:
+            page_prods[pagenum] = [p]
+
+    matches = []
+    for s in scribbles:
+        pagenum = s['page']
+        prods = page_prods[pagenum]
+        for p in prods:
+            for box in s['bboxes']:
+                if overlaps(p['rect'], box, overlap_threshold):
+                    p['matched'] = s
+                    matches.append(p)
+
+    return matches

markup/pdf.py

@@ -1,28 +1,20 @@
-#from __future__ import absolute_import, unicode_literals
+import os
-
+import sys
-import sys, os.path, re, json, pickle, subprocess
+import subprocess
 import shutil
 
-#from pprint import pprint
-#import dumper
-
-#from pdfminer.psparser import PSKeyword, PSLiteral, LIT
 from pdfminer.pdfparser import PDFParser
-from pdfminer.pdfdocument import PDFDocument #, PDFNoOutlines
+from pdfminer.pdfdocument import PDFDocument
-#from pdfminer.pdftypes import PDFObjectNotFound, PDFValueError, PDFNotImplementedError
-#from pdfminer.pdftypes import dict_value, num_value, list_value
-#from pdfminer.pdftypes import PDFStream, PDFObjRef, resolve1, resolve_all, stream_value
 from pdfminer.pdftypes import PDFObjRef, resolve1
-#from pdfminer.pdfpage import PDFPage
-#from pdfminer.utils import isnumber
-#from pdfminer.image import ImageWriter
 
 from django.conf import settings
 
+from .utils import pdf_rect
+
 WORKDIR = os.path.join(settings.ASSET_DIR, 'markup', 'work')
 
 
-def make_product_box(obj, pagenum):
+def make_product_box(obj, pagenum, mediabox):
     name = obj['ProCatName'].decode()
     material = obj['ProCatMaterialNumber'].decode()
     color = obj['ProCatColor'].decode()
@@ -34,7 +26,7 @@ def make_product_box(obj, pagenum):
                  'name': name,
                  'color': color,
                  'gender': gender,
-                 'rect': rect,
+                 'rect': pdf_rect(rect, mediabox[3]),
                  'page': pagenum }
     else:
         print('Annotation without rect:')
@@ -42,30 +34,28 @@ def make_product_box(obj, pagenum):
         return None
 
 
-def make_scribble(obj, pagenum):
+def make_scribble(obj, pagenum, mediabox):
-    rect = obj['Rect'] # position
+    rect = obj['Rect'] # position on page
-    #print(obj)
 
     # walk the object tree down to the image
     appearance = resolve1(obj['AP'])
-    #print('app', appearance)
     normal_appearance = appearance['N']
     if not normal_appearance or normal_appearance.objid <= 0:
         print('skipping scribble - no normal appearance')
         return
 
     normal_appearance = resolve1(normal_appearance)
-    #print('norm app', normal_appearance)
     resources = resolve1(normal_appearance['Resources'])
     xobj = resolve1(resources['XObject'])
     im1 = resolve1(xobj['Im1']) # PDFStream of the image
 
     flter = im1['Filter']
     if flter.name == 'JPXDecode':
-        export_jp2(im1)
+        path = export_jp2(im1)
         return { 'page': pagenum,
-                 'rect': rect,
+                 'rect': pdf_rect(rect, mediabox[3]),
-                 'objid': im1.objid }
+                 'objid': im1.objid,
+                 'image': path }
     else:
         print('skipping non-jp2 image')
         return None
@@ -94,9 +84,10 @@ def export_jp2(obj):
         os.chmod(png_path, 0o664)
         shutil.chown(png_path, group='procat')
 
+    return png_path
+
 
 def parse_pdf(fname, debug=0):
-
     PDFDocument.debug = debug
     PDFParser.debug = debug
 
@@ -115,6 +106,11 @@ def parse_pdf(fname, debug=0):
         page = resolve1(page)
         if not 'Annots' in page: continue
 
+        mediabox = page['MediaBox']
+        # if 'CropBox' in page:
+        #     cropbox = page['CropBox']
+        #     print('crop',cropbox)
+
         annots = page['Annots']
         if isinstance(annots, PDFObjRef):
             annots = resolve1(annots)
@@ -122,9 +118,9 @@ def parse_pdf(fname, debug=0):
         for anno in annots:
             anno = resolve1(anno)
             if 'AAPL:AKExtras' in anno:
-                scribbles.append(make_scribble(anno, pagenum))
+                scribbles.append(make_scribble(anno, pagenum, mediabox))
             elif 'ProCatName' in anno:
-                prod_boxes.append(make_product_box(anno, pagenum))
+                prod_boxes.append(make_product_box(anno, pagenum, mediabox))
             else:
                 print('ignoring other annotation')
 

markup/utils.py

@@ -0,0 +1,66 @@
+def pdf_rect(rect, container_height):
+    x1 = min(rect[0], rect[2])
+    y1 = max(rect[1], rect[3])
+    x2 = max(rect[0], rect[2])
+    y2 = min(rect[1], rect[3])
+    # and convert from pdf to image coords
+    return Rect(x1, container_height - y1, x2, container_height - y2)
+
+
+def cv2_rect(l, t, w, h):
+    return Rect(l, t, l + w, t + h)
+
+
+def overlaps(r1, r2, threshold):
+    A = r1.to_dict()
+    B = r2.to_dict()
+
+    # https://stackoverflow.com/questions/9324339/how-much-do-two-rectangles-overlap
+    SA = A['w'] * A['h']
+    SB = B['w'] * B['h']
+    SI = max([0, 1 + min([A['x2'], B['x2']]) - max([A['x1'], B['x1']])]) * max([0, 1 + min([A['y2'], B['y2']]) - max([A['y1'], B['y1']])])
+    SU = SA + SB - SI
+    overlap = float(SI) / float(SU)
+
+    print('overlap: {}'.format(int(overlap * 100)))
+    return overlap > threshold
+
+
+class Rect(object):
+
+    def __init__(self, l, t, r, b):
+        self.left = l
+        self.top = t
+        self.right = r
+        self.bottom = b
+
+    def translate(self, x, y):
+        self.left += x
+        self.top += y
+        self.right += x
+        self.bottom += y
+        return self
+
+    def scale(self, x, y):
+        self.left *= x
+        self.top *= y
+        self.right *= x
+        self.bottom *= y
+        return self
+
+    def p1(self, page_height):
+        return (self.left, self.top)
+
+    def p2(self, page_height):
+        return (self.right, self.bottom)
+
+    def to_dict(self):
+        return {'x1': self.left,
+                'y1': self.top,
+                'x2': self.right,
+                'y2': self.bottom,
+                'w': self.right - self.left,
+                'h': self.bottom - self.top }
+
+    def __repr__(self):
+        return 'Rect[l={}, t={}, r={}, b={}]'.format(int(self.left), int(self.top), int(self.right), int(self.bottom))