From aa2a6bc4ca3bdce0eb4f1496d3cf935cda86fbbf Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Wed, 4 Dec 2019 17:02:21 -0800 Subject: [PATCH 1/9] markup: handle netpbm encoded annotations --- markup/img.py | 14 +++++++----- markup/pdf.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/markup/img.py b/markup/img.py index dfcf031..ebeb5f8 100644 --- a/markup/img.py +++ b/markup/img.py @@ -20,12 +20,16 @@ def find_shapes(image_path): """ path = Path(image_path) - img = Image.open(image_path, 'r') - if not img.mode in ('RGBA', 'LA'): - print('no alpha channel: {}'.format(img.mode)) - return None + print('finding shapes in {}'.format(image_path)) - alpha_layer = img.convert('RGBA').split()[-1] + img = Image.open(image_path, 'r') + if img.mode == 'RGBA': + alpha_layer = img.convert('RGBA').split()[-1] + elif img.mode == 'L': + alpha_layer = img + else: + print('unhandled image mode: {}'.format(img.mode)) + return None alpha_layer = alpha_layer.filter(ImageFilter.GaussianBlur(5)) diff --git a/markup/pdf.py b/markup/pdf.py index ff1d7eb..863f5b0 100644 --- a/markup/pdf.py +++ b/markup/pdf.py @@ -2,7 +2,9 @@ import os import sys import subprocess import shutil +import dumper +from pdfminer.psparser import LIT from pdfminer.pdfparser import PDFParser from pdfminer.pdfdocument import PDFDocument from pdfminer.pdftypes import PDFObjRef, resolve1 @@ -59,8 +61,15 @@ def make_scribble(obj, pagenum, mediabox, workdir): 'rect': pdf_rect(rect, mediabox[3]), 'objid': im1.objid, 'image': path } + elif flter.name == 'FlateDecode': + path = export_netpbm(im1, workdir, pagenum) + return { 'page': pagenum, + 'rect': pdf_rect(rect, mediabox[3]), + 'objid': im1.objid, + 'image': path } else: - print('skipping non-jp2 image') + print('skipping unrecognized image') + # print(dumper.dump(im1)) return None @@ -85,6 +94,55 @@ def export_jp2(obj, workdir, pagenum): return png_path +def export_netpbm(obj, workdir, pagenum): + oid = obj.objid + ensure_dir(workdir) + + pbm_base = os.path.join(workdir, f"export-page{pagenum:03d}-obj{oid:05d}") + pbm_path = write_pbm(obj, pbm_base) + + # stencil mask - use instead if present + smask = obj.attrs['SMask'] + if smask: + print('extracting pbm mask') + mask = resolve1(smask) + mask_base = os.path.join(workdir, f"export-page{pagenum:03d}-obj{oid:05d}-mask") + mask_path = write_pbm(smask, mask_base) + pbm_path = mask_path + + return pbm_path + + +def write_pbm(obj, base_path): + obj = resolve1(obj) + color_space = resolve1(obj.attrs['ColorSpace']) + + suffix = '.pgm' if color_space == LIT('DeviceGray') else '.ppm' + path = base_path + suffix + + print('writing pbm: {}'.format(path)) + + data = obj.get_data() + with open(path, 'wb') as out: + if suffix == '.pgm': + out.write("P5\n".encode()) + else: + out.write("P6\n".encode()) + + out.write("{} {}\n".format(obj.attrs['Width'], obj.attrs['Height']).encode()) + + if obj.attrs['BitsPerComponent'] == 8: + out.write("255\n".encode()) + else: + out.write("65535\n".encode()) + + out.write(data) + + set_file_perms(path) + + return path + + def parse_pdf(fname, workdir, debug=0): PDFDocument.debug = debug PDFParser.debug = debug From 45d3c5a7900e0dd6d5941c5d388ee7b096cbdca5 Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:26:17 -0800 Subject: [PATCH 2/9] gitignore: markup/work/*.{pdf,eml} --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f43aa4a..c4d530d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ /debug.log /cateditor/vue.config.js* /users.csv +/markup/work/*.pdf +/markup/work/*.eml \ No newline at end of file From 8a88968cc85175127f89feb8d7fe674741b3a261 Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:26:59 -0800 Subject: [PATCH 3/9] markup/pdf.py: print ignored annotations --- markup/pdf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/markup/pdf.py b/markup/pdf.py index 863f5b0..dd1b7d1 100644 --- a/markup/pdf.py +++ b/markup/pdf.py @@ -178,7 +178,8 @@ def parse_pdf(fname, workdir, debug=0): elif 'ProCatName' in anno: prod_boxes.append(make_product_box(anno, pagenum, mediabox)) else: - print('ignoring other annotation') + print('ignoring other annotation:') + print(anno) fp.close() From 6897fd491079712abcc858c3dad7334cbe514672 Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:30:50 -0800 Subject: [PATCH 4/9] products/models.py: add Product.season_ranking field --- products/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/products/models.py b/products/models.py index f1f0f70..1fa4db9 100644 --- a/products/models.py +++ b/products/models.py @@ -14,6 +14,7 @@ class Product(models.Model): id = models.CharField(max_length=20, db_column='id', primary_key=True) sap = models.CharField(max_length=10, db_column='material') season = models.CharField(max_length=10, db_column='season') + season_ranking = models.DecimalField(max_digits=6, decimal_places=1) name = models.CharField(max_length=100, db_column='model') model = models.CharField(max_length=100, db_column='modelcode') gender = models.CharField(max_length=100, db_column='gender') From fd21c1c6b3b5920216406250456551ccba38b739 Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:40:41 -0800 Subject: [PATCH 5/9] products/views.py: improve product search --- products/views.py | 76 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/products/views.py b/products/views.py index 9ff2f65..3dcc6a4 100644 --- a/products/views.py +++ b/products/views.py @@ -2,6 +2,7 @@ from django.http import HttpResponseRedirect, HttpResponse, JsonResponse from django.shortcuts import render, get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods +from django.db import connections import json import logging @@ -44,30 +45,75 @@ def search_products(request): return HttpResponse('Bad request: no region found', status=400) ids = Product.find_sap_ids(text) - log.info('found ids %s in %s', ids, text) + log.info('found %s ids %s in %s', len(ids), ids, text) - prods = [] - missing = [] + final_prods = [] + missing_ids = [] + restricted_ids = [] if ids: - srm = SeasonRegionMaterial.objects.filter(material__in=ids, season=season.id, region__iexact=region.id).distinct('material') + srm = SeasonRegionMaterial.objects.filter(material__in=ids, season=season.id, region__iexact=region.id).order_by('ranking') srm_ids = [x.material for x in srm] - log.info('found SRM ids %s', srm_ids) + log.debug('found %s ids: %s', len(srm_ids), srm_ids) - search_prods = Product.objects.filter(sap__in=srm_ids,season=season.id).distinct('sap') + # in search list but not found in current season/region ranking + missing_ids = (list(set(ids) - set(srm_ids))) + log.debug('no season/region hits on %s ids: %s', len(missing_ids), missing_ids) - # fix product order to match input ids and find missing ids - prod_dict = dict([(p.sap, p) for p in search_prods]) + # missing ids ranked *somewhere*. these are our restricted ids. + restricted = SeasonRegionMaterial.objects.filter(material__in=missing_ids).order_by('material').distinct('material') + restricted_ids = [x.material for x in restricted] + log.debug('found %s restricted ids: %s', len(restricted_ids), restricted_ids) - for i in ids: - if prod_dict.get(i): - prods.append(prod_dict[i]) - else: - missing.append(i) + # don't include restricted in the missing list + missing_ids = (list(set(missing_ids) - set(restricted_ids))) + log.debug('actually missing %s ids: %s', len(missing_ids), missing_ids) + + # load product info for current season + load_ids = srm_ids + restricted_ids + prods = Product.objects.filter(sap__in=load_ids) + #prods = Product.objects.filter(sap__in=srm_ids, season=season.id) + prods_ids = [x.sap for x in prods] + log.debug('loaded %s season products: %s', len(prods), prods_ids) + + # find info for prods ranking in keen_season_region_material + # but not in keen_materials for the current season. in this + # case, fall back to the most recent data we have for the + # prods. if it's (S/R/M) ranked, it should be returned as a + # valid material as per Ed Reilly. + missing_prod_ids = (list(set(srm_ids) - set(prods_ids))) + fill_in_prods = [] + if missing_prod_ids: + log.debug('looking for %s out-of-season products: %s', len(missing_prod_ids), missing_prod_ids) + fill_in_prods = Product.objects.filter(sap__in=missing_prod_ids).order_by('sap', '-season_ranking').distinct('sap') + log.debug('found %s out-of-season products: %s', len(fill_in_prods), [x.sap for x in fill_in_prods]) + + all_prods = list(prods) + list(fill_in_prods) + prod_dict = dict([(p.sap, p) for p in all_prods]) + + # remove related kids who have a parent in the catalog. + # they'll get added automatically to the parent section. + with connections['products'].cursor() as cursor: + cursor.execute('select child from keen_related_kids where parent = any(%s)', [srm_ids]) + for row in cursor.fetchall(): + kid = row[0] + if kid in prod_dict: + log.debug('removing related kid: %s', kid) + del prod_dict[kid] + + # fix product order to match srm ranking + #for id in srm_ids: + for id in load_ids: + if prod_dict.get(id): + final_prods.append(prod_dict[id]) + + log.debug('returning %s products: %s', len(final_prods), [x.sap for x in final_prods]) out = { - 'found': [p.serialize() for p in prods], - 'missing': missing, + 'found': [p.serialize() for p in final_prods], + 'missing': missing_ids, + 'restricted': [] #restricted_ids + #'load_ids': load_ids } return JsonResponse(out) From 5e1d6db078d633937cc80e3ba0630d77f30d387b Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:41:37 -0800 Subject: [PATCH 6/9] add search for all model names --- procat2/urls.py | 3 ++- products/views.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/procat2/urls.py b/procat2/urls.py index f6bac76..d4539d4 100644 --- a/procat2/urls.py +++ b/procat2/urls.py @@ -22,7 +22,7 @@ from lazysignup.views import convert from dashboard.views import dashboard from cataloglist.views import cataloglist, my_catalogs, public_catalogs from catalogedit.views import catalogedit, get_catalog, save_catalog -from products.views import search_products +from products.views import search_products, all_models from .forms import UserCreationForm from .views import login_guest, lazy_convert_done @@ -43,6 +43,7 @@ urlpatterns = [ path('api/v1/catalogs/save', save_catalog, name='save_catalog'), path('api/v1/products/search', search_products, name='search_products'), + path('api/v1/products/models', all_models, name='all_models'), path('admin/', admin.site.urls), path("account/", include("account.urls")), diff --git a/products/views.py b/products/views.py index 3dcc6a4..a7b2d3f 100644 --- a/products/views.py +++ b/products/views.py @@ -3,6 +3,7 @@ from django.shortcuts import render, get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods from django.db import connections +from django.db.models import Count import json import logging @@ -117,3 +118,35 @@ def search_products(request): } return JsonResponse(out) + + +@csrf_exempt +@login_required +@require_http_methods(["GET"]) +def all_models(request): + # TODO ignoring for now + + # body = request.body + # if not body or len(body) < 1: + # return HttpResponse('Bad request: no data', status=400) + + # data = json.loads(body.decode('utf-8')) + + # season_id = data.get('season') + # if not season_id or len(season_id) < 1: + # return HttpResponse('Bad request: no season id', status=400) + # season = Season.objects.get(id=season_id) + # if not season: + # return HttpResponse('Bad request: no season found', status=400) + + # region_id = data.get('region') + # if not region_id or len(region_id) < 1: + # return HttpResponse('Bad request: no region id', status=400) + # region = Region.objects.get(id=region_id) + # if not region: + # return HttpResponse('Bad request: no region found', status=400) + + models = Product.objects.all().values('name').annotate(total=Count('id')).order_by('name') + log.debug('loaded %s model names', len(models)) + + return JsonResponse(list(models), safe=False) From 1889fd2e74a96a83349eedafcd242be3234f4b6d Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:44:14 -0800 Subject: [PATCH 7/9] catalogedit.html: add material icons css link --- templates/catalogedit/catalogedit.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/catalogedit/catalogedit.html b/templates/catalogedit/catalogedit.html index 8c45945..8420ad1 100644 --- a/templates/catalogedit/catalogedit.html +++ b/templates/catalogedit/catalogedit.html @@ -10,6 +10,7 @@ ProCatalog Editor + {{ catalogID | json_script:"catalogID" }} {{ regions | json_script:"regions" }} {{ seasons | json_script:"seasons" }} From 7e0ecc77eb3f9dbf3831c1e4f3202048ac47cabc Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:49:11 -0800 Subject: [PATCH 8/9] CatalogContents.vue: placeholder for pagination css work --- cateditor/src/components/CatalogContents.vue | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cateditor/src/components/CatalogContents.vue b/cateditor/src/components/CatalogContents.vue index 4b4a584..6eacb68 100644 --- a/cateditor/src/components/CatalogContents.vue +++ b/cateditor/src/components/CatalogContents.vue @@ -2,7 +2,9 @@ - Add products + + Add materials + @@ -64,6 +66,7 @@ @@ -438,4 +441,8 @@ function addSectionDrops(myvue) { max-height: 400px; overflow-y: auto; } + +.foo { + border-top: 3px solid red; +} From cde0805992fe52c781b9b104127816a5d9200a41 Mon Sep 17 00:00:00 2001 From: Seth Ladygo Date: Tue, 14 Jan 2020 12:50:14 -0800 Subject: [PATCH 9/9] product picker: add tabs for new picking methods WIP --- cateditor/src/components/AddCatalogPanel.vue | 89 ++++++++++++++++++ cateditor/src/components/AddMaterialPanel.vue | 44 +++++++++ cateditor/src/components/AddModelPanel.vue | 92 +++++++++++++++++++ cateditor/src/components/AddProductDialog.vue | 73 ++++++++++----- 4 files changed, 273 insertions(+), 25 deletions(-) create mode 100644 cateditor/src/components/AddCatalogPanel.vue create mode 100644 cateditor/src/components/AddMaterialPanel.vue create mode 100644 cateditor/src/components/AddModelPanel.vue diff --git a/cateditor/src/components/AddCatalogPanel.vue b/cateditor/src/components/AddCatalogPanel.vue new file mode 100644 index 0000000..8bf7373 --- /dev/null +++ b/cateditor/src/components/AddCatalogPanel.vue @@ -0,0 +1,89 @@ + + + diff --git a/cateditor/src/components/AddMaterialPanel.vue b/cateditor/src/components/AddMaterialPanel.vue new file mode 100644 index 0000000..e8e3737 --- /dev/null +++ b/cateditor/src/components/AddMaterialPanel.vue @@ -0,0 +1,44 @@ +b + + + diff --git a/cateditor/src/components/AddModelPanel.vue b/cateditor/src/components/AddModelPanel.vue new file mode 100644 index 0000000..9c3437d --- /dev/null +++ b/cateditor/src/components/AddModelPanel.vue @@ -0,0 +1,92 @@ + + + diff --git a/cateditor/src/components/AddProductDialog.vue b/cateditor/src/components/AddProductDialog.vue index bf60303..37b5669 100644 --- a/cateditor/src/components/AddProductDialog.vue +++ b/cateditor/src/components/AddProductDialog.vue @@ -1,33 +1,39 @@