Merge branch 'prodpicker_improvements'

.gitignore

@@ -4,3 +4,5 @@
 /debug.log
 /cateditor/vue.config.js*
 /users.csv
+/markup/work/*.pdf
+/markup/work/*.eml

cateditor/src/components/AddCatalogPanel.vue

@@ -0,0 +1,89 @@
+<template>
+  <v-card flat>
+    <v-card-text>
+      <v-container fluid class="pa-0 ma-0">
+        <v-row class="pa-0 ma-0">
+          <v-col cols="6" class="pa-0 ma-0">
+            <v-radio-group v-model="catalogType" class="pa-0 ma-0">
+              <v-radio class="pa-0 ma-0"
+                       label="My catalogs"
+                       on-icon="radio_button_checked"
+                       off-icon="radio_button_unchecked"
+                       value="mine"/>
+            </v-radio-group>
+          </v-col>
+          <v-col cols="6" class="pa-0 ma-0">
+            <v-radio-group v-model="catalogType" class="pa-0 ma-0">
+              <v-radio class="pa-0 ma-0"
+                       label="Public catalogs"
+                       value="public"/>
+            </v-radio-group>
+          </v-col>
+        </v-row>
+      </v-container>
+
+      <v-autocomplete
+        v-model="selectedCatalog"
+        :items="catalogChoices"
+        :loading="isLoading"
+        :search-input.sync="search"
+        item-text="name"
+        item-value="id"
+        label="Select catalog"
+        :no-data-text="noDataText"
+        outlined
+      />
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  props: {
+    value: {
+      type: Number,
+      required: true,
+      default: -1,
+    },
+  },
+  data: () => ({
+    myCatalogs: null,
+    publicCatalogs: null,
+    catalogTypeInternal: 'mine',
+    isLoading: false,
+    search: null,
+  }),
+  computed: {
+    catalogType: {
+      get() {
+        return this.catalogTypeInternal
+      },
+      set(value) {
+        this.catalogTypeInternal = value
+        this.value = -1
+      }
+    },
+    catalogChoices: {
+      get() {
+        if (this.catalogTypeInternal === 'mine') {
+          //if (this.myCatalogs.length < 1) {
+          //this.isLoading = true
+          /* fetch('/api/v1/catalogs/mine')
+             *   .then(res => this.myCatalogs = res.json())
+             *   .catch(err => {
+             *     // TODO
+             *     console.log(err)
+             *   })
+             *   .finally(() => (this.isLoading = false))
+             * } */
+          //}
+          //return this.myCatalogs
+          return [{ name: 'SS20 New Zealand' }, { name: 'SS20 Australia' }]
+        } else {
+          return [{ name: 'SS20 New Zealand - Joe User' }, { name: 'SS20 Australia - Joe User' }]
+        }
+      },
+    },
+  },
+}
+</script>

cateditor/src/components/AddMaterialPanel.vue

@@ -0,0 +1,44 @@
+b
+<template>
+  <v-card flat>
+    <v-card-text>
+      <v-textarea
+        v-model="text"
+        label="Enter material numbers"
+        autofocus
+        outlined
+        clearable
+        clear-icon="cancel"
+        rows="6"
+        placeholder="1001234
+1001235..."
+      />
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+//import { mapActions } from 'vuex'
+
+export default {
+  props: {
+    value: {
+      type: String,
+      required: true,
+      default: '',
+    },
+  },
+  data: () => ({
+  }),
+  computed: {
+    text: {
+      get() {
+        return this.value
+      },
+      set(value) {
+        this.$emit('input', value)
+      }
+    },
+  },
+}
+</script>

cateditor/src/components/AddModelPanel.vue

@@ -0,0 +1,92 @@
+<template>
+  <v-card flat>
+    <v-card-text>
+      <v-autocomplete
+        v-model="selectedModels"
+        :items="items"
+        :loading="isLoading"
+        :search-input.sync="search"
+        item-text="name"
+        item-value="name"
+        label="Select models by name"
+        :no-data-text="noDataText"
+        multiple
+        chips
+        small-chips
+        deletable-chips
+        dense
+        outlined>
+
+        <template v-slot:item="data">
+          <v-list-item-content>
+            <v-list-item-title v-text="data.item.name"/>
+            <v-list-item-subtitle>{{ data.item.total }} {{ data.item.total | pluralize('material') }}</v-list-item-subtitle>
+          </v-list-item-content>
+        </template>
+
+      </v-autocomplete>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  props: {
+    value: {
+      type: Array,
+      required: true,
+    },
+  },
+  data: () => ({
+    entries: [],
+    isLoading: false,
+    search: null,
+  }),
+  computed: {
+    noDataText() {
+      if (this.items.length > 0) {
+        return 'No models found for: ' + this.search
+      } else {
+        return 'Start typing to search...'
+      }
+    },
+    selectedModels: {
+      get() {
+        return this.value
+      },
+      set(value) {
+        this.$emit('input', value)
+      }
+    },
+    items() {
+      return this.entries
+      /* return this.entries.map(entry => {
+       *   const Description = entry.Description.length > this.descriptionLimit
+       *     ? entry.Description.slice(0, this.descriptionLimit) + '...'
+       *     : entry.Description
+
+       *   return Object.assign({}, entry, { Description })
+      })
+       */
+    },
+  },
+  watch: {
+    search(val) {
+      if (this.items.length > 0) return
+      if (this.isLoading) return
+      this.isLoading = true
+
+      fetch('/api/v1/products/models')
+        .then(res => res.json())
+        .then(res => {
+          this.entries = res
+        })
+        .catch(err => {
+          // TODO
+          console.log(err)
+        })
+        .finally(() => (this.isLoading = false))
+    },
+  },
+}
+</script>

cateditor/src/components/AddProductDialog.vue

@@ -1,33 +1,39 @@
 <template>
   <div class="text-center">
-    <v-dialog v-model="show" width="280">
+    <v-dialog v-model="show" width="500">
       <v-card>
+        <DialogHeading title="Add Materials">
+          <v-btn icon class="ma-0 pa-0" @click="show = false">
+            <v-icon color="white">clear</v-icon>
+          </v-btn>
+        </DialogHeading>
 
-        <v-card class="subheading font-weight-bold ma-0 pa-0 white--text" color="keen_dark_grey">
+        <v-card-text class="ma-0 pa-0">
-          <v-card-title class="ma-0 pa-2 pl-3">
-            Add Materials
-            <v-spacer/>
-            <v-icon color="white" @click="show = false">clear</v-icon>
-          </v-card-title>
-        </v-card>
 
-        <v-card-text>
+          <v-tabs v-model="tab" grow>
-          <p>Enter material numbers to add to the catalog</p>
+            <v-tab>By ID</v-tab>
-          <v-textarea
+            <v-tab>By Name</v-tab>
-            v-model="text"
+            <v-tab>From Catalog</v-tab>
-            autofocus
+          </v-tabs>
-            label="Material numbers"
+
-            placeholder="1001234
+          <v-tabs-items v-model="tab">
-1001235..."
+            <v-tab-item>
-            clearable
+              <AddMaterialPanel v-model="materialText"/>
-          />
+            </v-tab-item>
+            <v-tab-item>
+              <AddModelPanel v-model="models"/>
+            </v-tab-item>
+            <v-tab-item>
+              <AddCatalogPanel v-model="catalog"/>
+            </v-tab-item>
+          </v-tabs-items>
         </v-card-text>
 
         <v-card-actions>
           <v-spacer/>
-          <v-btn small text @click="show = false">Cancel</v-btn>
+          <v-btn depressed @click="show = false">Cancel</v-btn>
           <v-spacer/>
-          <v-btn small text color="primary" @click="doAdd()">Add</v-btn>
+          <v-btn depressed color="primary" class="black--text" @click="doAdd()">Add</v-btn>
           <v-spacer/>
         </v-card-actions>
 
@@ -38,13 +44,26 @@
 
 <script>
 import { mapActions } from 'vuex'
+import AddMaterialPanel from './AddMaterialPanel'
+import AddModelPanel from './AddModelPanel'
+import AddCatalogPanel from './AddCatalogPanel'
+import DialogHeading from './DialogHeading'
 
 export default {
+  components: {
+    AddMaterialPanel,
+    AddModelPanel,
+    AddCatalogPanel,
+    DialogHeading,
+  },
   props: {
     value: Boolean,
   },
   data: () => ({
-    text: null,
+    tab: 1,
+    materialText: '',
+    models: [],
+    catalog: null,
   }),
   computed: {
     show: {
@@ -60,11 +79,15 @@ export default {
     ...mapActions([
       'fetchProducts',
     ]),
-
     doAdd: function() {
-      console.log('adding text', this.text)
+      if (this.tab === 0) {
-      this.show = false
+        this.fetchProducts(this.materialText)
-      this.fetchProducts(this.text)
+      } else if (this.tab === 1) {
+        console.log('adding models', this.models)
+      } else {
+        console.log('adding from catalog', this.catalog)
+      }
+      /*       this.show = false */
     },
 
   },

cateditor/src/components/CatalogContents.vue

@@ -2,7 +2,9 @@
   <v-container fluid>
 
     <v-row class="fill-height">
-      <v-btn @click="showAddProductDialog = true">Add products</v-btn>
+      <v-col cols="4">
+        <v-btn @click="showAddProductDialog = true">Add materials</v-btn>
+      </v-col>
     </v-row>
 
     <v-row class="fill-height">
@@ -64,6 +66,7 @@
                 <v-list-item
                   slot-scope="{ hover }"
                   :class="modelListItemClasses(item.model, hover)"
+                  class="foo"
                   @click="selectModel(item.model)"
                 >
                   <v-list-item-content>
@@ -438,4 +441,8 @@ function addSectionDrops(myvue) {
   max-height: 400px;
   overflow-y: auto;
 }
+
+.foo {
+  border-top: 3px solid red;
+}
 </style>

markup/img.py

@@ -20,12 +20,16 @@ def find_shapes(image_path):
     """
     path = Path(image_path)
 
-    img = Image.open(image_path, 'r')
+    print('finding shapes in {}'.format(image_path))
-    if not img.mode in ('RGBA', 'LA'):
-        print('no alpha channel: {}'.format(img.mode))
-        return None
 
+    img = Image.open(image_path, 'r')
+    if img.mode == 'RGBA':
         alpha_layer = img.convert('RGBA').split()[-1]
+    elif img.mode == 'L':
+        alpha_layer = img
+    else:
+        print('unhandled image mode: {}'.format(img.mode))
+        return None
 
     alpha_layer = alpha_layer.filter(ImageFilter.GaussianBlur(5))
 

markup/pdf.py

@@ -2,7 +2,9 @@ import os
 import sys
 import subprocess
 import shutil
+import dumper
 
+from pdfminer.psparser import LIT
 from pdfminer.pdfparser import PDFParser
 from pdfminer.pdfdocument import PDFDocument
 from pdfminer.pdftypes import PDFObjRef, resolve1
@@ -59,8 +61,15 @@ def make_scribble(obj, pagenum, mediabox, workdir):
                  'rect': pdf_rect(rect, mediabox[3]),
                  'objid': im1.objid,
                  'image': path }
+    elif flter.name == 'FlateDecode':
+        path = export_netpbm(im1, workdir, pagenum)
+        return { 'page': pagenum,
+                 'rect': pdf_rect(rect, mediabox[3]),
+                 'objid': im1.objid,
+                 'image': path }
     else:
-        print('skipping non-jp2 image')
+        print('skipping unrecognized image')
+        # print(dumper.dump(im1))
         return None
 
 
@@ -85,6 +94,55 @@ def export_jp2(obj, workdir, pagenum):
     return png_path
 
 
+def export_netpbm(obj, workdir, pagenum):
+    oid = obj.objid
+    ensure_dir(workdir)
+
+    pbm_base = os.path.join(workdir, f"export-page{pagenum:03d}-obj{oid:05d}")
+    pbm_path = write_pbm(obj, pbm_base)
+
+    # stencil mask - use instead if present
+    smask = obj.attrs['SMask']
+    if smask:
+        print('extracting pbm mask')
+        mask = resolve1(smask)
+        mask_base = os.path.join(workdir, f"export-page{pagenum:03d}-obj{oid:05d}-mask")
+        mask_path = write_pbm(smask, mask_base)
+        pbm_path = mask_path
+
+    return pbm_path
+
+
+def write_pbm(obj, base_path):
+    obj = resolve1(obj)
+    color_space = resolve1(obj.attrs['ColorSpace'])
+
+    suffix = '.pgm' if color_space == LIT('DeviceGray') else '.ppm'
+    path = base_path + suffix
+
+    print('writing pbm: {}'.format(path))
+
+    data = obj.get_data()
+    with open(path, 'wb') as out:
+        if suffix == '.pgm':
+            out.write("P5\n".encode())
+        else:
+            out.write("P6\n".encode())
+
+        out.write("{} {}\n".format(obj.attrs['Width'], obj.attrs['Height']).encode())
+
+        if obj.attrs['BitsPerComponent'] == 8:
+            out.write("255\n".encode())
+        else:
+            out.write("65535\n".encode())
+
+        out.write(data)
+
+    set_file_perms(path)
+
+    return path
+
+
 def parse_pdf(fname, workdir, debug=0):
     PDFDocument.debug = debug
     PDFParser.debug = debug
@@ -120,7 +178,8 @@ def parse_pdf(fname, workdir, debug=0):
             elif 'ProCatName' in anno:
                 prod_boxes.append(make_product_box(anno, pagenum, mediabox))
             else:
-                print('ignoring other annotation')
+                print('ignoring other annotation:')
+                print(anno)
 
     fp.close()
 

procat2/urls.py

@@ -22,7 +22,7 @@ from lazysignup.views import convert
 from dashboard.views import dashboard
 from cataloglist.views import cataloglist, my_catalogs, public_catalogs
 from catalogedit.views import catalogedit, get_catalog, save_catalog
-from products.views import search_products
+from products.views import search_products, all_models
 
 from .forms import UserCreationForm
 from .views import login_guest, lazy_convert_done
@@ -43,6 +43,7 @@ urlpatterns = [
     path('api/v1/catalogs/save', save_catalog, name='save_catalog'),
 
     path('api/v1/products/search', search_products, name='search_products'),
+    path('api/v1/products/models', all_models, name='all_models'),
 
     path('admin/', admin.site.urls),
     path("account/", include("account.urls")),

products/models.py

@@ -14,6 +14,7 @@ class Product(models.Model):
     id = models.CharField(max_length=20, db_column='id', primary_key=True)
     sap = models.CharField(max_length=10, db_column='material')
     season = models.CharField(max_length=10, db_column='season')
+    season_ranking = models.DecimalField(max_digits=6, decimal_places=1)
     name = models.CharField(max_length=100, db_column='model')
     model = models.CharField(max_length=100, db_column='modelcode')
     gender = models.CharField(max_length=100, db_column='gender')

products/views.py

@@ -2,6 +2,8 @@ from django.http import HttpResponseRedirect, HttpResponse, JsonResponse
 from django.shortcuts import render, get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.http import require_http_methods
+from django.db import connections
+from django.db.models import Count
 
 import json
 import logging
@@ -44,30 +46,107 @@ def search_products(request):
         return HttpResponse('Bad request: no region found', status=400)
 
     ids = Product.find_sap_ids(text)
-    log.info('found ids %s in %s', ids, text)
+    log.info('found %s ids %s in %s', len(ids), ids, text)
 
-    prods = []
+    final_prods = []
-    missing = []
+    missing_ids = []
+    restricted_ids = []
 
     if ids:
-        srm = SeasonRegionMaterial.objects.filter(material__in=ids, season=season.id, region__iexact=region.id).distinct('material')
+        srm = SeasonRegionMaterial.objects.filter(material__in=ids, season=season.id, region__iexact=region.id).order_by('ranking')
         srm_ids = [x.material for x in srm]
-        log.info('found SRM ids %s', srm_ids)
+        log.debug('found %s ids: %s', len(srm_ids), srm_ids)
 
-        search_prods = Product.objects.filter(sap__in=srm_ids,season=season.id).distinct('sap')
+        # in search list but not found in current season/region ranking
+        missing_ids = (list(set(ids) - set(srm_ids)))
+        log.debug('no season/region hits on %s ids: %s', len(missing_ids), missing_ids)
 
-        # fix product order to match input ids and find missing ids
+        # missing ids ranked *somewhere*. these are our restricted ids.
-        prod_dict = dict([(p.sap, p) for p in search_prods])
+        restricted = SeasonRegionMaterial.objects.filter(material__in=missing_ids).order_by('material').distinct('material')
+        restricted_ids = [x.material for x in restricted]
+        log.debug('found %s restricted ids: %s', len(restricted_ids), restricted_ids)
 
-        for i in ids:
+        # don't include restricted in the missing list
-            if prod_dict.get(i):
+        missing_ids = (list(set(missing_ids) - set(restricted_ids)))
-                prods.append(prod_dict[i])
+        log.debug('actually missing %s ids: %s', len(missing_ids), missing_ids)
-            else:
+
-                missing.append(i)
+        # load product info for current season
+        load_ids = srm_ids + restricted_ids
+        prods = Product.objects.filter(sap__in=load_ids)
+        #prods = Product.objects.filter(sap__in=srm_ids, season=season.id)
+        prods_ids = [x.sap for x in prods]
+        log.debug('loaded %s season products: %s', len(prods), prods_ids)
+
+        # find info for prods ranking in keen_season_region_material
+        # but not in keen_materials for the current season.  in this
+        # case, fall back to the most recent data we have for the
+        # prods.  if it's (S/R/M) ranked, it should be returned as a
+        # valid material as per Ed Reilly.
+        missing_prod_ids = (list(set(srm_ids) - set(prods_ids)))
+        fill_in_prods = []
+        if missing_prod_ids:
+            log.debug('looking for %s out-of-season products: %s', len(missing_prod_ids), missing_prod_ids)
+            fill_in_prods = Product.objects.filter(sap__in=missing_prod_ids).order_by('sap', '-season_ranking').distinct('sap')
+            log.debug('found %s out-of-season products: %s', len(fill_in_prods), [x.sap for x in fill_in_prods])
+
+        all_prods = list(prods) + list(fill_in_prods)
+        prod_dict = dict([(p.sap, p) for p in all_prods])
+
+        # remove related kids who have a parent in the catalog.
+        # they'll get added automatically to the parent section.
+        with connections['products'].cursor() as cursor:
+            cursor.execute('select child from keen_related_kids where parent = any(%s)', [srm_ids])
+            for row in cursor.fetchall():
+                kid = row[0]
+                if kid in prod_dict:
+                    log.debug('removing related kid: %s', kid)
+                    del prod_dict[kid]
+
+        # fix product order to match srm ranking
+        #for id in srm_ids:
+        for id in load_ids:
+            if prod_dict.get(id):
+                final_prods.append(prod_dict[id])
+
+        log.debug('returning %s products: %s', len(final_prods), [x.sap for x in final_prods])
 
     out = {
-        'found': [p.serialize() for p in prods],
+        'found': [p.serialize() for p in final_prods],
-        'missing': missing,
+        'missing': missing_ids,
+        'restricted': [] #restricted_ids
+        #'load_ids': load_ids
     }
 
     return JsonResponse(out)
+
+
+@csrf_exempt
+@login_required
+@require_http_methods(["GET"])
+def all_models(request):
+    # TODO ignoring for now
+
+    # body = request.body
+    # if not body or len(body) < 1:
+    #     return HttpResponse('Bad request: no data', status=400)
+
+    # data = json.loads(body.decode('utf-8'))
+
+    # season_id = data.get('season')
+    # if not season_id or len(season_id) < 1:
+    #     return HttpResponse('Bad request: no season id', status=400)
+    # season = Season.objects.get(id=season_id)
+    # if not season:
+    #     return HttpResponse('Bad request: no season found', status=400)
+
+    # region_id = data.get('region')
+    # if not region_id or len(region_id) < 1:
+    #     return HttpResponse('Bad request: no region id', status=400)
+    # region = Region.objects.get(id=region_id)
+    # if not region:
+    #     return HttpResponse('Bad request: no region found', status=400)
+
+    models = Product.objects.all().values('name').annotate(total=Count('id')).order_by('name')
+    log.debug('loaded %s model names', len(models))
+
+    return JsonResponse(list(models), safe=False)

templates/catalogedit/catalogedit.html

@@ -10,6 +10,7 @@
     <link href="{% static 'img/favicon.ico' %}" rel="shortcut icon" />
     <title>ProCatalog Editor</title>
     <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900">
+    <link href="https://fonts.googleapis.com/css?family=Material+Icons" rel="stylesheet">
     {{ catalogID | json_script:"catalogID" }}
     {{ regions | json_script:"regions" }}
     {{ seasons | json_script:"seasons" }}