Add options to group within item lists

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Reduce updates when showing item lists; add a waiting spinner ([#1653](../../pull/1653))
 - Update item lists check for large images when toggling recurse ([#1654](../../pull/1654))
 - Support named item lists ([#1665](../../pull/1665))
+- Add options to group within item lists ([#1666](../../pull/1666))
 
 ### Changes
 

docs/conf.py

@@ -13,8 +13,6 @@
 import os
 import sys
 
-import sphinx_rtd_theme
-
 docs_dir = os.path.dirname(__file__)
 sys.path.insert(0, os.path.abspath(os.path.join(docs_dir, '..', '..')))
 
@@ -62,7 +60,6 @@
 # a list of builtin themes.
 #
 html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 pygments_style = 'sphinx'
 

docs/girder_config_options.rst

@@ -61,11 +61,35 @@ This is used to specify how items appear in item lists.  There are two settings,
     itemList:
       # layout does not need to be specified.
       layout:
+        # The default list (with flatten: false) shows only the items in the
+        # current folder; flattening the list shows items in the current folder
+        # and all subfolders.  This can also be "only", in which case the
+        # flatten option will start enabled and, when flattened, the folder
+        # list will be hidden.
+        flatten: true
         # The default layout is a list.  This can optionally be "grid"
         mode: grid
         # max-width is only used in grid mode.  It is the maximum width in
         # pixels for grid entries.  It defaults to 250.
         max-width: 250
+      # group does not need to be specified.  Instead of listing items
+      # directly, multiple items can be grouped  together.
+      group:
+        # keys is a single metadata value reference (see the column metadata
+        # records), or a list of such records.
+        keys: dicom.PatientID
+        # counts is optional.  If specified, the left side is either a metadata
+        # value references or "_id" to just count total items.  The right side
+        # is where, conceptually, the count is stored in the item.meta record.
+        # to show a column of the counts, add a metadata column with a value
+        # equal to this.  That is, in this example, all items with the same
+        # meta.dicom.PatientID are grouped as a single row, and two count
+        # columns are generated.  The unique values for each group row of
+        # meta.dicom.StudyInstanceUID and counted and that count is added to
+        # meta._count.studiescount.
+        counts:
+          dicom.StudyInstanceUID: _count.studiescount
+          dicom.SeriesInstanceUID: _count.seriescount
       # show these columns in order from left to right.  Each column has a
       # "type" and "value".  It optionally has a "title" used for the column
       # header, and a "format" used for searching and filtering.  The "label",

girder/girder_large_image/rest/__init__.py

@@ -1,3 +1,4 @@
+import collections
 import json
 
 from girder import logger
@@ -9,7 +10,7 @@
 from girder.models.item import Item
 
 
-def addSystemEndpoints(apiRoot):
+def addSystemEndpoints(apiRoot):  # noqa
     """
     This adds endpoints to routes that already exist in Girder.
 
@@ -29,6 +30,9 @@ def altItemFind(self, folderId, text, name, limit, offset, sort, filters=None):
         if text and text.startswith('_recurse_:'):
             recurse = True
             text = text.split('_recurse_:', 1)[1]
+        group = None
+        if text and text.startswith('_group_:') and len(text.split(':', 2)) >= 3:
+            _, group, text = text.split(':', 2)
         if filters is None and text and text.startswith('_filter_:'):
             try:
                 filters = json.loads(text.split('_filter_:', 1)[1].strip())
@@ -40,9 +44,10 @@ def altItemFind(self, folderId, text, name, limit, offset, sort, filters=None):
                 logger.debug('Item find filters: %s', json.dumps(filters))
             except Exception:
                 pass
-        if recurse:
+        if recurse or group:
             return _itemFindRecursive(
-                self, origItemFind, folderId, text, name, limit, offset, sort, filters)
+                self, origItemFind, folderId, text, name, limit, offset, sort,
+                filters, recurse, group)
         return origItemFind(folderId, text, name, limit, offset, sort, filters)
 
     @boundHandler(apiRoot.item)
@@ -58,7 +63,55 @@ def altFolderFind(self, parentType, parentId, text, name, limit, offset, sort, f
         altFolderFind._origFunc = origFolderFind
 
 
-def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, sort, filters):
+def _groupingPipeline(initialPipeline, cbase, grouping, sort=None):
+    """
+    Modify the recursive pipeline to add grouping and counts.
+
+    :param initialPipeline: a pipeline to extend.
+    :param cbase: a unique value for each grouping set.
+    :param grouping: a dictionary where 'keys' is a list of data to group by
+        and, optionally, 'counts' is a dictionary of data to count as keys and
+        names where to add the results.  For instance, this could be
+        {'keys': ['meta.dicom.PatientID'], 'counts': {
+        'meta.dicom.StudyInstanceUID': 'meta._count.studycount',
+        'meta.dicom.SeriesInstanceUID': 'meta._count.seriescount'}}
+    :param sort: an optional lost of (key, direction) tuples
+    """
+    for gidx, gr in enumerate(grouping['keys']):
+        grsort = [(gr, 1)] + (sort or [])
+        initialPipeline.extend([{
+            '$match': {gr: {'$exists': True}},
+        }, {
+            '$sort': collections.OrderedDict(grsort),
+        }, {
+            '$group': {
+                '_id': f'${gr}',
+                'firstOrder': {'$first': '$$ROOT'},
+            },
+        }])
+        groupStep = initialPipeline[-1]['$group']
+        if not gidx and grouping['counts']:
+            for cidx, (ckey, cval) in enumerate(grouping['counts'].items()):
+                groupStep[f'count_{cbase}_{cidx}'] = {'$addToSet': f'${ckey}'}
+                cparts = cval.split('.')
+                centry = {cparts[-1]: {'$size': f'$count_{cbase}_{cidx}'}}
+                for cidx in range(len(cparts) - 2, -1, -1):
+                    centry = {
+                        cparts[cidx]: {
+                            '$mergeObjects': [
+                                '$firstOrder.' + '.'.join(cparts[:cidx + 1]),
+                                centry,
+                            ],
+                        },
+                    }
+                initialPipeline.append({'$set': {'firstOrder': {
+                    '$mergeObjects': ['$firstOrder', centry]}}})
+        initialPipeline.append({'$replaceRoot': {'newRoot': '$firstOrder'}})
+
+
+def _itemFindRecursive(  # noqa
+        self, origItemFind, folderId, text, name, limit, offset, sort, filters,
+        recurse=True, group=None):
     """
     If a recursive search within a folderId is specified, use an aggregation to
     find all folders that are descendants of the specified folder.  If there
@@ -73,20 +126,23 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset,
     from bson.objectid import ObjectId
 
     if folderId:
-        pipeline = [
-            {'$match': {'_id': ObjectId(folderId)}},
-            {'$graphLookup': {
-                'from': 'folder',
-                'connectFromField': '_id',
-                'connectToField': 'parentId',
-                'depthField': '_depth',
-                'as': '_folder',
-                'startWith': '$_id',
-            }},
-            {'$group': {'_id': '$_folder._id'}},
-        ]
-        children = [ObjectId(folderId)] + next(Folder().collection.aggregate(pipeline))['_id']
-        if len(children) > 1:
+        if recurse:
+            pipeline = [
+                {'$match': {'_id': ObjectId(folderId)}},
+                {'$graphLookup': {
+                    'from': 'folder',
+                    'connectFromField': '_id',
+                    'connectToField': 'parentId',
+                    'depthField': '_depth',
+                    'as': '_folder',
+                    'startWith': '$_id',
+                }},
+                {'$group': {'_id': '$_folder._id'}},
+            ]
+            children = [ObjectId(folderId)] + next(Folder().collection.aggregate(pipeline))['_id']
+        else:
+            children = [ObjectId(folderId)]
+        if len(children) > 1 or group:
             filters = (filters.copy() if filters else {})
             if text:
                 filters['$text'] = {
@@ -98,6 +154,69 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset,
             user = self.getCurrentUser()
             if isinstance(sort, list):
                 sort.append(('parentId', 1))
+
+            # This is taken from girder.utility.acl_mixin.findWithPermissions,
+            # except it adds a grouping stage
+            initialPipeline = [
+                {'$match': filters},
+                {'$lookup': {
+                    'from': 'folder',
+                    'localField': Item().resourceParent,
+                    'foreignField': '_id',
+                    'as': '__parent',
+                }},
+                {'$match': Item().permissionClauses(user, AccessType.READ, '__parent.')},
+                {'$project': {'__parent': False}},
+            ]
+            if group is not None:
+                if not isinstance(group, list):
+                    group = [gr for gr in group.split(',') if gr]
+                groups = []
+                idx = 0
+                while idx < len(group):
+                    if group[idx] != '_count_':
+                        if not len(groups) or groups[-1]['counts']:
+                            groups.append({'keys': [], 'counts': {}})
+                        groups[-1]['keys'].append(group[idx])
+                        idx += 1
+                    else:
+                        if idx + 3 <= len(group):
+                            groups[-1]['counts'][group[idx + 1]] = group[idx + 2]
+                        idx += 3
+                for gidx, grouping in enumerate(groups):
+                    _groupingPipeline(initialPipeline, gidx, grouping, sort)
+            fullPipeline = initialPipeline
+            countPipeline = initialPipeline + [
+                {'$count': 'count'},
+            ]
+            if sort is not None:
+                fullPipeline.append({'$sort': collections.OrderedDict(sort)})
+            if limit:
+                fullPipeline.append({'$limit': limit + (offset or 0)})
+            if offset:
+                fullPipeline.append({'$skip': offset})
+
+            logger.debug('Find item pipeline %r', fullPipeline)
+
+            options = {
+                'allowDiskUse': True,
+                'cursor': {'batchSize': 0},
+            }
+            result = Item().collection.aggregate(fullPipeline, **options)
+
+            def count():
+                try:
+                    return next(iter(
+                        Item().collection.aggregate(countPipeline, **options)))['count']
+                except StopIteration:
+                    # If there are no values, this won't return the count, in
+                    # which case it is zero.
+                    return 0
+
+            result.count = count
+            result.fromAggregate = True
+            return result
+
             return Item().findWithPermissions(filters, offset, limit, sort=sort, user=user)
     return origItemFind(folderId, text, name, limit, offset, sort, filters)
 

girder/girder_large_image/web_client/templates/itemList.pug

@@ -92,7 +92,7 @@ ul.g-item-list.li-item-list(layout_mode=(itemList.layout || {}).mode || '')
                     != String(value).replace(/&/g, '&amp;').replace(/</, '&lt;').replace(/>/, '&gt;').replace(/"/, '&quot').replace(/'/, '&#39;').replace(/\./g, '.&shy;').replace(/_/g, '_&shy;')
                   else
                     = value
-                if value
+                if value && column.format !== 'count'
                   span.li-item-list-cell-filter(title="Only show items that match this metadata value exactly", filter-value=value, column-value=column.value)
                     i.icon-filter
   if (hasMore && !paginated)

girder/girder_large_image/web_client/views/itemList.js

@@ -30,12 +30,16 @@ wrap(HierarchyWidget, 'initialize', function (initialize, settings) {
 
 wrap(HierarchyWidget, 'render', function (render) {
     render.call(this);
+    if (this.parentModel.resourceName !== 'folder') {
+        this.$('.g-folder-list-container').toggleClass('hidden', false);
+    }
     if (!this.$('#flattenitemlist').length && this.$('.g-item-list-container').length && this.itemListView && this.itemListView.setFlatten) {
         $('button.g-checked-actions-button').parent().after(
             '<div class="li-flatten-item-list" title="Check to show items in all subfolders in this list"><input type="checkbox" id="flattenitemlist"></input><label for="flattenitemlist">Flatten</label></div>'
         );
-        if ((this.itemListView || {})._recurse) {
+        if ((this.itemListView || {})._recurse && this.parentModel.resourceName === 'folder') {
             this.$('#flattenitemlist').prop('checked', true);
+            this.$('.g-folder-list-container').toggleClass('hidden', this.itemListView._hideFoldersOnFlatten);
         }
         this.events['click #flattenitemlist'] = (evt) => {
             this.itemListView.setFlatten(this.$('#flattenitemlist').is(':checked'));
@@ -96,6 +100,16 @@ wrap(ItemListWidget, 'initialize', function (initialize, settings) {
             this.render();
             return;
         }
+        if (!_.isEqual(val, this._liconfig) && !this.$el.closest('.modal-dialog').length && val) {
+            this._liconfig = val;
+            const list = this._confList();
+            if (list.layout && list.layout.flatten !== undefined) {
+                this._recurse = !!list.layout.flatten;
+                this.parentView.$('#flattenitemlist').prop('checked', this._recurse);
+            }
+            this._hideFoldersOnFlatten = !!(list.layout && list.layout.flatten === 'only');
+            this.parentView.$('.g-folder-list-container').toggleClass('hidden', this._hideFoldersOnFlatten);
+        }
         delete this._lastSort;
         this._liconfig = val;
         const curRoute = Backbone.history.fragment;
@@ -138,6 +152,7 @@ wrap(ItemListWidget, 'initialize', function (initialize, settings) {
     this.setFlatten = (flatten) => {
         if (!!flatten !== !!this._recurse) {
             this._recurse = !!flatten;
+            this.parentView.$('.g-folder-list-container').toggleClass('hidden', this._hideFoldersOnFlatten && this._recurse);
             this._setFilter();
             this.render();
         }
@@ -335,6 +350,23 @@ wrap(ItemListWidget, 'render', function (render) {
                 filter = '_filter_:' + JSON.stringify(filter);
             }
         }
+        const group = (this._confList() || {}).group || undefined;
+        if (group) {
+            if (group.keys.length) {
+                let grouping = '_group_:meta.' + group.keys.join(',meta.');
+                if (group.counts) {
+                    for (let [gkey, gval] of Object.entries(group.counts)) {
+                        if (!gkey.includes(',') && !gkey.includes(':') && !gval.includes(',') && !gval.includes(':')) {
+                            if (gkey !== '_id') {
+                                gkey = `meta.${gkey}`;
+                            }
+                            grouping += `,_count_,${gkey},meta.${gval}`;
+                        }
+                    }
+                }
+                filter = grouping + ':' + (filter || '');
+            }
+        }
         if (this._recurse) {
             filter = '_recurse_:' + (filter || '');
         }
@@ -485,9 +517,7 @@ function sortColumn(evt) {
     }
 }
 
-function itemListCellFilter(evt) {
-    evt.preventDefault();
-    const cell = $(evt.target).closest('.li-item-list-cell-filter');
+function addCellToFilter(cell, update) {
     let filter = this._generalFilter || '';
     let val = cell.attr('filter-value');
     let col = cell.attr('column-value');
@@ -499,7 +529,15 @@ function itemListCellFilter(evt) {
     filter = filter.trim();
     this.$el.closest('.g-hierarchy-widget').find('.li-item-list-filter-input').val(filter);
     this._generalFilter = filter;
-    this._setFilter();
+    if (update !== false) {
+        this._setFilter();
+    }
+}
+
+function itemListCellFilter(evt) {
+    evt.preventDefault();
+    const cell = $(evt.target).closest('.li-item-list-cell-filter');
+    addCellToFilter.call(this, cell);
     addToRoute({filter: this._generalFilter});
     this._setSort();
     return false;