Skip to content

Commit

Permalink
Merge pull request #1666 from girder/group-lists
Browse files Browse the repository at this point in the history 
Add options to group within item lists
  • Loading branch information
@manthey
manthey authored Oct 7, 2024
2 parents d0c49f5 + 8f6a4d2 commit ba82d9a
Show file tree
Hide file tree
Showing 6 changed files with 206 additions and 27 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
- Reduce updates when showing item lists; add a waiting spinner ([#1653](../../pull/1653))
- Update item lists check for large images when toggling recurse ([#1654](../../pull/1654))
- Support named item lists ([#1665](../../pull/1665))
- Add options to group within item lists ([#1666](../../pull/1666))

### Changes

Expand Down
3 changes: 0 additions & 3 deletions docs/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
import os
import sys

import sphinx_rtd_theme

docs_dir = os.path.dirname(__file__)
sys.path.insert(0, os.path.abspath(os.path.join(docs_dir, '..', '..')))

Expand Down Expand Up @@ -62,7 +60,6 @@
# a list of builtin themes.
#
html_theme = 'sphinx_rtd_theme'
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

pygments_style = 'sphinx'

Expand Down
24 changes: 24 additions & 0 deletions docs/girder_config_options.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,35 @@ This is used to specify how items appear in item lists. There are two settings,
itemList:
# layout does not need to be specified.
layout:
# The default list (with flatten: false) shows only the items in the
# current folder; flattening the list shows items in the current folder
# and all subfolders. This can also be "only", in which case the
# flatten option will start enabled and, when flattened, the folder
# list will be hidden.
flatten: true
# The default layout is a list. This can optionally be "grid"
mode: grid
# max-width is only used in grid mode. It is the maximum width in
# pixels for grid entries. It defaults to 250.
max-width: 250
# group does not need to be specified. Instead of listing items
# directly, multiple items can be grouped together.
group:
# keys is a single metadata value reference (see the column metadata
# records), or a list of such records.
keys: dicom.PatientID
# counts is optional. If specified, the left side is either a metadata
# value references or "_id" to just count total items. The right side
# is where, conceptually, the count is stored in the item.meta record.
# to show a column of the counts, add a metadata column with a value
# equal to this. That is, in this example, all items with the same
# meta.dicom.PatientID are grouped as a single row, and two count
# columns are generated. The unique values for each group row of
# meta.dicom.StudyInstanceUID and counted and that count is added to
# meta._count.studiescount.
counts:
dicom.StudyInstanceUID: _count.studiescount
dicom.SeriesInstanceUID: _count.seriescount
# show these columns in order from left to right. Each column has a
# "type" and "value". It optionally has a "title" used for the column
# header, and a "format" used for searching and filtering. The "label",
Expand Down
155 changes: 137 additions & 18 deletions girder/girder_large_image/rest/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import collections
import json

from girder import logger
Expand All @@ -9,7 +10,7 @@
from girder.models.item import Item


def addSystemEndpoints(apiRoot):
def addSystemEndpoints(apiRoot): # noqa
"""
This adds endpoints to routes that already exist in Girder.
Expand All @@ -29,6 +30,9 @@ def altItemFind(self, folderId, text, name, limit, offset, sort, filters=None):
if text and text.startswith('_recurse_:'):
recurse = True
text = text.split('_recurse_:', 1)[1]
group = None
if text and text.startswith('_group_:') and len(text.split(':', 2)) >= 3:
_, group, text = text.split(':', 2)
if filters is None and text and text.startswith('_filter_:'):
try:
filters = json.loads(text.split('_filter_:', 1)[1].strip())
Expand All @@ -40,9 +44,10 @@ def altItemFind(self, folderId, text, name, limit, offset, sort, filters=None):
logger.debug('Item find filters: %s', json.dumps(filters))
except Exception:
pass
if recurse:
if recurse or group:
return _itemFindRecursive(
self, origItemFind, folderId, text, name, limit, offset, sort, filters)
self, origItemFind, folderId, text, name, limit, offset, sort,
filters, recurse, group)
return origItemFind(folderId, text, name, limit, offset, sort, filters)

@boundHandler(apiRoot.item)
Expand All @@ -58,7 +63,55 @@ def altFolderFind(self, parentType, parentId, text, name, limit, offset, sort, f
altFolderFind._origFunc = origFolderFind


def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset, sort, filters):
def _groupingPipeline(initialPipeline, cbase, grouping, sort=None):
"""
Modify the recursive pipeline to add grouping and counts.
:param initialPipeline: a pipeline to extend.
:param cbase: a unique value for each grouping set.
:param grouping: a dictionary where 'keys' is a list of data to group by
and, optionally, 'counts' is a dictionary of data to count as keys and
names where to add the results. For instance, this could be
{'keys': ['meta.dicom.PatientID'], 'counts': {
'meta.dicom.StudyInstanceUID': 'meta._count.studycount',
'meta.dicom.SeriesInstanceUID': 'meta._count.seriescount'}}
:param sort: an optional lost of (key, direction) tuples
"""
for gidx, gr in enumerate(grouping['keys']):
grsort = [(gr, 1)] + (sort or [])
initialPipeline.extend([{
'$match': {gr: {'$exists': True}},
}, {
'$sort': collections.OrderedDict(grsort),
}, {
'$group': {
'_id': f'${gr}',
'firstOrder': {'$first': '$$ROOT'},
},
}])
groupStep = initialPipeline[-1]['$group']
if not gidx and grouping['counts']:
for cidx, (ckey, cval) in enumerate(grouping['counts'].items()):
groupStep[f'count_{cbase}_{cidx}'] = {'$addToSet': f'${ckey}'}
cparts = cval.split('.')
centry = {cparts[-1]: {'$size': f'$count_{cbase}_{cidx}'}}
for cidx in range(len(cparts) - 2, -1, -1):
centry = {
cparts[cidx]: {
'$mergeObjects': [
'$firstOrder.' + '.'.join(cparts[:cidx + 1]),
centry,
],
},
}
initialPipeline.append({'$set': {'firstOrder': {
'$mergeObjects': ['$firstOrder', centry]}}})
initialPipeline.append({'$replaceRoot': {'newRoot': '$firstOrder'}})


def _itemFindRecursive( # noqa
self, origItemFind, folderId, text, name, limit, offset, sort, filters,
recurse=True, group=None):
"""
If a recursive search within a folderId is specified, use an aggregation to
find all folders that are descendants of the specified folder. If there
Expand All @@ -73,20 +126,23 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset,
from bson.objectid import ObjectId

if folderId:
pipeline = [
{'$match': {'_id': ObjectId(folderId)}},
{'$graphLookup': {
'from': 'folder',
'connectFromField': '_id',
'connectToField': 'parentId',
'depthField': '_depth',
'as': '_folder',
'startWith': '$_id',
}},
{'$group': {'_id': '$_folder._id'}},
]
children = [ObjectId(folderId)] + next(Folder().collection.aggregate(pipeline))['_id']
if len(children) > 1:
if recurse:
pipeline = [
{'$match': {'_id': ObjectId(folderId)}},
{'$graphLookup': {
'from': 'folder',
'connectFromField': '_id',
'connectToField': 'parentId',
'depthField': '_depth',
'as': '_folder',
'startWith': '$_id',
}},
{'$group': {'_id': '$_folder._id'}},
]
children = [ObjectId(folderId)] + next(Folder().collection.aggregate(pipeline))['_id']
else:
children = [ObjectId(folderId)]
if len(children) > 1 or group:
filters = (filters.copy() if filters else {})
if text:
filters['$text'] = {
Expand All @@ -98,6 +154,69 @@ def _itemFindRecursive(self, origItemFind, folderId, text, name, limit, offset,
user = self.getCurrentUser()
if isinstance(sort, list):
sort.append(('parentId', 1))

# This is taken from girder.utility.acl_mixin.findWithPermissions,
# except it adds a grouping stage
initialPipeline = [
{'$match': filters},
{'$lookup': {
'from': 'folder',
'localField': Item().resourceParent,
'foreignField': '_id',
'as': '__parent',
}},
{'$match': Item().permissionClauses(user, AccessType.READ, '__parent.')},
{'$project': {'__parent': False}},
]
if group is not None:
if not isinstance(group, list):
group = [gr for gr in group.split(',') if gr]
groups = []
idx = 0
while idx < len(group):
if group[idx] != '_count_':
if not len(groups) or groups[-1]['counts']:
groups.append({'keys': [], 'counts': {}})
groups[-1]['keys'].append(group[idx])
idx += 1
else:
if idx + 3 <= len(group):
groups[-1]['counts'][group[idx + 1]] = group[idx + 2]
idx += 3
for gidx, grouping in enumerate(groups):
_groupingPipeline(initialPipeline, gidx, grouping, sort)
fullPipeline = initialPipeline
countPipeline = initialPipeline + [
{'$count': 'count'},
]
if sort is not None:
fullPipeline.append({'$sort': collections.OrderedDict(sort)})
if limit:
fullPipeline.append({'$limit': limit + (offset or 0)})
if offset:
fullPipeline.append({'$skip': offset})

logger.debug('Find item pipeline %r', fullPipeline)

options = {
'allowDiskUse': True,
'cursor': {'batchSize': 0},
}
result = Item().collection.aggregate(fullPipeline, **options)

def count():
try:
return next(iter(
Item().collection.aggregate(countPipeline, **options)))['count']
except StopIteration:
# If there are no values, this won't return the count, in
# which case it is zero.
return 0

result.count = count
result.fromAggregate = True
return result

return Item().findWithPermissions(filters, offset, limit, sort=sort, user=user)
return origItemFind(folderId, text, name, limit, offset, sort, filters)

Expand Down
2 changes: 1 addition & 1 deletion girder/girder_large_image/web_client/templates/itemList.pug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ ul.g-item-list.li-item-list(layout_mode=(itemList.layout || {}).mode || '')
!= String(value).replace(/&/g, '&amp;').replace(/</, '&lt;').replace(/>/, '&gt;').replace(/"/, '&quot').replace(/'/, '&#39;').replace(/\./g, '.&shy;').replace(/_/g, '_&shy;')
else
= value
if value
if value && column.format !== 'count'
span.li-item-list-cell-filter(title="Only show items that match this metadata value exactly", filter-value=value, column-value=column.value)
i.icon-filter
if (hasMore && !paginated)
Expand Down
48 changes: 43 additions & 5 deletions girder/girder_large_image/web_client/views/itemList.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,16 @@ wrap(HierarchyWidget, 'initialize', function (initialize, settings) {

wrap(HierarchyWidget, 'render', function (render) {
render.call(this);
if (this.parentModel.resourceName !== 'folder') {
this.$('.g-folder-list-container').toggleClass('hidden', false);
}
if (!this.$('#flattenitemlist').length && this.$('.g-item-list-container').length && this.itemListView && this.itemListView.setFlatten) {
$('button.g-checked-actions-button').parent().after(
'<div class="li-flatten-item-list" title="Check to show items in all subfolders in this list"><input type="checkbox" id="flattenitemlist"></input><label for="flattenitemlist">Flatten</label></div>'
);
if ((this.itemListView || {})._recurse) {
if ((this.itemListView || {})._recurse && this.parentModel.resourceName === 'folder') {
this.$('#flattenitemlist').prop('checked', true);
this.$('.g-folder-list-container').toggleClass('hidden', this.itemListView._hideFoldersOnFlatten);
}
this.events['click #flattenitemlist'] = (evt) => {
this.itemListView.setFlatten(this.$('#flattenitemlist').is(':checked'));
Expand Down Expand Up @@ -96,6 +100,16 @@ wrap(ItemListWidget, 'initialize', function (initialize, settings) {
this.render();
return;
}
if (!_.isEqual(val, this._liconfig) && !this.$el.closest('.modal-dialog').length && val) {
this._liconfig = val;
const list = this._confList();
if (list.layout && list.layout.flatten !== undefined) {
this._recurse = !!list.layout.flatten;
this.parentView.$('#flattenitemlist').prop('checked', this._recurse);
}
this._hideFoldersOnFlatten = !!(list.layout && list.layout.flatten === 'only');
this.parentView.$('.g-folder-list-container').toggleClass('hidden', this._hideFoldersOnFlatten);
}
delete this._lastSort;
this._liconfig = val;
const curRoute = Backbone.history.fragment;
Expand Down Expand Up @@ -138,6 +152,7 @@ wrap(ItemListWidget, 'initialize', function (initialize, settings) {
this.setFlatten = (flatten) => {
if (!!flatten !== !!this._recurse) {
this._recurse = !!flatten;
this.parentView.$('.g-folder-list-container').toggleClass('hidden', this._hideFoldersOnFlatten && this._recurse);
this._setFilter();
this.render();
}
Expand Down Expand Up @@ -335,6 +350,23 @@ wrap(ItemListWidget, 'render', function (render) {
filter = '_filter_:' + JSON.stringify(filter);
}
}
const group = (this._confList() || {}).group || undefined;
if (group) {
if (group.keys.length) {
let grouping = '_group_:meta.' + group.keys.join(',meta.');
if (group.counts) {
for (let [gkey, gval] of Object.entries(group.counts)) {
if (!gkey.includes(',') && !gkey.includes(':') && !gval.includes(',') && !gval.includes(':')) {
if (gkey !== '_id') {
gkey = `meta.${gkey}`;
}
grouping += `,_count_,${gkey},meta.${gval}`;
}
}
}
filter = grouping + ':' + (filter || '');
}
}
if (this._recurse) {
filter = '_recurse_:' + (filter || '');
}
Expand Down Expand Up @@ -485,9 +517,7 @@ function sortColumn(evt) {
}
}

function itemListCellFilter(evt) {
evt.preventDefault();
const cell = $(evt.target).closest('.li-item-list-cell-filter');
function addCellToFilter(cell, update) {
let filter = this._generalFilter || '';
let val = cell.attr('filter-value');
let col = cell.attr('column-value');
Expand All @@ -499,7 +529,15 @@ function itemListCellFilter(evt) {
filter = filter.trim();
this.$el.closest('.g-hierarchy-widget').find('.li-item-list-filter-input').val(filter);
this._generalFilter = filter;
this._setFilter();
if (update !== false) {
this._setFilter();
}
}

function itemListCellFilter(evt) {
evt.preventDefault();
const cell = $(evt.target).closest('.li-item-list-cell-filter');
addCellToFilter.call(this, cell);
addToRoute({filter: this._generalFilter});
this._setSort();
return false;
Expand Down

0 comments on commit ba82d9a

Please sign in to comment.