Fix for gaps in segments causing unfilled pie chart plots

gwsumm/plot/segments.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (C) Duncan Macleod (2013)
+#               Evan Goetz (2023)
 #
 # This file is part of GWSumm.
 #
@@ -1006,15 +1007,54 @@
         wedgeargs = self.parse_wedge_kwargs()
         plotargs = self.parse_plot_kwargs()
 
+        # Boolean logic flags to determine if this code is currently running:
+        # - before the span of interest (current time the code is running <
+        #   start of span)
+        # - after the span of interest (current time the code is running >= end
+        #   of span)
+        # - during the span of interest (any other time)
+        # The flag is set to True for the appropriate noun (before, during, or
+        # after).
+        # These flags are then used to set plot titles and labels and determine
+        # if there is any missing data.
+        before = during = after = False
+        if globalv.NOW < int(self.span[0]):
+            before = True
+        elif globalv.NOW >= int(self.span[1]):
+            after = True
+        else:
+            during = True
+
         # use state to generate suptitle with GPS span
+        # this will be different depending on if `include_future` is given
+        # as an option or whether running before, during, or after the time
+        # interval requested
         if self.state:
-            self.pargs.setdefault(
-                'suptitle',
-                '[%s-%s, state: %s]' % (self.span[0], self.span[1],
-                                        texify(str(self.state))))
+            if future or after:
+                self.pargs.setdefault(
+                    'suptitle',
+                    (f'[{self.span[0]}-{self.span[1]}, '
+                     f'state: {texify(str(self.state))}]'))
+            elif before:
+                self.pargs.setdefault(
+                    'suptitle',
+                    (f'[{self.span[0]}-{self.span[0]}, '
+                     f'state: {texify(str(self.state))}]'))
+            else:
+                self.pargs.setdefault(
+                    'suptitle',
+                    (f'[{self.span[0]}-{globalv.NOW}, '
+                     f'state: {texify(str(self.state))}]'))
         else:
-            self.pargs.setdefault(
-                'suptitle', '[%s-%s]' % (self.span[0], self.span[1]))
+            if future or after:
+                self.pargs.setdefault(
+                    'suptitle', f'[{self.span[0]}-{self.span[1]}]')
+            elif before:
+                self.pargs.setdefault(
+                    'suptitle', f'[{self.span[0]}-{self.span[0]}]')
+            else:
+                self.pargs.setdefault(
+                    'suptitle', f'[{self.span[0]}-{globalv.NOW}]')
 
         # get segments
         data = []
@@ -1028,12 +1068,39 @@
                                 padding=self.padding).coalesce()
             data.append(float(abs(segs.active)))
 
-        # handle missing or future data
+        # handle missing (undefined) segments
+        # if running before then all the time is future because segments
+        # haven't been generated
+        # if running after then some segments may not cover the whole time
+        # if during, it is somewhere in between
         total = float(sum(data))
-        if future or (total < alltime):
-            data.append(alltime - total)
+        undefined = future_seg = 0
+        if before:
+            future_seg = alltime
+        elif after:
+            undefined = alltime - total
+        elif during:
+            future_seg = int(self.span[1]) - globalv.NOW
+            undefined = alltime - future_seg - total
+            current_total = globalv.NOW - int(self.span[0])
+
+        # figure out the extra pieces to include in the pie chart and labels
+        # TODO: There is something messed up about "labels" and
+        #       "label" that should be cleaned up
+        if undefined > 0:
+            data.append(undefined)
+            if 'labels' in plotargs:
+                plotargs['labels'] = list(plotargs['labels']) + ['Undefined']
+            elif 'label' in plotargs:
+                plotargs['label'] = list(plotargs['label']) + ['Undefined']
+            if 'colors' in plotargs:
+                plotargs['colors'] = list(plotargs['colors']) + ['black']
+        if future or before:
+            data.append(future_seg)
             if 'labels' in plotargs:
                 plotargs['labels'] = list(plotargs['labels']) + [' ']
+            elif 'label' in plotargs:
+                plotargs['label'] = list(plotargs['label']) + [' ']
             if 'colors' in plotargs:
                 plotargs['colors'] = list(plotargs['colors']) + ['white']
 
@@ -1058,7 +1125,12 @@
                 pclabels.append(label)
             else:
                 try:
-                    pc = d / (total if future else alltime) * 100
+                    if future or after:
+                        pc = d / alltime * 100
+                    elif during:
+                        pc = d / current_total * 100
+                    else:
+                        pc = 0.0
                 except ZeroDivisionError:
                     pc = 0.0
                 pclabels.append(texify(
@@ -1150,28 +1222,45 @@
         # construct compound flags for each network size
         flags = dict((f[:2], f) for f in self.flags)
         network = ''.join(sorted(set(flags)))
-        self.pargs.setdefault('title', '%s network duty factor' % network)
+        self.pargs.setdefault('title', f'{network} network duty factor')
         networkflags = []
         colors = []
         labels = []
+        # define an exclude DQ flag so that each subsequent time through
+        # We exclude triple time from double time and double time from single
+        # time
         exclude = DataQualityFlag()
         for i in list(range(len(flags)+1))[::-1]:
             name = self.NETWORK_NAME[i]
-            flag = '%s:%s' % (network, name)
+            flag = f'{network}:{name}'
             networksegs = DataQualityFlag(flag, known=valid)
+            # loop over the possible combinations inserting the flag to the
+            # network segments dictionary
             for ifoset in combinations(flags, i):
                 if not ifoset:
-                    compound = '!%s' % '!'.join(list(flags.values()))
+                    compound = f"!{'!'.join(list(flags.values()))}"
                 else:
                     compound = '&'.join(flags[ifo] for ifo in ifoset)
-                segs = get_segments(compound, validity=valid, query=False,
-                                    padding=self.padding).coalesce()
+                segs = get_segments(
+                    compound, validity=valid, query=False,
+                    padding=self.padding, ignore_undefined=True).coalesce()
                 networksegs += segs
+            # Final step in the loop for no detectors: if not wanting to plot
+            # future times, then exclude the time from now to the end of the
+            # span from the no detector network
+            if (i == 0 and
+                    not self.pargs.get('include_future', False) and
+                    globalv.NOW < self.span[1]):
+                exclude.active += SegmentList(
+                    [Segment(globalv.NOW, self.span[1])])
+            # insert this flag into the segments global variable and exclude
+            # any of the previous network (more detectors) time from this time
             globalv.SEGMENTS[flag] = networksegs.copy()
             globalv.SEGMENTS[flag].active -= exclude.active
+            # update the segements of times to exclude
             exclude = networksegs
             networkflags.append(flag)
-            labels.append('%s interferometer' % name.title())
+            labels.append(f'{name.title()} interferometer')
             colors.append(self.NETWORK_COLOR.get(name))
 
         self.pargs.setdefault('colors', colors)

gwsumm/segments.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (C) Duncan Macleod (2013)
+#               Evan Goetz (2023)
 #
 # This file is part of GWSumm.
 #
@@ -22,7 +23,6 @@
 import sys
 import operator
 import warnings
-from functools import reduce
 from collections import OrderedDict
 from configparser import (
     DEFAULTSECT,
@@ -34,7 +34,7 @@
 from astropy.io.registry import IORegistryError
 
 from gwpy.segments import (DataQualityFlag, DataQualityDict,
-                           SegmentList, Segment)
+                           SegmentListDict, SegmentList, Segment)
 
 from . import globalv
 from .utils import (
@@ -56,7 +56,8 @@
 
 def get_segments(flag, validity=None, config=ConfigParser(), cache=None,
                  query=True, return_=True, coalesce=True, padding=None,
-                 segdb_error='raise', url=None, **read_kw):
+                 ignore_undefined=False, segdb_error='raise', url=None,
+                 **read_kw):
     """Retrieve the segments for a given flag
 
     Segments will be loaded from global memory if already defined,
@@ -97,6 +98,11 @@
         `(start, end)` padding with which to pad segments that are
         downloaded/read
 
+    ignore_undefined : `bool`, optional, default: `False`
+        Special case needed for network calculation compound flags so that
+        when this is True, DataQualityFlag.known values are set to the same
+        value as ``validity``
+
     segdb_error : `str`, optional, default: ``'raise'``
         how to handle errors returned from the segment database, one of
 
@@ -168,11 +174,12 @@
     for f in allflags:
         globalv.SEGMENTS.setdefault(f, DataQualityFlag(f))
 
-    # read segments from global memory and get the union of needed times
+    # read segments from global memory and get the intersection of needed times
     try:
-        old = reduce(
-            operator.and_,
-            (globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags))
+        old = SegmentListDict()
+        for f in flags:
+            old[f] = globalv.SEGMENTS.get(f, DataQualityFlag(f)).known
+        old = SegmentList(old.intersection(flags))
     except TypeError:
         old = SegmentList()
     newsegs = validity - old
@@ -260,15 +267,17 @@
         for compound in flags:
             union, intersection, exclude, notequal = split_compound_flag(
                 compound)
-            if len(union + intersection) == 1:
-                out[compound].description = globalv.SEGMENTS[f].description
-                out[compound].padding = padding.get(f, (0, 0))
+            if len(f := (union + intersection)) == 1:
+                out[compound].description = globalv.SEGMENTS[f[0]].description
+                out[compound].padding = padding.get(f[0], (0, 0))
             for flist, op in zip([exclude, intersection, union, notequal],
                                  [operator.sub, operator.and_, operator.or_,
                                   not_equal]):
                 for f in flist:
                     pad = padding.get(f, (0, 0))
                     segs = globalv.SEGMENTS[f].copy()
+                    if ignore_undefined:
+                        segs.known = validity
                     if isinstance(pad, (float, int)):
                         segs = segs.pad(pad, pad)
                     elif pad is not None: