Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Update Bias Monitor to use Django DB Models #1503

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

[WIP] Update Bias Monitor to use Django DB Models #1503

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dbea5c7
updated bias monitor to work with new django databases
bsunnquist Feb 28, 2024
6740da6
bias monitor bokeh plots now working with django databases
bsunnquist Feb 28, 2024
a8a8311
removing unnecessary imports
bsunnquist Feb 28, 2024
f0f5d91
Merge branch 'develop' into bias-monitor-django-db
mfixstsci Nov 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 28 additions & 26 deletions jwql/instrument_monitors/common_monitors/bias_monitor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import datetime
import logging
import os
from time import sleep

from astropy.io import fits
from astropy.stats import sigma_clip, sigma_clipped_stats
Expand All @@ -47,21 +46,26 @@
from mpl_toolkits.axes_grid1 import make_axes_locatable # noqa: E402 (module import not at top)
import numpy as np # noqa: E402 (module import not at top)
from pysiaf import Siaf # noqa: E402 (module import not at top)
from sqlalchemy.sql.expression import and_ # noqa: E402 (module import not at top)

from jwql.database.database_interface import session, engine # noqa: E402 (module import not at top)
from jwql.database.database_interface import NIRCamBiasQueryHistory, NIRCamBiasStats, NIRISSBiasQueryHistory # noqa: E402 (module import not at top)
from jwql.database.database_interface import NIRISSBiasStats, NIRSpecBiasQueryHistory, NIRSpecBiasStats # noqa: E402 (module import not at top)
from jwql.instrument_monitors import pipeline_tools # noqa: E402 (module import not at top)
from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline # noqa: E402 (module import not at top)
from jwql.shared_tasks.shared_tasks import only_one, run_parallel_pipeline # noqa: E402 (module import not at top)
from jwql.utils import instrument_properties, monitor_utils # noqa: E402 (module import not at top)
from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE # noqa: E402 (module import not at top)
from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS, ON_READTHEDOCS # noqa: E402 (module import not at top)
from jwql.utils.logging_functions import log_info, log_fail # noqa: E402 (module import not at top)
from jwql.utils.monitor_utils import update_monitor_table # noqa: E402 (module import not at top)
from jwql.utils.permissions import set_permissions # noqa: E402 (module import not at top)
from jwql.utils.utils import copy_files, ensure_dir_exists, filesystem_path, get_config # noqa: E402 (module import not at top)
from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config # noqa: E402 (module import not at top)
from jwql.website.apps.jwql.monitor_pages.monitor_bias_bokeh import BiasMonitorPlots # noqa: E402 (module import not at top)

if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
# Need to set up django apps before we can access the models
import django # noqa: E402 (module level import not at top of file)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
django.setup()

# Import * is okay here because this module specifically only contains database models
# for this monitor
from jwql.website.apps.jwql.monitor_models.bias import * # noqa: E402 (module level import not at top of file)


class Bias():
"""Class for executing the bias monitor.
Expand Down Expand Up @@ -201,15 +205,13 @@ def file_exists_in_database(self, filename):
``True`` if filename exists in the bias stats database.
"""

query = session.query(self.stats_table)
results = query.filter(self.stats_table.uncal_filename == filename).all()
records = self.stats_table.objects.filter(uncal_filename__iexact=filename).all()

if len(results) != 0:
if len(records) != 0:
file_exists = True
else:
file_exists = False

session.close()
return file_exists

def get_amp_medians(self, image, amps):
Expand Down Expand Up @@ -346,16 +348,16 @@ def most_recent_search(self):
where the bias monitor was run.
"""

query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture,
self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all() # noqa: E348 (comparison to true)
filters = {'aperture__iexact': self.aperture,
'run_monitor': True}
record = self.query_table.objects.filter(**filters).order_by('-end_time_mjd').first()

if len(query) == 0:
if record is None:
query_result = 59607.0 # a.k.a. Jan 28, 2022 == First JWST images (MIRI)
logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'.format(self.aperture, query_result)))
else:
query_result = query[-1].end_time_mjd
query_result = record.end_time_mjd

session.close()
return query_result

def process(self, file_list):
Expand Down Expand Up @@ -420,18 +422,18 @@ def process(self, file_list):
'mean': float(mean),
'median': float(median),
'stddev': float(stddev),
'collapsed_rows': collapsed_rows.astype(float),
'collapsed_columns': collapsed_columns.astype(float),
'counts': counts.astype(float),
'bin_centers': bin_centers.astype(float),
'collapsed_rows': list(collapsed_rows.astype(float)),
'collapsed_columns': list(collapsed_columns.astype(float)),
'counts': list(counts.astype(float)),
'bin_centers': list(bin_centers.astype(float)),
'entry_date': datetime.datetime.now()
}
for key in amp_medians.keys():
bias_db_entry[key] = float(amp_medians[key])

# Add this new entry to the bias database table
with engine.begin() as connection:
connection.execute(self.stats_table.__table__.insert(), bias_db_entry)
entry = self.stats_table(**bias_db_entry)
entry.save()

# Don't print long arrays of numbers to the log file
log_dict = {}
Expand Down Expand Up @@ -545,8 +547,8 @@ def run(self):
'files_found': len(new_files),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()}
with engine.begin() as connection:
connection.execute(self.query_table.__table__.insert(), new_entry)
entry = self.query_table(**new_entry)
entry.save()
logging.info('\tUpdated the query history table')

# Update the bias monitor plots
Expand Down
126 changes: 51 additions & 75 deletions jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,30 +20,34 @@
monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL')
"""

from datetime import datetime, timedelta
from datetime import datetime
import os

from astropy.stats import sigma_clip

from bokeh.embed import components, file_html
from bokeh.layouts import layout
from bokeh.models import ColorBar, ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearAxis
from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool
from bokeh.models.layouts import Tabs, TabPanel
from bokeh.plotting import figure, output_file, save
from bokeh.resources import CDN
from datetime import datetime, timedelta
from datetime import datetime
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Repeated import of datetime

import numpy as np
import pandas as pd
from PIL import Image
from sqlalchemy import func

from jwql.bokeh_templating import BokehTemplate
from jwql.database.database_interface import get_unique_values_per_column, NIRCamBiasStats, NIRISSBiasStats, NIRSpecBiasStats, session
from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE
from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS, ON_READTHEDOCS
from jwql.utils.permissions import set_permissions
from jwql.utils.utils import read_png
from jwql.website.apps.jwql.bokeh_utils import PlaceholderPlot

if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
# Need to set up django apps before we can access the models
import django # noqa: E402 (module level import not at top of file)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
django.setup()

# Import * is okay here because this module specifically only contains database models
# for this monitor
from jwql.website.apps.jwql.monitor_models.bias import * # noqa: E402 (module level import not at top of file)


SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
TEMPLATE_DIR = os.path.join(SCRIPT_DIR, '../templates')
Expand All @@ -67,8 +71,8 @@ class BiasMonitorData():
Latest bias data for a particular aperture, from the
stats_table

stats_table : sqlalchemy.orm.decl_api.DeclarativeMeta
Bias stats sqlalchemy table
stats_table : jwql.website.apps.jwql.monitor_models.bias.NIRCamBiasStats
Bias stats django database

trending_data : pandas.DataFrame
Data from the stats table to be used for the trending plot
Expand All @@ -93,35 +97,24 @@ def retrieve_trending_data(self, aperture):
"""
# Query database for all data in bias stats with a matching aperture,
# and sort the data by exposure start time.
tmp_trending_data = session.query(self.stats_table.amp1_even_med,
self.stats_table.amp1_odd_med,
self.stats_table.amp2_even_med,
self.stats_table.amp2_odd_med,
self.stats_table.amp3_even_med,
self.stats_table.amp3_odd_med,
self.stats_table.amp4_even_med,
self.stats_table.amp4_odd_med,
self.stats_table.expstart,
self.stats_table.uncal_filename) \
.filter(self.stats_table.aperture == aperture) \
.order_by(self.stats_table.expstart) \
.all()

session.close()
columns = ['amp1_even_med', 'amp1_odd_med', 'amp2_even_med', 'amp2_odd_med',
'amp3_even_med', 'amp3_odd_med', 'amp4_even_med', 'amp4_odd_med',
'expstart', 'uncal_filename']
tmp_trending_data = self.stats_table.objects.filter(aperture__iexact=aperture).order_by('expstart').all().values(*columns)

# Convert the query results to a pandas dataframe
self.trending_data = pd.DataFrame(tmp_trending_data, columns=['amp1_even_med', 'amp1_odd_med',
'amp2_even_med', 'amp2_odd_med',
'amp3_even_med', 'amp3_odd_med',
'amp4_even_med', 'amp4_odd_med',
'expstart_str', 'uncal_filename'])
uncal_basename = [os.path.basename(e) for e in self.trending_data['uncal_filename']]
self.trending_data['uncal_filename'] = uncal_basename

# Add a column of expstart values that are datetime objects
format_data = "%Y-%m-%dT%H:%M:%S.%f"
datetimes = [datetime.strptime(entry, format_data) for entry in self.trending_data['expstart_str']]
self.trending_data['expstart'] = datetimes
if len(tmp_trending_data) != 0:
self.trending_data = pd.DataFrame.from_records(tmp_trending_data)
uncal_basename = [os.path.basename(e) for e in self.trending_data['uncal_filename']]
self.trending_data['uncal_filename'] = uncal_basename

# Add a column of expstart values that are datetime objects
format_data = "%Y-%m-%dT%H:%M:%S.%f"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If there are complaints in the log about missing timezone info, you could update the strings and use the %z option in strptime. For this to work, the strings would have to change to be e.g. '2021-09-01 15:27:05.004573 +0000'

datetimes = [datetime.strptime(entry, format_data) for entry in self.trending_data['expstart']]
self.trending_data['expstart_str'] = self.trending_data['expstart']
self.trending_data['expstart'] = datetimes
else:
self.trending_data = pd.DataFrame(None, columns=columns + ['uncal_filename', 'expstart_str'])

def retrieve_latest_data(self, aperture):
"""Query the database table to get the data needed for the non-trending
Expand All @@ -132,40 +125,23 @@ def retrieve_latest_data(self, aperture):
aperture : str
Aperture name (e.g. NRCA1_FULL)
"""
subq = (session.query(self.stats_table.aperture, func.max(self.stats_table.expstart).label("max_created"))
.group_by(self.stats_table.aperture)
.subquery()
)

query = (session.query(self.stats_table.aperture,
self.stats_table.uncal_filename,
self.stats_table.cal_filename,
self.stats_table.cal_image,
self.stats_table.expstart,
self.stats_table.collapsed_rows,
self.stats_table.collapsed_columns,
self.stats_table.counts,
self.stats_table.bin_centers,
self.stats_table.entry_date)
.filter(self.stats_table.aperture == aperture)
.order_by(self.stats_table.entry_date) \
.join(subq, self.stats_table.expstart == subq.c.max_created)
)

latest_data = query.all()
session.close()

# Put the returned data in a dataframe. Include only the most recent entry.
# The query has already filtered to include only entries using the latest
# expstart value.
self.latest_data = pd.DataFrame(latest_data[-1:], columns=['aperture', 'uncal_filename', 'cal_filename',
'cal_image', 'expstart_str', 'collapsed_rows',
'collapsed_columns', 'counts', 'bin_centers',
'entry_date'])
# Add a column of expstart values that are datetime objects
format_data = "%Y-%m-%dT%H:%M:%S.%f"
datetimes = [datetime.strptime(entry, format_data) for entry in self.latest_data['expstart_str']]
self.latest_data['expstart'] = datetimes
# Query database for the most recent bias stats entry with a matching aperture
columns = ['aperture', 'uncal_filename', 'cal_filename', 'cal_image', 'expstart',
'collapsed_rows', 'collapsed_columns', 'counts', 'bin_centers', 'entry_date']
tmp_data = self.stats_table.objects.filter(aperture__iexact=aperture).order_by('-expstart').all().values(*columns).first()

# Put the returned data in a dataframe
if tmp_data is not None:
# Orient and transpose needed due to list column entries e.g. counts
self.latest_data = pd.DataFrame.from_dict(tmp_data, orient='index').transpose()

# Add a column of expstart values that are datetime objects
format_data = "%Y-%m-%dT%H:%M:%S.%f"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here, if the log complains about a lack of timezone info

datetimes = [datetime.strptime(entry, format_data) for entry in self.latest_data['expstart']]
self.latest_data['expstart_str'] = self.latest_data['expstart']
self.latest_data['expstart'] = datetimes
else:
self.latest_data = pd.DataFrame(None, columns=columns + ['expstart_str'])


class BiasMonitorPlots():
Expand Down Expand Up @@ -232,7 +208,7 @@ def __init__(self, instrument):
self.db = BiasMonitorData(self.instrument)

# Now we need to loop over the available apertures and create plots for each
self.available_apertures = get_unique_values_per_column(self.db.stats_table, 'aperture')
self.available_apertures = sorted(self.db.stats_table.objects.values_list('aperture', flat=True).distinct())

# Make sure all full frame apertures are present. If there are no data for a
# particular full frame entry, then produce an empty plot, in order to
Expand Down Expand Up @@ -298,7 +274,7 @@ def ensure_all_full_frame_apertures(self):
self.available_apertures.append(ap)

def modify_bokeh_saved_html(self):
"""Given an html string produced by Bokeh when saving bad pixel monitor plots,
"""Given an html string produced by Bokeh when saving bias monitor plots,
make tweaks such that the page follows the general JWQL page formatting.
"""
# Insert into our html template and save
Expand Down
Loading