From 79c8f36a43b5d98c48b51121ef9156e25d069481 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Wed, 21 Feb 2024 16:58:37 -0800 Subject: [PATCH 1/6] Update filesystem monitor to use django db --- jwql/jwql_monitors/monitor_filesystem.py | 42 +++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 2c1f4d379..bcb531458 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -49,22 +49,27 @@ import numpy as np from sqlalchemy.exc import DataError -from jwql.database.database_interface import engine -from jwql.database.database_interface import session -from jwql.database.database_interface import FilesystemCharacteristics -from jwql.database.database_interface import FilesystemGeneral -from jwql.database.database_interface import FilesystemInstrument -from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table from jwql.utils.protect_module import lock_module from jwql.website.apps.jwql.data_containers import get_instrument_proposals +if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: + # Need to set up django apps before we can access the models + import django # noqa: E402 (module level import not at top of file) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") + django.setup() + + # Import * is okay here because this module specifically only contains database models + # for this monitor + from jwql.website.apps.jwql.monitor_models.common import * # noqa: E402 (module level import not at top of file) + SETTINGS = get_config() FILESYSTEM = SETTINGS['filesystem'] PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary') @@ -426,9 +431,9 @@ def update_central_store_database(central_storage_dict): new_record['size'] = central_storage_dict[area]['size'] new_record['used'] = central_storage_dict[area]['used'] new_record['available'] = central_storage_dict[area]['available'] - with engine.begin() as connection: - connection.execute(CentralStore.__table__.insert(), new_record) - session.close() + + entry = CentralStorage(**new_record) + entry.save() def update_characteristics_database(char_info): @@ -454,11 +459,9 @@ def update_characteristics_database(char_info): new_record['instrument'] = instrument new_record['filter_pupil'] = optics new_record['obs_per_filter_pupil'] = values - with engine.begin() as connection: - connection.execute( - FilesystemCharacteristics.__table__.insert(), new_record) - session.close() + entry = FilesystemCharacteristics(**new_record) + entry.save() def update_database(general_results_dict, instrument_results_dict, central_storage_dict): @@ -474,8 +477,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora """ logging.info('\tUpdating the database') - with engine.begin() as connection: - connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict) + fs_general_entry = FilesystemGeneral(**general_results_dict) + fs_general_entry.save() # Add data to filesystem_instrument table for instrument in JWST_INSTRUMENT_NAMES: @@ -489,13 +492,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora # Protect against updated enum options that have not been propagated to # the table definition - try: - with engine.begin() as connection: - connection.execute(FilesystemInstrument.__table__.insert(), new_record) - except DataError as e: - logging.error(e) - - session.close() + fs_instrument_entry = FilesystemInstrument(**new_record) + fs_instrument_entry.save() @lock_module From c023f81f57db153d1386ec78a296ce3ecfd44897 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 12 Nov 2024 20:36:08 -0500 Subject: [PATCH 2/6] Updating fields to be array fields with proper fields and defaults --- .../apps/jwql/monitor_models/common.py | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index ebcf7c82a..96e6938c7 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -172,11 +172,14 @@ class Meta: For more information please see: ```https://docs.djangoproject.com/en/2.0/topics/db/models/``` """ + # This is an auto-generated Django model module. # Feel free to rename the models, but don't rename db_table values or field names. from django.db import models from django.contrib.postgres.fields import ArrayField +from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER + class Monitor(models.Model): monitor_name = models.CharField() @@ -187,7 +190,7 @@ class Monitor(models.Model): class Meta: managed = True - db_table = 'monitor' + db_table = "monitor" class CentralStorage(models.Model): @@ -199,18 +202,34 @@ class CentralStorage(models.Model): class Meta: managed = True - db_table = 'central_storage' + db_table = "central_storage" class FilesystemCharacteristics(models.Model): date = models.DateTimeField() instrument = models.TextField() # This field type is a guess. - filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. - obs_per_filter_pupil = models.TextField(blank=True, null=True) # This field type is a guess. + filter_pupil = ArrayField( + models.CharField( + max_length=MAX_LEN_FILTER, + help_text="source file names", + default=DEFAULT_MODEL_CHARFIELD, + ), + blank=True, + null=True, + ) + obs_per_filter_pupil = ArrayField( + models.CharField( + max_length=MAX_LEN_FILTER, + help_text="source file names", + default=DEFAULT_MODEL_CHARFIELD, + ), + blank=True, + null=True, + ) class Meta: managed = True - db_table = 'filesystem_characteristics' + db_table = "filesystem_characteristics" class FilesystemGeneral(models.Model): @@ -224,7 +243,7 @@ class FilesystemGeneral(models.Model): class Meta: managed = True - db_table = 'filesystem_general' + db_table = "filesystem_general" class FilesystemInstrument(models.Model): @@ -236,8 +255,8 @@ class FilesystemInstrument(models.Model): class Meta: managed = True - db_table = 'filesystem_instrument' - unique_together = (('date', 'instrument', 'filetype'),) + db_table = "filesystem_instrument" + unique_together = (("date", "instrument", "filetype"),) class FgsAnomaly(models.Model): @@ -257,7 +276,7 @@ class FgsAnomaly(models.Model): class Meta: managed = True - db_table = 'fgs_anomaly' + db_table = "fgs_anomaly" class MiriAnomaly(models.Model): @@ -274,15 +293,19 @@ class MiriAnomaly(models.Model): row_pull_down = models.BooleanField() other = models.BooleanField() column_pull_down = models.BooleanField() - mrs_glow = models.BooleanField(db_column='MRS_Glow') # Field name made lowercase. - mrs_zipper = models.BooleanField(db_column='MRS_Zipper') # Field name made lowercase. + mrs_glow = models.BooleanField(db_column="MRS_Glow") # Field name made lowercase. + mrs_zipper = models.BooleanField( + db_column="MRS_Zipper" + ) # Field name made lowercase. row_pull_up = models.BooleanField() - lrs_contamination = models.BooleanField(db_column='LRS_Contamination') # Field name made lowercase. + lrs_contamination = models.BooleanField( + db_column="LRS_Contamination" + ) # Field name made lowercase. tree_rings = models.BooleanField() class Meta: managed = True - db_table = 'miri_anomaly' + db_table = "miri_anomaly" class NircamAnomaly(models.Model): @@ -307,7 +330,7 @@ class NircamAnomaly(models.Model): class Meta: managed = True - db_table = 'nircam_anomaly' + db_table = "nircam_anomaly" class NirissAnomaly(models.Model): @@ -329,7 +352,7 @@ class NirissAnomaly(models.Model): class Meta: managed = True - db_table = 'niriss_anomaly' + db_table = "niriss_anomaly" class NirspecAnomaly(models.Model): @@ -345,10 +368,12 @@ class NirspecAnomaly(models.Model): data_transfer_error = models.BooleanField() ghost = models.BooleanField() snowball = models.BooleanField() - dominant_msa_leakage = models.BooleanField(db_column='Dominant_MSA_Leakage') # Field name made lowercase. + dominant_msa_leakage = models.BooleanField( + db_column="Dominant_MSA_Leakage" + ) # Field name made lowercase. optical_short = models.BooleanField() other = models.BooleanField() class Meta: managed = True - db_table = 'nirspec_anomaly' + db_table = "nirspec_anomaly" From 69680686437f478cef6016ff944c056d0dac158e Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 12 Nov 2024 21:25:25 -0500 Subject: [PATCH 3/6] Updating field from guess of char to int --- jwql/website/apps/jwql/monitor_models/common.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 96e6938c7..527130c25 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -211,18 +211,14 @@ class FilesystemCharacteristics(models.Model): filter_pupil = ArrayField( models.CharField( max_length=MAX_LEN_FILTER, - help_text="source file names", + help_text="filter and/or pupil name", default=DEFAULT_MODEL_CHARFIELD, ), blank=True, null=True, ) obs_per_filter_pupil = ArrayField( - models.CharField( - max_length=MAX_LEN_FILTER, - help_text="source file names", - default=DEFAULT_MODEL_CHARFIELD, - ), + models.IntegerField(), blank=True, null=True, ) From 1e83d51afb127caae72e99ca49b6ef6e0d2b6c8b Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 14 Nov 2024 11:23:41 -0500 Subject: [PATCH 4/6] Adding timezone info --- jwql/jwql_monitors/monitor_filesystem.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 0809a2e69..295d2ea01 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -79,6 +79,7 @@ PREVIEW_IMAGES = SETTINGS['preview_image_filesystem'] THUMBNAILS = SETTINGS['thumbnail_filesystem'] LOGS = SETTINGS['log_dir'] +WORKING = SETTINGS['working'] def files_per_filter(): @@ -237,7 +238,8 @@ def get_area_stats(central_storage_dict): 'logs': LOGS, 'preview_images': PREVIEW_IMAGES, 'thumbnails': THUMBNAILS, - 'all': CENTRAL} + 'all': CENTRAL, + 'working':WORKING} counteddirs = [] @@ -373,7 +375,7 @@ def initialize_results_dicts(): A dictionary for the ``central_storage`` database table """ - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) general_results_dict = {} general_results_dict['date'] = now @@ -452,7 +454,7 @@ def update_characteristics_database(char_info): using that filter/pupil. """ logging.info('\tUpdating the characteristics database') - now = datetime.datetime.now() + now = datetime.datetime.now(datetime.timezone.utc) # Add data to filesystem_instrument table for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: From 7c5481ba9f04b35d5ddc007b133b126d51e6a561 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 14 Nov 2024 11:24:11 -0500 Subject: [PATCH 5/6] Adding migration file --- ...emcharacteristics_filter_pupil_and_more.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py diff --git a/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py new file mode 100644 index 000000000..2e4e04a7d --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1 on 2024-11-13 14:56 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0027_alter_fgsbadpixelstats_source_files_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='filesystemcharacteristics', + name='filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None), + ), + migrations.AlterField( + model_name='filesystemcharacteristics', + name='obs_per_filter_pupil', + field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None), + ), + ] From d4d44e25a2eb1721098477ec53dcb211de9d63c9 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 18 Nov 2024 17:25:05 -0500 Subject: [PATCH 6/6] Generating filetype and count to be EnumFields --- jwql/website/apps/jwql/monitor_models/common.py | 11 +++++++---- pyproject.toml | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_models/common.py b/jwql/website/apps/jwql/monitor_models/common.py index 527130c25..8c523d9df 100644 --- a/jwql/website/apps/jwql/monitor_models/common.py +++ b/jwql/website/apps/jwql/monitor_models/common.py @@ -177,8 +177,12 @@ class Meta: # Feel free to rename the models, but don't rename db_table values or field names. from django.db import models from django.contrib.postgres.fields import ArrayField +from django_enum import EnumField +from enum import Enum -from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER +from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER, FILE_SUFFIX_TYPES + +FILE_SUFFIX_CLASS = Enum('FILE_SUFFIX_CLASS', FILE_SUFFIX_TYPES) class Monitor(models.Model): @@ -244,9 +248,8 @@ class Meta: class FilesystemInstrument(models.Model): date = models.DateTimeField() - instrument = models.TextField() # This field type is a guess. - filetype = models.TextField() # This field type is a guess. - count = models.IntegerField() + filetype = EnumField(FILE_SUFFIX_CLASS) # This field type is a guess. + count = EnumField(FILE_SUFFIX_CLASS) size = models.FloatField() class Meta: diff --git a/pyproject.toml b/pyproject.toml index 51a66c40b..306d3c631 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "celery>=5.3.6,<6", "crds>=11.17.19,<12", "django>=5.0.3,<6", + "django-enum>=2.0.2,<3", "gunicorn>=22.0.0,<23.0.0", "inflection>=0.5.1,<0.6", "jsonschema>=4.21.1,<5",