Update Filesystem Monitor to use Django DB Models

jwql/jwql_monitors/monitor_filesystem.py

@@ -49,22 +49,27 @@
 import numpy as np
 from sqlalchemy.exc import DataError
 
-from jwql.database.database_interface import engine
-from jwql.database.database_interface import session
-from jwql.database.database_interface import FilesystemCharacteristics
-from jwql.database.database_interface import FilesystemGeneral
-from jwql.database.database_interface import FilesystemInstrument
-from jwql.database.database_interface import CentralStore
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
 from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS
 from jwql.utils.utils import filename_parser
 from jwql.utils.utils import get_config
 from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
 from jwql.utils.protect_module import lock_module
 from jwql.website.apps.jwql.data_containers import get_instrument_proposals
 
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # Need to set up django apps before we can access the models
+    import django  # noqa: E402 (module level import not at top of file)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    django.setup()
+
+    # Import * is okay here because this module specifically only contains database models
+    # for this monitor
+    from jwql.website.apps.jwql.monitor_models.common import *  # noqa: E402 (module level import not at top of file)
+
 SETTINGS = get_config()
 FILESYSTEM = SETTINGS['filesystem']
 PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary')
@@ -74,6 +79,7 @@
 PREVIEW_IMAGES = SETTINGS['preview_image_filesystem']
 THUMBNAILS = SETTINGS['thumbnail_filesystem']
 LOGS = SETTINGS['log_dir']
+WORKING = SETTINGS['working']
 
 
 def files_per_filter():
@@ -232,7 +238,8 @@ def get_area_stats(central_storage_dict):
              'logs': LOGS,
              'preview_images': PREVIEW_IMAGES,
              'thumbnails': THUMBNAILS,
-             'all': CENTRAL}
+             'all': CENTRAL,
+             'working':WORKING}
 
     counteddirs = []
 
@@ -368,7 +375,7 @@ def initialize_results_dicts():
         A dictionary for the ``central_storage`` database table
     """
 
-    now = datetime.datetime.now()
+    now = datetime.datetime.now(datetime.timezone.utc)
 
     general_results_dict = {}
     general_results_dict['date'] = now
@@ -430,9 +437,9 @@ def update_central_store_database(central_storage_dict):
         new_record['size'] = central_storage_dict[area]['size']
         new_record['used'] = central_storage_dict[area]['used']
         new_record['available'] = central_storage_dict[area]['available']
-        with engine.begin() as connection:
-            connection.execute(CentralStore.__table__.insert(), new_record)
-    session.close()
+
+        entry = CentralStorage(**new_record)
+        entry.save()
 
 
 def update_characteristics_database(char_info):
@@ -447,7 +454,7 @@ def update_characteristics_database(char_info):
         using that filter/pupil.
     """
     logging.info('\tUpdating the characteristics database')
-    now = datetime.datetime.now()
+    now = datetime.datetime.now(datetime.timezone.utc)
 
     # Add data to filesystem_instrument table
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
@@ -458,11 +465,9 @@ def update_characteristics_database(char_info):
         new_record['instrument'] = instrument
         new_record['filter_pupil'] = optics
         new_record['obs_per_filter_pupil'] = values
-        with engine.begin() as connection:
-            connection.execute(
-                FilesystemCharacteristics.__table__.insert(), new_record)
 
-    session.close()
+        entry = FilesystemCharacteristics(**new_record)
+        entry.save()
 
 
 def update_database(general_results_dict, instrument_results_dict, central_storage_dict):
@@ -478,8 +483,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
     """
     logging.info('\tUpdating the database')
 
-    with engine.begin() as connection:
-        connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict)
+    fs_general_entry = FilesystemGeneral(**general_results_dict)
+    fs_general_entry.save()
 
     # Add data to filesystem_instrument table
     for instrument in JWST_INSTRUMENT_NAMES:
@@ -493,13 +498,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
 
             # Protect against updated enum options that have not been propagated to
             # the table definition
-            try:
-                with engine.begin() as connection:
-                    connection.execute(FilesystemInstrument.__table__.insert(), new_record)
-            except DataError as e:
-                logging.error(e)
-
-    session.close()
+            fs_instrument_entry = FilesystemInstrument(**new_record)
+            fs_instrument_entry.save()
 
 
 @lock_module

jwql/website/apps/jwql/migrations/0028_alter_filesystemcharacteristics_filter_pupil_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.1 on 2024-11-13 14:56
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0027_alter_fgsbadpixelstats_source_files_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='filesystemcharacteristics',
+            name='filter_pupil',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(default='empty', help_text='filter and/or pupil name', max_length=7), blank=True, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='filesystemcharacteristics',
+            name='obs_per_filter_pupil',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None),
+        ),
+    ]

jwql/website/apps/jwql/monitor_models/common.py

@@ -172,10 +172,17 @@ class Meta:
     For more information please see:
         ```https://docs.djangoproject.com/en/2.0/topics/db/models/```
 """
+
 # This is an auto-generated Django model module.
 # Feel free to rename the models, but don't rename db_table values or field names.
 from django.db import models
 from django.contrib.postgres.fields import ArrayField
+from django_enum import EnumField
+from enum import Enum
+
+from jwql.utils.constants import DEFAULT_MODEL_CHARFIELD, MAX_LEN_FILTER, FILE_SUFFIX_TYPES
+
+FILE_SUFFIX_CLASS = Enum('FILE_SUFFIX_CLASS', FILE_SUFFIX_TYPES)
 
 
 class Monitor(models.Model):
@@ -187,7 +194,7 @@ class Monitor(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'monitor'
+        db_table = "monitor"
 
 
 class CentralStorage(models.Model):
@@ -199,18 +206,30 @@ class CentralStorage(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'central_storage'
+        db_table = "central_storage"
 
 
 class FilesystemCharacteristics(models.Model):
     date = models.DateTimeField()
     instrument = models.TextField()  # This field type is a guess.
-    filter_pupil = models.TextField(blank=True, null=True)  # This field type is a guess.
-    obs_per_filter_pupil = models.TextField(blank=True, null=True)  # This field type is a guess.
+    filter_pupil = ArrayField(
+        models.CharField(
+            max_length=MAX_LEN_FILTER,
+            help_text="filter and/or pupil name",
+            default=DEFAULT_MODEL_CHARFIELD,
+        ),
+        blank=True,
+        null=True,
+    )
+    obs_per_filter_pupil = ArrayField(
+        models.IntegerField(),
+        blank=True,
+        null=True,
+    )
 
     class Meta:
         managed = True
-        db_table = 'filesystem_characteristics'
+        db_table = "filesystem_characteristics"
 
 
 class FilesystemGeneral(models.Model):
@@ -224,20 +243,19 @@ class FilesystemGeneral(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'filesystem_general'
+        db_table = "filesystem_general"
 
 
 class FilesystemInstrument(models.Model):
     date = models.DateTimeField()
-    instrument = models.TextField()  # This field type is a guess.
-    filetype = models.TextField()  # This field type is a guess.
-    count = models.IntegerField()
+    filetype = EnumField(FILE_SUFFIX_CLASS)  # This field type is a guess.
+    count = EnumField(FILE_SUFFIX_CLASS)
     size = models.FloatField()
 
     class Meta:
         managed = True
-        db_table = 'filesystem_instrument'
-        unique_together = (('date', 'instrument', 'filetype'),)
+        db_table = "filesystem_instrument"
+        unique_together = (("date", "instrument", "filetype"),)
 
 
 class FgsAnomaly(models.Model):
@@ -257,7 +275,7 @@ class FgsAnomaly(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'fgs_anomaly'
+        db_table = "fgs_anomaly"
 
 
 class MiriAnomaly(models.Model):
@@ -274,15 +292,19 @@ class MiriAnomaly(models.Model):
     row_pull_down = models.BooleanField()
     other = models.BooleanField()
     column_pull_down = models.BooleanField()
-    mrs_glow = models.BooleanField(db_column='MRS_Glow')  # Field name made lowercase.
-    mrs_zipper = models.BooleanField(db_column='MRS_Zipper')  # Field name made lowercase.
+    mrs_glow = models.BooleanField(db_column="MRS_Glow")  # Field name made lowercase.
+    mrs_zipper = models.BooleanField(
+        db_column="MRS_Zipper"
+    )  # Field name made lowercase.
     row_pull_up = models.BooleanField()
-    lrs_contamination = models.BooleanField(db_column='LRS_Contamination')  # Field name made lowercase.
+    lrs_contamination = models.BooleanField(
+        db_column="LRS_Contamination"
+    )  # Field name made lowercase.
     tree_rings = models.BooleanField()
 
     class Meta:
         managed = True
-        db_table = 'miri_anomaly'
+        db_table = "miri_anomaly"
 
 
 class NircamAnomaly(models.Model):
@@ -307,7 +329,7 @@ class NircamAnomaly(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'nircam_anomaly'
+        db_table = "nircam_anomaly"
 
 
 class NirissAnomaly(models.Model):
@@ -329,7 +351,7 @@ class NirissAnomaly(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'niriss_anomaly'
+        db_table = "niriss_anomaly"
 
 
 class NirspecAnomaly(models.Model):
@@ -345,10 +367,12 @@ class NirspecAnomaly(models.Model):
     data_transfer_error = models.BooleanField()
     ghost = models.BooleanField()
     snowball = models.BooleanField()
-    dominant_msa_leakage = models.BooleanField(db_column='Dominant_MSA_Leakage')  # Field name made lowercase.
+    dominant_msa_leakage = models.BooleanField(
+        db_column="Dominant_MSA_Leakage"
+    )  # Field name made lowercase.
     optical_short = models.BooleanField()
     other = models.BooleanField()
 
     class Meta:
         managed = True
-        db_table = 'nirspec_anomaly'
+        db_table = "nirspec_anomaly"

pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "celery>=5.3.6,<6",
     "crds>=11.17.19,<12",
     "django>=5.0.3,<6",
+    "django-enum>=2.0.2,<3",
     "gunicorn>=22.0.0,<23.0.0",
     "inflection>=0.5.1,<0.6",
     "jsonschema>=4.21.1,<5",