Merge pull request #98 from cracraft/filesystem-monitor

WIP: Stats on files by instrument in monitor_filesystem

docs/source/index.rst

@@ -19,12 +19,14 @@ API documentation
    :maxdepth: 1
    :caption: Contents:
 
+   logging.rst
+   monitor_filesystem.rst
    monitor_mast.rst
    permissions.rst
    preview_image.rst
    utils.rst
    tests.rst
-   logging.rst
+   monitor_filesystem.rst
 
 
 Indices and tables

docs/source/monitor_filesystem.rst

@@ -0,0 +1,7 @@
+***********
+monitor_filesystem
+***********
+
+.. automodule:: jwql.monitor_filesystem.monitor_filesystem
+    :members:
+    :undoc-members:

jwql/monitor_filesystem/monitor_filesystem.py

@@ -94,6 +94,10 @@ def filesystem_monitor():
                 suffix = filename_parser(filename)['suffix']
                 results_dict[suffix] += 1
                 size_dict[suffix] += os.path.getsize(file_path)
+                detector = filename_parser(filename)['detector']
+                instrument = detector[0:3]  # first three characters of detector specify instrument
+                results_dict[instrument] += 1
+                size_dict[instrument] += os.path.getsize(file_path)
 
     # Get df style stats on file system
     out = subprocess.check_output('df {}'.format(filesystem), shell=True)
@@ -119,18 +123,20 @@ def filesystem_monitor():
     # set up and read out stats on files by type
     filesbytype = os.path.join(outputs_dir, 'filesbytype.txt')
     with open(filesbytype, "a+") as f2:
-        f2.write("{0} {1} {2} {3} {4} {5}\n".format(results_dict['fits_files'],
+        f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'],
                  results_dict['uncal'], results_dict['cal'], results_dict['rate'],
-                 results_dict['rateints'], results_dict['i2d']))
-    set_permissions(filesbytype)
+                 results_dict['rateints'], results_dict['i2d'], results_dict['nrc'],
+                 results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui']))
+    set_permissions(filesbytype, verbose=False)
 
     # set up file size by type file
     sizebytype = os.path.join(outputs_dir, 'sizebytype.txt')
     with open(sizebytype, "a+") as f3:
-        f3.write("{0} {1} {2} {3} {4} {5}\n".format(size_dict['size_fits'],
+        f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'],
                  size_dict['uncal'], size_dict['cal'], size_dict['rate'],
-                 size_dict['rateints'], size_dict['i2d']))
-    set_permissions(sizebytype)
+                 size_dict['rateints'], size_dict['i2d'], size_dict['nrc'],
+                 size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui']))
+    set_permissions(sizebytype, verbose=False)
 
 
 def plot_system_stats(stats_file, filebytype, sizebytype):
@@ -153,8 +159,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype):
 
     # read in file of statistics
     date, f_count, sysize, frsize, used, percent = np.loadtxt(os.path.join(outputs_dir, stats_file), dtype=str, unpack=True)
-    fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True)
-    fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True)
+    fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True)
+    fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True)
 
     # put in proper np array types and convert to GB sizes
     dates = np.array(date, dtype='datetime64')
@@ -169,25 +175,35 @@ def plot_system_stats(stats_file, filebytype, sizebytype):
     rate = ratefiles.astype(int)
     rateints = rateintsfiles.astype(int)
     i2d = i2dfiles.astype(int)
+    nircam = nrcfiles.astype(int)
+    nirspec = nrsfiles.astype(int)
+    niriss = nisfiles.astype(int)
+    miri = mirfiles.astype(int)
+    fgs = fgsfiles.astype(int)
 
     fits_size = fits_sz.astype(float) / (1024.**3)
     uncal_size = uncal_sz.astype(float) / (1024.**3)
     cal_size = cal_sz.astype(float) / (1024.**3)
     rate_size = rate_sz.astype(float) / (1024.**3)
     rateints_size = rateints_sz.astype(float) / (1024.**3)
     i2d_size = i2d_sz.astype(float) / (1024.**3)
+    nircam_size = nrc_sz.astype(float) / (1024.**3)
+    nirspec_size = nrs_sz.astype(float) / (1024.**3)
+    niriss_size = nis_sz.astype(float) / (1024.**3)
+    miri_size = mir_sz.astype(float) / (1024.**3)
+    fgs_size = fgs_sz.astype(float) / (1024.**3)
 
     # plot the data
     # Plot filecount vs. date
     p1 = figure(
-       tools='pan,box_zoom,reset,save', x_axis_type='datetime',
+       tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_type='datetime',
        title="Total File Counts", x_axis_label='Date', y_axis_label='Count')
     p1.line(dates, file_count, line_width=2, line_color='blue')
     p1.circle(dates, file_count, color='blue')
 
     # Plot system stats vs. date
     p2 = figure(
-      tools='pan,box_zoom,reset,save', x_axis_type='datetime',
+      tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime',
       title='System stats', x_axis_label='Date', y_axis_label='GB')
     p2.line(dates, systemsize, legend='Total size', line_color='red')
     p2.circle(dates, systemsize, color='red')
@@ -198,7 +214,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype):
 
     # Plot fits files by type vs. date
     p3 = figure(
-       tools='pan,box_zoom,reset,save', x_axis_type='datetime',
+       tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime',
        title="Total File Counts by Type", x_axis_label='Date', y_axis_label='Count')
     p3.line(dates, fits, legend='Total fits files', line_color='black')
     p3.circle(dates, fits, color='black')
@@ -212,10 +228,20 @@ def plot_system_stats(stats_file, filebytype, sizebytype):
     p3.asterisk(dates, rateints, color='orange')
     p3.line(dates, i2d, legend='i2d fits files', line_color='purple')
     p3.x(dates, i2d, color='purple')
+    p3.line(dates, nircam, legend='nircam fits files', line_color='midnightblue')
+    p3.x(dates, nircam, color='midnightblue')
+    p3.line(dates, nirspec, legend='nirspec fits files', line_color='springgreen')
+    p3.x(dates, nirspec, color='springgreen')
+    p3.line(dates, niriss, legend='niriss fits files', line_color='darkcyan')
+    p3.x(dates, niriss, color='darkcyan')
+    p3.line(dates, miri, legend='miri fits files', line_color='dodgerblue')
+    p3.x(dates, miri, color='dodgerblue')
+    p3.line(dates, fgs, legend='fgs fits files', line_color='darkred')
+    p3.x(dates, fgs, color='darkred')
 
     # plot size of total fits files by type
     p4 = figure(
-       tools='pan,box_zoom,reset,save', x_axis_type='datetime',
+       tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime',
        title="Total File Sizes by Type", x_axis_label='Date', y_axis_label='GB')
     p4.line(dates, fits_size, legend='Total fits files', line_color='black')
     p4.circle(dates, fits_size, color='black')
@@ -229,6 +255,16 @@ def plot_system_stats(stats_file, filebytype, sizebytype):
     p4.asterisk(dates, rateints_size, color='orange')
     p4.line(dates, i2d_size, legend='i2d fits files', line_color='purple')
     p4.x(dates, i2d_size, color='purple')
+    p4.line(dates, nircam_size, legend='nircam fits files', line_color='midnightblue')
+    p4.x(dates, nircam_size, color='midnightblue')
+    p4.line(dates, nirspec_size, legend='nirspec fits files', line_color='springgreen')
+    p4.x(dates, nirspec_size, color='springgreen')
+    p4.line(dates, niriss_size, legend='niriss fits files', line_color='darkcyan')
+    p4.x(dates, niriss_size, color='darkcyan')
+    p4.line(dates, miri_size, legend='miri fits files', line_color='dodgerblue')
+    p4.x(dates, miri_size, color='dodgerblue')
+    p4.line(dates, fgs_size, legend='fgs fits files', line_color='darkred')
+    p4.x(dates, fgs_size, color='darkred')
 
     # create a layout with a grid pattern
     grid = gridplot([[p1, p2], [p3, p4]])