From f8bffd86c708f58934892d75b4da4ee73e573a61 Mon Sep 17 00:00:00 2001 From: cracraft Date: Fri, 22 Jun 2018 13:36:43 -0400 Subject: [PATCH 1/4] Updated code to include stats on files by instrument and add sphinx documentation --- docs/source/index.rst | 1 + docs/source/monitor_filesystem.rst | 7 +++ jwql/monitor_filesystem/monitor_filesystem.py | 58 +++++++++++++++---- 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100755 docs/source/monitor_filesystem.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 000931490..eb66a6b87 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -24,6 +24,7 @@ API documentation preview_image.rst utils.rst tests.rst + monitor_filesystem.rst Indices and tables diff --git a/docs/source/monitor_filesystem.rst b/docs/source/monitor_filesystem.rst new file mode 100755 index 000000000..50c6a3df6 --- /dev/null +++ b/docs/source/monitor_filesystem.rst @@ -0,0 +1,7 @@ +*********** +monitor_filesystem +*********** + +.. automodule:: jwql.monitor_filesystem.monitor_filesystem + :members: + :undoc-members: \ No newline at end of file diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index a12a76c8a..787da07d1 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -94,6 +94,11 @@ def filesystem_monitor(): suffix = filename_parser(filename)['suffix'] results_dict[suffix] += 1 size_dict[suffix] += os.path.getsize(file_path) + detector = filename_parser(filename)['detector'] + instrument = detector[0:3] # first three characters of detector specify instrument + results_dict[instrument] += 1 + size_dict[instrument] += os.path.getsize(file_path) + # Get df style stats on file system out = subprocess.check_output('df {}'.format(filesystem), shell=True) @@ -119,17 +124,19 @@ def filesystem_monitor(): # set up and read out stats on files by type filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') with open(filesbytype, "a+") as f2: - f2.write("{0} {1} {2} {3} {4} {5}\n".format(results_dict['fits_files'], + f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'], results_dict['uncal'], results_dict['cal'], results_dict['rate'], - results_dict['rateints'], results_dict['i2d'])) + results_dict['rateints'], results_dict['i2d'], results_dict['nrc'], + results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui'])) set_permissions(filesbytype, verbose=False) # set up file size by type file sizebytype = os.path.join(outputs_dir, 'sizebytype.txt') with open(sizebytype, "a+") as f3: - f3.write("{0} {1} {2} {3} {4} {5}\n".format(size_dict['size_fits'], + f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'], size_dict['uncal'], size_dict['cal'], size_dict['rate'], - size_dict['rateints'], size_dict['i2d'])) + size_dict['rateints'], size_dict['i2d'], size_dict['nrc'], + size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui'])) set_permissions(sizebytype, verbose=False) @@ -153,8 +160,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): # read in file of statistics date, f_count, sysize, frsize, used, percent = np.loadtxt(os.path.join(outputs_dir, stats_file), dtype=str, unpack=True) - fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True) - fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True) + fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True) + fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True) # put in proper np array types and convert to GB sizes dates = np.array(date, dtype='datetime64') @@ -169,6 +176,11 @@ def plot_system_stats(stats_file, filebytype, sizebytype): rate = ratefiles.astype(int) rateints = rateintsfiles.astype(int) i2d = i2dfiles.astype(int) + nircam = nrcfiles.astype(int) + nirspec = nrsfiles.astype(int) + niriss = nisfiles.astype(int) + miri = mirfiles.astype(int) + fgs = fgsfiles.astype(int) fits_size = fits_sz.astype(float) / (1024.**3) uncal_size = uncal_sz.astype(float) / (1024.**3) @@ -176,18 +188,23 @@ def plot_system_stats(stats_file, filebytype, sizebytype): rate_size = rate_sz.astype(float) / (1024.**3) rateints_size = rateints_sz.astype(float) / (1024.**3) i2d_size = i2d_sz.astype(float) / (1024.**3) + nircam_size = nrc_sz.astype(float) / (1024.**3) + nirspec_size = nrs_sz.astype(float) / (1024.**3) + niriss_size = nis_sz.astype(float) / (1024.**3) + miri_size = mir_sz.astype(float) / (1024.**3) + fgs_size = fgs_sz.astype(float) / (1024.**3) # plot the data # Plot filecount vs. date p1 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_type='datetime', title="Total File Counts", x_axis_label='Date', y_axis_label='Count') p1.line(dates, file_count, line_width=2, line_color='blue') p1.circle(dates, file_count, color='blue') # Plot system stats vs. date p2 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title='System stats', x_axis_label='Date', y_axis_label='GB') p2.line(dates, systemsize, legend='Total size', line_color='red') p2.circle(dates, systemsize, color='red') @@ -198,7 +215,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype): # Plot fits files by type vs. date p3 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title="Total File Counts by Type", x_axis_label='Date', y_axis_label='Count') p3.line(dates, fits, legend='Total fits files', line_color='black') p3.circle(dates, fits, color='black') @@ -212,10 +229,22 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p3.asterisk(dates, rateints, color='orange') p3.line(dates, i2d, legend='i2d fits files', line_color='purple') p3.x(dates, i2d, color='purple') + p3.line(dates, nircam, legend='nircam fits files', line_color='midnightblue') + p3.x(dates, nircam, color='midnightblue') + p3.line(dates, nirspec, legend='nirspec fits files', line_color='springgreen') + p3.x(dates, nirspec, color='springgreen') + p3.line(dates, niriss, legend='niriss fits files', line_color='darkcyan') + p3.x(dates, niriss, color='darkcyan') + p3.line(dates, miri, legend='miri fits files', line_color='dodgerblue') + p3.x(dates, miri, color='dodgerblue') + p3.line(dates, fgs, legend='fgs fits files', line_color='darkred') + p3.x(dates, fgs, color='darkred') + + # plot size of total fits files by type p4 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title="Total File Sizes by Type", x_axis_label='Date', y_axis_label='GB') p4.line(dates, fits_size, legend='Total fits files', line_color='black') p4.circle(dates, fits_size, color='black') @@ -229,6 +258,15 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p4.asterisk(dates, rateints_size, color='orange') p4.line(dates, i2d_size, legend='i2d fits files', line_color='purple') p4.x(dates, i2d_size, color='purple') + p3.x(dates, nircam, color='midnightblue') + p4.line(dates, nirspec_size, legend='nirspec fits files', line_color='springgreen') + p4.x(dates, nirspec_size, color='springgreen') + p4.line(dates, niriss_size, legend='niriss fits files', line_color='darkcyan') + p4.x(dates, niriss_size, color='darkcyan') + p4.line(dates, miri_size, legend='miri fits files', line_color='dodgerblue') + p4.x(dates, miri_size, color='dodgerblue') + p4.line(dates, fgs_size, legend='fgs fits files', line_color='darkred') + p4.x(dates, fgs_size, color='darkred') # create a layout with a grid pattern grid = gridplot([[p1, p2], [p3, p4]]) From 82fd1a1d29be6ff613d7311d93f21fb548e87580 Mon Sep 17 00:00:00 2001 From: cracraft Date: Mon, 23 Jul 2018 09:30:43 -0400 Subject: [PATCH 2/4] Added NIRCam to plot 4 (size by instrument). --- jwql/monitor_filesystem/monitor_filesystem.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index 787da07d1..0a814fa1f 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -99,7 +99,6 @@ def filesystem_monitor(): results_dict[instrument] += 1 size_dict[instrument] += os.path.getsize(file_path) - # Get df style stats on file system out = subprocess.check_output('df {}'.format(filesystem), shell=True) outstring = out.decode("utf-8") # put into string for parsing from byte format @@ -240,8 +239,6 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p3.line(dates, fgs, legend='fgs fits files', line_color='darkred') p3.x(dates, fgs, color='darkred') - - # plot size of total fits files by type p4 = figure( tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', @@ -258,7 +255,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p4.asterisk(dates, rateints_size, color='orange') p4.line(dates, i2d_size, legend='i2d fits files', line_color='purple') p4.x(dates, i2d_size, color='purple') - p3.x(dates, nircam, color='midnightblue') + p4.line(dates, nircam_size, legend='nircam fits files', line_color='midnightblue') + p4.x(dates, nircam_size, color='midnightblue') p4.line(dates, nirspec_size, legend='nirspec fits files', line_color='springgreen') p4.x(dates, nirspec_size, color='springgreen') p4.line(dates, niriss_size, legend='niriss fits files', line_color='darkcyan') From 498ca0978b50a7953e6997b0e2e0fe06af0ac200 Mon Sep 17 00:00:00 2001 From: cracraft Date: Fri, 22 Jun 2018 13:36:43 -0400 Subject: [PATCH 3/4] Updated code to include stats on files by instrument and add sphinx documentation --- docs/source/index.rst | 3 +- docs/source/monitor_filesystem.rst | 7 +++ jwql/monitor_filesystem/monitor_filesystem.py | 62 +++++++++++++++---- 3 files changed, 59 insertions(+), 13 deletions(-) create mode 100755 docs/source/monitor_filesystem.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 2ca24a646..96ca99ff1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,12 +19,13 @@ API documentation :maxdepth: 1 :caption: Contents: + logging.rst + monitor_filesystem.rst monitor_mast.rst permissions.rst preview_image.rst utils.rst tests.rst - logging.rst Indices and tables diff --git a/docs/source/monitor_filesystem.rst b/docs/source/monitor_filesystem.rst new file mode 100755 index 000000000..50c6a3df6 --- /dev/null +++ b/docs/source/monitor_filesystem.rst @@ -0,0 +1,7 @@ +*********** +monitor_filesystem +*********** + +.. automodule:: jwql.monitor_filesystem.monitor_filesystem + :members: + :undoc-members: \ No newline at end of file diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index e61054d27..8d69016fd 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -94,6 +94,11 @@ def filesystem_monitor(): suffix = filename_parser(filename)['suffix'] results_dict[suffix] += 1 size_dict[suffix] += os.path.getsize(file_path) + detector = filename_parser(filename)['detector'] + instrument = detector[0:3] # first three characters of detector specify instrument + results_dict[instrument] += 1 + size_dict[instrument] += os.path.getsize(file_path) + # Get df style stats on file system out = subprocess.check_output('df {}'.format(filesystem), shell=True) @@ -119,18 +124,20 @@ def filesystem_monitor(): # set up and read out stats on files by type filesbytype = os.path.join(outputs_dir, 'filesbytype.txt') with open(filesbytype, "a+") as f2: - f2.write("{0} {1} {2} {3} {4} {5}\n".format(results_dict['fits_files'], + f2.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(results_dict['fits_files'], results_dict['uncal'], results_dict['cal'], results_dict['rate'], - results_dict['rateints'], results_dict['i2d'])) - set_permissions(filesbytype) + results_dict['rateints'], results_dict['i2d'], results_dict['nrc'], + results_dict['nrs'], results_dict['nis'], results_dict['mir'], results_dict['gui'])) + set_permissions(filesbytype, verbose=False) # set up file size by type file sizebytype = os.path.join(outputs_dir, 'sizebytype.txt') with open(sizebytype, "a+") as f3: - f3.write("{0} {1} {2} {3} {4} {5}\n".format(size_dict['size_fits'], + f3.write("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}\n".format(size_dict['size_fits'], size_dict['uncal'], size_dict['cal'], size_dict['rate'], - size_dict['rateints'], size_dict['i2d'])) - set_permissions(sizebytype) + size_dict['rateints'], size_dict['i2d'], size_dict['nrc'], + size_dict['nrs'], size_dict['nis'], size_dict['mir'], size_dict['gui'])) + set_permissions(sizebytype, verbose=False) def plot_system_stats(stats_file, filebytype, sizebytype): @@ -153,8 +160,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): # read in file of statistics date, f_count, sysize, frsize, used, percent = np.loadtxt(os.path.join(outputs_dir, stats_file), dtype=str, unpack=True) - fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True) - fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True) + fits_files, uncalfiles, calfiles, ratefiles, rateintsfiles, i2dfiles, nrcfiles, nrsfiles, nisfiles, mirfiles, fgsfiles = np.loadtxt(os.path.join(outputs_dir, filebytype), dtype=str, unpack=True) + fits_sz, uncal_sz, cal_sz, rate_sz, rateints_sz, i2d_sz, nrc_sz, nrs_sz, nis_sz, mir_sz, fgs_sz = np.loadtxt(os.path.join(outputs_dir, sizebytype), dtype=str, unpack=True) # put in proper np array types and convert to GB sizes dates = np.array(date, dtype='datetime64') @@ -169,6 +176,11 @@ def plot_system_stats(stats_file, filebytype, sizebytype): rate = ratefiles.astype(int) rateints = rateintsfiles.astype(int) i2d = i2dfiles.astype(int) + nircam = nrcfiles.astype(int) + nirspec = nrsfiles.astype(int) + niriss = nisfiles.astype(int) + miri = mirfiles.astype(int) + fgs = fgsfiles.astype(int) fits_size = fits_sz.astype(float) / (1024.**3) uncal_size = uncal_sz.astype(float) / (1024.**3) @@ -176,18 +188,23 @@ def plot_system_stats(stats_file, filebytype, sizebytype): rate_size = rate_sz.astype(float) / (1024.**3) rateints_size = rateints_sz.astype(float) / (1024.**3) i2d_size = i2d_sz.astype(float) / (1024.**3) + nircam_size = nrc_sz.astype(float) / (1024.**3) + nirspec_size = nrs_sz.astype(float) / (1024.**3) + niriss_size = nis_sz.astype(float) / (1024.**3) + miri_size = mir_sz.astype(float) / (1024.**3) + fgs_size = fgs_sz.astype(float) / (1024.**3) # plot the data # Plot filecount vs. date p1 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_type='datetime', title="Total File Counts", x_axis_label='Date', y_axis_label='Count') p1.line(dates, file_count, line_width=2, line_color='blue') p1.circle(dates, file_count, color='blue') # Plot system stats vs. date p2 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title='System stats', x_axis_label='Date', y_axis_label='GB') p2.line(dates, systemsize, legend='Total size', line_color='red') p2.circle(dates, systemsize, color='red') @@ -198,7 +215,7 @@ def plot_system_stats(stats_file, filebytype, sizebytype): # Plot fits files by type vs. date p3 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title="Total File Counts by Type", x_axis_label='Date', y_axis_label='Count') p3.line(dates, fits, legend='Total fits files', line_color='black') p3.circle(dates, fits, color='black') @@ -212,10 +229,22 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p3.asterisk(dates, rateints, color='orange') p3.line(dates, i2d, legend='i2d fits files', line_color='purple') p3.x(dates, i2d, color='purple') + p3.line(dates, nircam, legend='nircam fits files', line_color='midnightblue') + p3.x(dates, nircam, color='midnightblue') + p3.line(dates, nirspec, legend='nirspec fits files', line_color='springgreen') + p3.x(dates, nirspec, color='springgreen') + p3.line(dates, niriss, legend='niriss fits files', line_color='darkcyan') + p3.x(dates, niriss, color='darkcyan') + p3.line(dates, miri, legend='miri fits files', line_color='dodgerblue') + p3.x(dates, miri, color='dodgerblue') + p3.line(dates, fgs, legend='fgs fits files', line_color='darkred') + p3.x(dates, fgs, color='darkred') + + # plot size of total fits files by type p4 = figure( - tools='pan,box_zoom,reset,save', x_axis_type='datetime', + tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title="Total File Sizes by Type", x_axis_label='Date', y_axis_label='GB') p4.line(dates, fits_size, legend='Total fits files', line_color='black') p4.circle(dates, fits_size, color='black') @@ -229,6 +258,15 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p4.asterisk(dates, rateints_size, color='orange') p4.line(dates, i2d_size, legend='i2d fits files', line_color='purple') p4.x(dates, i2d_size, color='purple') + p3.x(dates, nircam, color='midnightblue') + p4.line(dates, nirspec_size, legend='nirspec fits files', line_color='springgreen') + p4.x(dates, nirspec_size, color='springgreen') + p4.line(dates, niriss_size, legend='niriss fits files', line_color='darkcyan') + p4.x(dates, niriss_size, color='darkcyan') + p4.line(dates, miri_size, legend='miri fits files', line_color='dodgerblue') + p4.x(dates, miri_size, color='dodgerblue') + p4.line(dates, fgs_size, legend='fgs fits files', line_color='darkred') + p4.x(dates, fgs_size, color='darkred') # create a layout with a grid pattern grid = gridplot([[p1, p2], [p3, p4]]) From ebfbd627a632b3788f6bfa8e6cb199de2eb56b7d Mon Sep 17 00:00:00 2001 From: cracraft Date: Mon, 23 Jul 2018 09:30:43 -0400 Subject: [PATCH 4/4] Added NIRCam to plot 4 (size by instrument). --- jwql/monitor_filesystem/monitor_filesystem.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/monitor_filesystem/monitor_filesystem.py b/jwql/monitor_filesystem/monitor_filesystem.py index 8d69016fd..d8131cfbc 100755 --- a/jwql/monitor_filesystem/monitor_filesystem.py +++ b/jwql/monitor_filesystem/monitor_filesystem.py @@ -99,7 +99,6 @@ def filesystem_monitor(): results_dict[instrument] += 1 size_dict[instrument] += os.path.getsize(file_path) - # Get df style stats on file system out = subprocess.check_output('df {}'.format(filesystem), shell=True) outstring = out.decode("utf-8") # put into string for parsing from byte format @@ -240,8 +239,6 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p3.line(dates, fgs, legend='fgs fits files', line_color='darkred') p3.x(dates, fgs, color='darkred') - - # plot size of total fits files by type p4 = figure( tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', @@ -258,7 +255,8 @@ def plot_system_stats(stats_file, filebytype, sizebytype): p4.asterisk(dates, rateints_size, color='orange') p4.line(dates, i2d_size, legend='i2d fits files', line_color='purple') p4.x(dates, i2d_size, color='purple') - p3.x(dates, nircam, color='midnightblue') + p4.line(dates, nircam_size, legend='nircam fits files', line_color='midnightblue') + p4.x(dates, nircam_size, color='midnightblue') p4.line(dates, nirspec_size, legend='nirspec fits files', line_color='springgreen') p4.x(dates, nirspec_size, color='springgreen') p4.line(dates, niriss_size, legend='niriss fits files', line_color='darkcyan')