From fd37bae1fbb7241186f2c000cb23f6016c78ae05 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:26:04 -0500 Subject: [PATCH 01/42] Update tacc_stats.ini --- tacc_stats.ini | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tacc_stats.ini b/tacc_stats.ini index 17da9ed..75964bb 100644 --- a/tacc_stats.ini +++ b/tacc_stats.ini @@ -1,22 +1,27 @@ -## Basic configuration options - modify these -# machine = unique name of machine/queue -# server = database and rmq server hostname -# data_dir = where data is stored [DEFAULT] -machine = stampede2 +machine = tacc-stats-sanga data_dir = /tacc_stats_site/%(machine)s -server = tacc-stats02.tacc.utexas.edu +server = tacc-stats04.tacc.utexas.edu -## RabbitMQ Configuration -# RMQ_SERVER = RMQ server -# RMQ_QUEUE = RMQ server [RMQ] rmq_server = %(server)s rmq_queue = %(machine)s -## Configuration for Web Portal Support [PORTAL] acct_path = %(data_dir)s/accounting archive_dir = %(data_dir)s/archive host_name_ext = %(machine)s.tacc.utexas.edu -dbname = %(machine)s_db +dbname = test_db +daily_archive_dir = %(data_dir)s/daily_archive +engine_name = django.db.backends.postgresql_psycopg2 +username = taccstats +password = taccstats +host = localhost +port = 5432 + +[XALT] +xalt_engine = none +xalt_name = none +xalt_user = none +xalt_password = none +xalt_host = none From 81ec55e285b603a616813adc4d82ab327f752b8f Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:29:03 -0500 Subject: [PATCH 02/42] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26610d9..73ad787 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ include_package_data = True, scripts = scripts, install_requires = ['argparse','numpy', 'psycopg2-binary', 'pandas', 'pgcopy', - 'bokeh', 'django', 'python-hostlist', 'PyMySQL', 'mod_wsgi', + 'bokeh', 'django==3.2.25', 'python-hostlist', 'PyMySQL', 'mod_wsgi', 'mysql-connector-python', 'python-memcached', 'pika', 'mysqlclient'], platforms = 'any', classifiers = [ From b15338f4e1327a43e359142eaef74bab8ddcee88 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:37:47 -0500 Subject: [PATCH 03/42] Create conf_parser.py --- conf_parser.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 conf_parser.py diff --git a/conf_parser.py b/conf_parser.py new file mode 100644 index 0000000..45731a5 --- /dev/null +++ b/conf_parser.py @@ -0,0 +1,94 @@ +import configparser +import os +import sys +import time + +sys.path.append("/home/sg99/tacc_stats") + +cfg = configparser.ConfigParser() +cfg.read('/home/sg99/tacc_stats/tacc_stats.ini') + +def get_db_connection_string(): + temp_string = "dbname={0} "+cfg.get('PORTAL', 'username')+" "+cfg.get('PORTAL', 'password')+" "+cfg.get('PORTAL', 'port') + connection_string = temp_string.format(cfg.get('PORTAL', 'dbname')) + return connection_string + +def get_db_name(): + db_name = cfg.get('PORTAL', 'dbname') + return db_name + +def get_archive_dir_path(): + archive_dir_path = cfg.get('PORTAL', 'archive_dir') + return archive_dir_path + +def get_host_name_ext(): + host_name_ext = cfg.get('PORTAL', 'host_name_ext') + return host_name_ext + +def get_accounting_path(): + accounting_path = cfg.get('PORTAL', 'acct_path') + return accounting_path + +def get_daily_archive_dir_path(): + daily_archive_dir_path = cfg.get('PORTAL', 'daily_archive_dir') + return daily_archive_dir_path + +def get_rmq_server(): + rmq_server = cfg.get('RMQ', 'rmq_server') + return rmq_server + +def get_rmq_queue(): + rmq_queue = cfg.get('RMQ', 'rmq_queue') + return rmq_queue + +def get_machine_name(): + machine_name = cfg.get('DEFAULT', 'machine') + return machine_name + +def get_server_name(): + server_name = cfg.get('DEFAULT', 'server') + return server_name + +def get_data_dir_path(): + data_dir_path = cfg.get('DEFAULT', 'data_dir') + return data_dir_path + +def get_engine_name(): + engine_name = cfg.get('PORTAL', 'engine_name') + return engine_name + +def get_username(): + username = cfg.get('PORTAL', 'username') + return username + +def get_password(): + password = cfg.get('PORTAL', 'password') + return password + +def get_host(): + host = cfg.get('PORTAL', 'host') + return host + +def get_port(): + port = cfg.get('PORTAL', 'port') + return port + +def get_xalt_engine(): + xalt_engine = cfg.get('XALT', 'xalt_engine') + return xalt_engine + +def get_xalt_name(): + xalt_name = cfg.get('XALT', 'xalt_name') + return xalt_name + +def get_xalt_user(): + xalt_user = cfg.get('XALT', 'xalt_user') + return xalt_user + +def get_xalt_password(): + xalt_password = cfg.get('XALT', 'xalt_password') + return xalt_password + +def get_xalt_host(): + xalt_host = cfg.get('XALT', 'xalt_host') + return xalt_host From 91a68c91e8923f357f3c5d8a14fa46304dea5fb3 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:46:21 -0500 Subject: [PATCH 04/42] Update listend.py --- tacc_stats/listend.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tacc_stats/listend.py b/tacc_stats/listend.py index fcf4979..7df77bb 100755 --- a/tacc_stats/listend.py +++ b/tacc_stats/listend.py @@ -2,9 +2,13 @@ import pika import os, sys import time -import tacc_stats.cfg as cfg + from fcntl import flock, LOCK_EX, LOCK_NB +sys.path.append("/home/sg99/tacc_stats") + +import conf_parser as cfg + def on_message(channel, method_frame, header_frame, body): try: @@ -20,7 +24,7 @@ def on_message(channel, method_frame, header_frame, body): host = message.split()[2] #if host == "localhost.localdomain": return - host_dir = os.path.join(cfg.archive_dir, host) + host_dir = os.path.join(cfg.get_archive_dir_path(), host) if not os.path.exists(host_dir): os.makedirs(host_dir) @@ -47,10 +51,10 @@ def on_message(channel, method_frame, header_frame, body): print("listend is already running") sys.exit() - parameters = pika.ConnectionParameters(cfg.rmq_server) + parameters = pika.ConnectionParameters(cfg.get_rmq_server()) connection = pika.BlockingConnection(parameters) channel = connection.channel() - channel.basic_consume(cfg.rmq_queue, on_message) + channel.basic_consume(cfg.get_rmq_queue(), on_message) try: channel.start_consuming() except KeyboardInterrupt: From 354d91391f2c5e1a365d6bf4cd898a4a9fedcbe0 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:53:27 -0500 Subject: [PATCH 05/42] Update sync_acct.py --- tacc_stats/dbload/sync_acct.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tacc_stats/dbload/sync_acct.py b/tacc_stats/dbload/sync_acct.py index 3442ede..68fd334 100755 --- a/tacc_stats/dbload/sync_acct.py +++ b/tacc_stats/dbload/sync_acct.py @@ -1,5 +1,8 @@ + #!/usr/bin/env python3 import os,sys,time +sys.path.append("/home/sg99/tacc_stats") + from datetime import timedelta, datetime import psycopg2 @@ -11,9 +14,10 @@ import hostlist from tacc_stats.analysis.gen.utils import read_sql -from tacc_stats import cfg -CONNECTION = "dbname={0} user=postgres port=5432".format(cfg.dbname) +import conf_parser as cfg + +CONNECTION = cfg.get_db_connection_string() query_create_jobdata_table = """CREATE TABLE IF NOT EXISTS job_data ( jid VARCHAR(32) NOT NULL, @@ -113,7 +117,7 @@ def sync_acct(acct_file, date_str): # Parse and convert raw stats files to pandas dataframe start = time.time() - directory = cfg.acct_path + directory = cfg.get_accounting_path() while startdate <= enddate: for entry in os.scandir(directory): From 710cd3ae5ff69bba5234a55224359b7e9ca9ab8f Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:53:40 -0500 Subject: [PATCH 06/42] Update sync_acct.py --- tacc_stats/dbload/sync_acct.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tacc_stats/dbload/sync_acct.py b/tacc_stats/dbload/sync_acct.py index 68fd334..cac7bc5 100755 --- a/tacc_stats/dbload/sync_acct.py +++ b/tacc_stats/dbload/sync_acct.py @@ -1,4 +1,3 @@ - #!/usr/bin/env python3 import os,sys,time sys.path.append("/home/sg99/tacc_stats") From 1354cabb7cfdfaaac4fef57ed21c4e6b6ba25cdf Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:55:08 -0500 Subject: [PATCH 07/42] Update sacct_gen.py --- tacc_stats/dbload/sacct_gen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tacc_stats/dbload/sacct_gen.py b/tacc_stats/dbload/sacct_gen.py index 2f2b1df..897b9ec 100755 --- a/tacc_stats/dbload/sacct_gen.py +++ b/tacc_stats/dbload/sacct_gen.py @@ -1,9 +1,10 @@ import os, sys +sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, date, datetime from dateutil.parser import parse -from tacc_stats import cfg +import conf_parser as cfg -acct_path = cfg.acct_path +acct_path = cfg.get_accounting_path() def daterange(start_date, end_date): for n in range(int ((end_date - start_date).days)): @@ -24,4 +25,3 @@ def daterange(start_date, end_date): sacct_command = "/bin/sacct -a -s CA,CD,F,NF,TO -P -X -S " + single_date.strftime("%Y-%m-%d") + " -E " + (single_date + timedelta(1)).strftime("%Y-%m-%d") +" -o JobID,User,Account,Start,End,Submit,Partition,TimeLimit,JobName,State,NNodes,ReqCPUS,NodeList > " + file_name print(sacct_command) os.system(sacct_command) - From 773f34346d94e1a34639bd6b8b6060052b466bc7 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:56:55 -0500 Subject: [PATCH 08/42] Update sync_timedb.py --- tacc_stats/dbload/sync_timedb.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tacc_stats/dbload/sync_timedb.py b/tacc_stats/dbload/sync_timedb.py index f3568f1..6a3d1f5 100755 --- a/tacc_stats/dbload/sync_timedb.py +++ b/tacc_stats/dbload/sync_timedb.py @@ -2,6 +2,7 @@ import psycopg2 from pgcopy import CopyManager import os, sys, stat +sys.path.append("/home/sg99/tacc_stats") import multiprocessing import itertools from multiprocessing import Pool, get_context, Lock, set_start_method @@ -17,7 +18,8 @@ from pandas import DataFrame, to_datetime, Timedelta, Timestamp, concat from tacc_stats.analysis.gen.utils import read_sql -from tacc_stats import cfg + +import conf_parser as cfg # archive toggle should_archive = True @@ -28,11 +30,10 @@ # Thread count for database loading and archival thread_count = 8 -tgz_archive_dir = "/tacc_stats_site/ls6/tgz_archives/" - +tgz_archive_dir = cfg.get_daily_archive_dir_path() -CONNECTION = "dbname={0} user=postgres port=5432".format(cfg.dbname) +CONNECTION = cfg.get_db_connection_string() amd64_pmc_eventmap = { 0x43ff03 : "FLOPS,W=48", 0x4300c2 : "BRANCH_INST_RETIRED,W=48", 0x4300c3: "BRANCH_INST_RETIRED_MISS,W=48", 0x4308af : "DISPATCH_STALL_CYCLES1,W=48", 0x43ffae :"DISPATCH_STALL_CYCLES0,W=48" } @@ -373,6 +374,7 @@ def database_startup(): host VARCHAR(64), jid VARCHAR(32), type VARCHAR(32), + dev VARCHAR(64), event VARCHAR(64), unit VARCHAR(16), value real, @@ -415,9 +417,9 @@ def database_startup(): #cur.execute(query_create_hostdata_hypertable) #cur.execute(query_create_compression) - # cur.execute(query_create_process_table) - # cur.execute(query_create_process_index) - cur.execute("SELECT pg_size_pretty(pg_database_size('{0}'));".format(cfg.dbname)) + #cur.execute(query_create_process_table) + #cur.execute(query_create_process_index) + cur.execute("SELECT pg_size_pretty(pg_database_size('{0}'));".format(cfg.get_db_name())) for x in cur.fetchall(): print("Database Size:", x[0]) if debug: @@ -464,7 +466,7 @@ def database_startup(): # Parse and convert raw stats files to pandas dataframe start = time.time() - directory = cfg.archive_dir + directory = cfg.get_archive_dir_path() stats_files = [] ar_file_mapping = {} From e7be8dc914d910dadf46a6fa7613995e436dfb2b Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 15:58:50 -0500 Subject: [PATCH 09/42] Update settings.py --- tacc_stats/site/tacc_stats_site/settings.py | 27 ++++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tacc_stats/site/tacc_stats_site/settings.py b/tacc_stats/site/tacc_stats_site/settings.py index b45c705..3393383 100644 --- a/tacc_stats/site/tacc_stats_site/settings.py +++ b/tacc_stats/site/tacc_stats_site/settings.py @@ -1,6 +1,9 @@ # Django settings for tacc_stats_site project. import os -import tacc_stats.cfg as cfg +import sys +sys.path.append("/home/sg99/tacc_stats") +import conf_parser as cfg + import tacc_stats.site.tacc_stats_site as tacc_stats_site DIR = os.path.dirname(os.path.abspath(__file__)) @@ -20,21 +23,21 @@ # Give a name that is unique for the computing platform DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME' : '{0}'.format(cfg.dbname), - 'USER' : 'postgres', - 'PASSWORD': 'taccstats', - 'HOST': 'localhost', - 'PORT': '5432', + 'ENGINE': cfg.get_engine_name(), + 'NAME' : '{0}'.format(cfg.get_db_name()), + 'USER' : cfg.get_username(), + 'PASSWORD': cfg.get_password(), + 'HOST': cfg.get_host(), + 'PORT': cfg.get_port(), }, # Uncomment this portion if an xalt database exists 'xalt' : { #'ENGINE' : 'mysql.connector.django', - 'ENGINE' : 'django.db.backends.mysql', - 'NAME' : 'xalt_' + cfg.machine, - 'USER' : 'xaltUser', - 'PASSWORD' : 'kutwgbh', - 'HOST' : 'xalt' + 'ENGINE' : cfg.get_xalt_engine() + 'NAME' : cfg.get_xalt_name(), + 'USER' : cfg.get_xalt_user(), + 'PASSWORD' : cfg.get_xalt_password(), + 'HOST' : cfg.get_xalt_host() } } From c615d4878d79eb1ec9b1b12c6c017105d69d7edb Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:00:24 -0500 Subject: [PATCH 10/42] Update update_xalt.py --- tacc_stats/site/machine/update_xalt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tacc_stats/site/machine/update_xalt.py b/tacc_stats/site/machine/update_xalt.py index f63104d..3a2f89c 100755 --- a/tacc_stats/site/machine/update_xalt.py +++ b/tacc_stats/site/machine/update_xalt.py @@ -1,11 +1,12 @@ #!/usr/bin/env python import os,sys +sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta,datetime os.environ['DJANGO_SETTINGS_MODULE']='tacc_stats.site.tacc_stats_site.settings' import django django.setup() from tacc_stats.site.machine import views -import tacc_stats.cfg as cfg +import conf_parser as cfg from tacc_stats.site.machine.models import Job, Libraries from tacc_stats.site.xalt.models import run, join_run_object, lib @@ -50,4 +51,3 @@ def daterange(start_date, end_date): library = Libraries(object_path = object_path, module_name = module_name) library.save() library.jobs.add(obj) - From 62f62f31e7f781427e808a03d2a51042ded48703 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:02:06 -0500 Subject: [PATCH 11/42] Update update_db.py --- tacc_stats/site/machine/update_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tacc_stats/site/machine/update_db.py b/tacc_stats/site/machine/update_db.py index 23b4f06..0dab298 100755 --- a/tacc_stats/site/machine/update_db.py +++ b/tacc_stats/site/machine/update_db.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import os,sys, pwd +sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, datetime from dateutil.parser import parse from fcntl import flock, LOCK_EX, LOCK_NB @@ -9,7 +10,7 @@ from tacc_stats.site.machine.models import Job, Host, Libraries from tacc_stats.site.xalt.models import run, join_run_object, lib from tacc_stats.analysis.metrics import metrics -import tacc_stats.cfg as cfg +import conf_parser as cfg from tacc_stats.progress import progress from tacc_stats.daterange import daterange import pytz, calendar @@ -23,7 +24,7 @@ def update_acct(date, rerun = False): tz = pytz.timezone('US/Central') ctr = 0 - with open(os.path.join(cfg.acct_path, date.strftime("%Y-%m-%d") + '.txt'), encoding = "latin1") as fd: + with open(os.path.join(cfg.get_accounting_path(), date.strftime("%Y-%m-%d") + '.txt'), encoding = "latin1") as fd: nrecords = sum(1 for record in csv.DictReader(fd)) fd.seek(0) From b4e25e87b1a05348567d7c8ada6808a979eb53df Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:05:08 -0500 Subject: [PATCH 12/42] Update update_metrics.py --- tacc_stats/analysis/metrics/update_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tacc_stats/analysis/metrics/update_metrics.py b/tacc_stats/analysis/metrics/update_metrics.py index 624325b..b67ff11 100755 --- a/tacc_stats/analysis/metrics/update_metrics.py +++ b/tacc_stats/analysis/metrics/update_metrics.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import os,sys, pwd +sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, datetime import psycopg2 @@ -10,11 +11,11 @@ from tacc_stats.site.machine.models import job_data, metrics_data from tacc_stats.analysis.metrics import metrics -import tacc_stats.cfg as cfg +import conf_parser as cfg from tacc_stats.progress import progress +CONNECTION = cfg.get_db_connection_string() -CONNECTION = "dbname={0} user=postgres port=5432".format(cfg.dbname) query_create_metric_table = """CREATE TABLE IF NOT EXISTS metrics_data ( id SERIAL PRIMARY KEY, jid VARCHAR(32), @@ -79,4 +80,3 @@ def update_metrics(date, rerun = False): while date <= enddate: update_metrics(date, rerun = False) date += timedelta(days=1) - From 0a874425456de13e1a6c4210d1e6403ee33fe27f Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:06:34 -0500 Subject: [PATCH 13/42] Update views.py --- tacc_stats/site/machine/views.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tacc_stats/site/machine/views.py b/tacc_stats/site/machine/views.py index 4295ea0..104f74a 100644 --- a/tacc_stats/site/machine/views.py +++ b/tacc_stats/site/machine/views.py @@ -1,3 +1,5 @@ +import sys +sys.path.append("/home/sg99/tacc_stats") from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.views.generic import DetailView, ListView @@ -11,13 +13,13 @@ from cryptography.fernet import Fernet from django.http import HttpRequest -import os,sys,pwd +import os,pwd from tacc_stats.analysis.metrics import metrics from tacc_stats.analysis.gen import jid_table from tacc_stats.site.machine.models import job_data, metrics_data from tacc_stats.site.xalt.models import run, join_run_object, lib -import tacc_stats.cfg as cfg +import conf_parser as cfg import tacc_stats.analysis.plot as plots #xalt @@ -56,7 +58,7 @@ def __init__(self): self.libset=[] -CONNECTION = "dbname={0} host=localhost user=postgres port=5432".format(cfg.dbname) +CONNECTION = cfg.get_db_connection_string() def home(request, error = False): field = {} @@ -69,7 +71,7 @@ def home(request, error = False): month_dict.setdefault(y + '-' + m, []) month_dict[y + '-' + m].append((str(date), d)) - field["machine_name"] = cfg.host_name_ext + field["machine_name"] = cfg.get_host_name_ext() field['date_list'] = sorted(month_dict.items())[::-1] field['error'] = error @@ -299,7 +301,7 @@ def get_context_data(self, **kwargs): ### Specific to TACC Splunk urlstring="https://scribe.tacc.utexas.edu:8000/en-US/app/search/search?q=search%20" - hoststring=urlstring + "%20host%3D" + j.acct_host_list[0] + cfg.host_name_ext + hoststring=urlstring + "%20host%3D" + j.acct_host_list[0] + cfg.get_host_name_ext() serverstring=urlstring + "%20mds*%20OR%20%20oss*" for host in j.acct_host_list[1:]: hoststring+="%20OR%20%20host%3D"+host+"*" @@ -324,7 +326,7 @@ def type_detail(request, jid, type_name): # Get job accounting data acct_data = read_sql("""select * from job_data where jid = '{0}'""".format(jid), conj) # job_data accounting host names must be converted to fqdn - acct_host_list = [h + '.' + cfg.host_name_ext for h in acct_data["host_list"].values[0]] + acct_host_list = [h + '.' + cfg.get_host_name_ext() for h in acct_data["host_list"].values[0]] start_time = acct_data["start_time"].dt.tz_convert('US/Central').dt.tz_localize(None).values[0] end_time = acct_data["end_time"].dt.tz_convert('US/Central').dt.tz_localize(None).values[0] @@ -479,4 +481,3 @@ class ChoiceForm(forms.Form): STATECHOICES = [('','')] + [(s, s) for s in states] print(STATECHOICES) state = forms.ChoiceField(choices=STATECHOICES, widget=forms.Select(choices=STATECHOICES)) - From 3b2dd392020ec9de642160506b3cb82f13ffe2b6 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:12:34 -0500 Subject: [PATCH 14/42] Update jid_table.py --- tacc_stats/analysis/gen/jid_table.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tacc_stats/analysis/gen/jid_table.py b/tacc_stats/analysis/gen/jid_table.py index b0421c8..f09dc35 100644 --- a/tacc_stats/analysis/gen/jid_table.py +++ b/tacc_stats/analysis/gen/jid_table.py @@ -1,13 +1,16 @@ -import time +import os,sys,time +sys.path.append("/home/sg99/tacc_stats") import psycopg2 -import tacc_stats.cfg as cfg +import conf_parser as cfg from tacc_stats.analysis.gen.utils import read_sql class jid_table: def __init__(self, jid): - CONNECTION = "dbname=ls6_db1 host=localhost user=postgres port=5432" + CONNECTION = cfg.get_db_connection_string() + print("SOMETHING SHOULD PRINT\n") + print(CONNECTION) print("Initializing table for job {0}".format(jid)) self.jid = jid @@ -18,7 +21,7 @@ def __init__(self, jid): # Get job accounting data acct_data = read_sql("""select * from job_data where jid = '{0}'""".format(jid), self.conj) # job_data accounting host names must be converted to fqdn - self.acct_host_list = [h + '.' + cfg.host_name_ext for h in acct_data["host_list"].values[0]] + self.acct_host_list = [h + '.' + cfg.get_host_name_ext() for h in acct_data["host_list"].values[0]] self.start_time = acct_data["start_time"].dt.tz_convert('US/Central').dt.tz_localize(None).values[0] self.end_time = acct_data["end_time"].dt.tz_convert('US/Central').dt.tz_localize(None).values[0] From a8c5899083bb5debe7bb6e3f6de804e802036b6e Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:02:13 -0500 Subject: [PATCH 15/42] Update conf_parser.py --- conf_parser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conf_parser.py b/conf_parser.py index 45731a5..6ecf30d 100644 --- a/conf_parser.py +++ b/conf_parser.py @@ -3,9 +3,12 @@ import sys import time -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") cfg = configparser.ConfigParser() + +# Append your local repository path here: cfg.read('/home/sg99/tacc_stats/tacc_stats.ini') def get_db_connection_string(): From ba240f6538767b0eaf0be4c55ea6164d43e88c32 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:03:04 -0500 Subject: [PATCH 16/42] Update listend.py --- tacc_stats/listend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/listend.py b/tacc_stats/listend.py index 7df77bb..98f8472 100755 --- a/tacc_stats/listend.py +++ b/tacc_stats/listend.py @@ -5,7 +5,8 @@ from fcntl import flock, LOCK_EX, LOCK_NB -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") import conf_parser as cfg From 72385534c0b63698ae34f5edc0374a31856a8832 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:03:31 -0500 Subject: [PATCH 17/42] Update sync_acct.py --- tacc_stats/dbload/sync_acct.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tacc_stats/dbload/sync_acct.py b/tacc_stats/dbload/sync_acct.py index cac7bc5..7bf8207 100755 --- a/tacc_stats/dbload/sync_acct.py +++ b/tacc_stats/dbload/sync_acct.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os,sys,time -sys.path.append("/home/sg99/tacc_stats") + +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, datetime From cb861b831695c0ae643e145eab4b8d01f01442b7 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:03:52 -0500 Subject: [PATCH 18/42] Update sync_timedb.py --- tacc_stats/dbload/sync_timedb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/dbload/sync_timedb.py b/tacc_stats/dbload/sync_timedb.py index 6a3d1f5..8e226a0 100755 --- a/tacc_stats/dbload/sync_timedb.py +++ b/tacc_stats/dbload/sync_timedb.py @@ -2,7 +2,8 @@ import psycopg2 from pgcopy import CopyManager import os, sys, stat -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") import multiprocessing import itertools from multiprocessing import Pool, get_context, Lock, set_start_method From 2f4e36bae24aa3198ccafc06d565c5a0df165bdc Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:04:12 -0500 Subject: [PATCH 19/42] Update sacct_gen.py --- tacc_stats/dbload/sacct_gen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/dbload/sacct_gen.py b/tacc_stats/dbload/sacct_gen.py index 897b9ec..bbb4ab3 100755 --- a/tacc_stats/dbload/sacct_gen.py +++ b/tacc_stats/dbload/sacct_gen.py @@ -1,5 +1,6 @@ import os, sys -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, date, datetime from dateutil.parser import parse import conf_parser as cfg From 3c8046dacaa0e6f5f648d8fc14430b414aaac076 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:04:51 -0500 Subject: [PATCH 20/42] Update settings.py --- tacc_stats/site/tacc_stats_site/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/site/tacc_stats_site/settings.py b/tacc_stats/site/tacc_stats_site/settings.py index 3393383..f0902cb 100644 --- a/tacc_stats/site/tacc_stats_site/settings.py +++ b/tacc_stats/site/tacc_stats_site/settings.py @@ -1,7 +1,8 @@ # Django settings for tacc_stats_site project. import os import sys -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") import conf_parser as cfg import tacc_stats.site.tacc_stats_site as tacc_stats_site From d7e15334b863f5959bce1b914a4fb57b20abe83c Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:06:02 -0500 Subject: [PATCH 21/42] Update update_xalt.py --- tacc_stats/site/machine/update_xalt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/site/machine/update_xalt.py b/tacc_stats/site/machine/update_xalt.py index 3a2f89c..17b0517 100755 --- a/tacc_stats/site/machine/update_xalt.py +++ b/tacc_stats/site/machine/update_xalt.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os,sys -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta,datetime os.environ['DJANGO_SETTINGS_MODULE']='tacc_stats.site.tacc_stats_site.settings' import django From 2f54f2771728e106cad340ad93512275779a7f2b Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:06:28 -0500 Subject: [PATCH 22/42] Update update_db.py --- tacc_stats/site/machine/update_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/site/machine/update_db.py b/tacc_stats/site/machine/update_db.py index 0dab298..e4930c9 100755 --- a/tacc_stats/site/machine/update_db.py +++ b/tacc_stats/site/machine/update_db.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os,sys, pwd -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, datetime from dateutil.parser import parse from fcntl import flock, LOCK_EX, LOCK_NB From a84893df0117cff7486756e051d8a33bd04b7fab Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:07:05 -0500 Subject: [PATCH 23/42] Update update_metrics.py --- tacc_stats/analysis/metrics/update_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/analysis/metrics/update_metrics.py b/tacc_stats/analysis/metrics/update_metrics.py index b67ff11..7dd8c93 100755 --- a/tacc_stats/analysis/metrics/update_metrics.py +++ b/tacc_stats/analysis/metrics/update_metrics.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os,sys, pwd -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, datetime import psycopg2 From cfe7e3b381a252d7de0e2ec6e1fcfee693c0f2a0 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:08:07 -0500 Subject: [PATCH 24/42] Update views.py --- tacc_stats/site/machine/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/site/machine/views.py b/tacc_stats/site/machine/views.py index 104f74a..8f68c92 100644 --- a/tacc_stats/site/machine/views.py +++ b/tacc_stats/site/machine/views.py @@ -1,5 +1,6 @@ import sys -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") from django.http import HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.views.generic import DetailView, ListView From ce38589f5d5721fddeed767fce8ec6270c579cab Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:08:46 -0500 Subject: [PATCH 25/42] Update jid_table.py --- tacc_stats/analysis/gen/jid_table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tacc_stats/analysis/gen/jid_table.py b/tacc_stats/analysis/gen/jid_table.py index f09dc35..32737c8 100644 --- a/tacc_stats/analysis/gen/jid_table.py +++ b/tacc_stats/analysis/gen/jid_table.py @@ -1,5 +1,6 @@ import os,sys,time -sys.path.append("/home/sg99/tacc_stats") +# Append your local repository path here: +# sys.path.append("/home/sg99/tacc_stats") import psycopg2 import conf_parser as cfg from tacc_stats.analysis.gen.utils import read_sql From acb44dccdf6a74084649ec92592eb4861b3e9680 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Tue, 1 Oct 2024 20:15:42 -0500 Subject: [PATCH 26/42] Update tacc_stats.ini --- tacc_stats.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tacc_stats.ini b/tacc_stats.ini index 75964bb..dec01b2 100644 --- a/tacc_stats.ini +++ b/tacc_stats.ini @@ -1,7 +1,7 @@ [DEFAULT] -machine = tacc-stats-sanga +machine = machine-name data_dir = /tacc_stats_site/%(machine)s -server = tacc-stats04.tacc.utexas.edu +server = servername.tacc.utexas.edu [RMQ] rmq_server = %(server)s @@ -14,8 +14,8 @@ host_name_ext = %(machine)s.tacc.utexas.edu dbname = test_db daily_archive_dir = %(data_dir)s/daily_archive engine_name = django.db.backends.postgresql_psycopg2 -username = taccstats -password = taccstats +username = user +password = password host = localhost port = 5432 From df04d18187086dc81a960e8889e7b8de020f2150 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Tue, 1 Oct 2024 20:16:37 -0500 Subject: [PATCH 27/42] Update jid_table.py --- tacc_stats/analysis/gen/jid_table.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tacc_stats/analysis/gen/jid_table.py b/tacc_stats/analysis/gen/jid_table.py index 32737c8..8a1bb03 100644 --- a/tacc_stats/analysis/gen/jid_table.py +++ b/tacc_stats/analysis/gen/jid_table.py @@ -10,7 +10,6 @@ class jid_table: def __init__(self, jid): CONNECTION = cfg.get_db_connection_string() - print("SOMETHING SHOULD PRINT\n") print(CONNECTION) print("Initializing table for job {0}".format(jid)) From f8d80e59e7633d0f083fe5f990443fbe9e8c3829 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:18:21 -0500 Subject: [PATCH 28/42] Update conf_parser.py --- conf_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf_parser.py b/conf_parser.py index 6ecf30d..d05beed 100644 --- a/conf_parser.py +++ b/conf_parser.py @@ -4,12 +4,12 @@ import time # Append your local repository path here: -# sys.path.append("/home/sg99/tacc_stats") +# sys.path.append("/home/username/tacc_stats") cfg = configparser.ConfigParser() # Append your local repository path here: -cfg.read('/home/sg99/tacc_stats/tacc_stats.ini') +cfg.read('/home/username/tacc_stats/tacc_stats.ini') def get_db_connection_string(): temp_string = "dbname={0} "+cfg.get('PORTAL', 'username')+" "+cfg.get('PORTAL', 'password')+" "+cfg.get('PORTAL', 'port') From 39e8e41300d56f6eb39bbcd006db89c65a38c1fb Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:03:05 -0500 Subject: [PATCH 29/42] Update settings.py --- tacc_stats/site/tacc_stats_site/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/site/tacc_stats_site/settings.py b/tacc_stats/site/tacc_stats_site/settings.py index f0902cb..a1615e1 100644 --- a/tacc_stats/site/tacc_stats_site/settings.py +++ b/tacc_stats/site/tacc_stats_site/settings.py @@ -34,7 +34,7 @@ # Uncomment this portion if an xalt database exists 'xalt' : { #'ENGINE' : 'mysql.connector.django', - 'ENGINE' : cfg.get_xalt_engine() + 'ENGINE' : cfg.get_xalt_engine(), 'NAME' : cfg.get_xalt_name(), 'USER' : cfg.get_xalt_user(), 'PASSWORD' : cfg.get_xalt_password(), From 45c401b4d7b5c12501ed54a4a9472f286d54a8cc Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:18:59 -0500 Subject: [PATCH 30/42] Create conf_parser.py --- tacc_stats/conf_parser.py | 97 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tacc_stats/conf_parser.py diff --git a/tacc_stats/conf_parser.py b/tacc_stats/conf_parser.py new file mode 100644 index 0000000..d05beed --- /dev/null +++ b/tacc_stats/conf_parser.py @@ -0,0 +1,97 @@ +import configparser +import os +import sys +import time + +# Append your local repository path here: +# sys.path.append("/home/username/tacc_stats") + +cfg = configparser.ConfigParser() + +# Append your local repository path here: +cfg.read('/home/username/tacc_stats/tacc_stats.ini') + +def get_db_connection_string(): + temp_string = "dbname={0} "+cfg.get('PORTAL', 'username')+" "+cfg.get('PORTAL', 'password')+" "+cfg.get('PORTAL', 'port') + connection_string = temp_string.format(cfg.get('PORTAL', 'dbname')) + return connection_string + +def get_db_name(): + db_name = cfg.get('PORTAL', 'dbname') + return db_name + +def get_archive_dir_path(): + archive_dir_path = cfg.get('PORTAL', 'archive_dir') + return archive_dir_path + +def get_host_name_ext(): + host_name_ext = cfg.get('PORTAL', 'host_name_ext') + return host_name_ext + +def get_accounting_path(): + accounting_path = cfg.get('PORTAL', 'acct_path') + return accounting_path + +def get_daily_archive_dir_path(): + daily_archive_dir_path = cfg.get('PORTAL', 'daily_archive_dir') + return daily_archive_dir_path + +def get_rmq_server(): + rmq_server = cfg.get('RMQ', 'rmq_server') + return rmq_server + +def get_rmq_queue(): + rmq_queue = cfg.get('RMQ', 'rmq_queue') + return rmq_queue + +def get_machine_name(): + machine_name = cfg.get('DEFAULT', 'machine') + return machine_name + +def get_server_name(): + server_name = cfg.get('DEFAULT', 'server') + return server_name + +def get_data_dir_path(): + data_dir_path = cfg.get('DEFAULT', 'data_dir') + return data_dir_path + +def get_engine_name(): + engine_name = cfg.get('PORTAL', 'engine_name') + return engine_name + +def get_username(): + username = cfg.get('PORTAL', 'username') + return username + +def get_password(): + password = cfg.get('PORTAL', 'password') + return password + +def get_host(): + host = cfg.get('PORTAL', 'host') + return host + +def get_port(): + port = cfg.get('PORTAL', 'port') + return port + +def get_xalt_engine(): + xalt_engine = cfg.get('XALT', 'xalt_engine') + return xalt_engine + +def get_xalt_name(): + xalt_name = cfg.get('XALT', 'xalt_name') + return xalt_name + +def get_xalt_user(): + xalt_user = cfg.get('XALT', 'xalt_user') + return xalt_user + +def get_xalt_password(): + xalt_password = cfg.get('XALT', 'xalt_password') + return xalt_password + +def get_xalt_host(): + xalt_host = cfg.get('XALT', 'xalt_host') + return xalt_host From f3c84a8f488f8b8bba27d27ddc8806549e842b58 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:20:18 -0500 Subject: [PATCH 31/42] Delete conf_parser.py --- conf_parser.py | 97 -------------------------------------------------- 1 file changed, 97 deletions(-) delete mode 100644 conf_parser.py diff --git a/conf_parser.py b/conf_parser.py deleted file mode 100644 index d05beed..0000000 --- a/conf_parser.py +++ /dev/null @@ -1,97 +0,0 @@ -import configparser -import os -import sys -import time - -# Append your local repository path here: -# sys.path.append("/home/username/tacc_stats") - -cfg = configparser.ConfigParser() - -# Append your local repository path here: -cfg.read('/home/username/tacc_stats/tacc_stats.ini') - -def get_db_connection_string(): - temp_string = "dbname={0} "+cfg.get('PORTAL', 'username')+" "+cfg.get('PORTAL', 'password')+" "+cfg.get('PORTAL', 'port') - connection_string = temp_string.format(cfg.get('PORTAL', 'dbname')) - return connection_string - -def get_db_name(): - db_name = cfg.get('PORTAL', 'dbname') - return db_name - -def get_archive_dir_path(): - archive_dir_path = cfg.get('PORTAL', 'archive_dir') - return archive_dir_path - -def get_host_name_ext(): - host_name_ext = cfg.get('PORTAL', 'host_name_ext') - return host_name_ext - -def get_accounting_path(): - accounting_path = cfg.get('PORTAL', 'acct_path') - return accounting_path - -def get_daily_archive_dir_path(): - daily_archive_dir_path = cfg.get('PORTAL', 'daily_archive_dir') - return daily_archive_dir_path - -def get_rmq_server(): - rmq_server = cfg.get('RMQ', 'rmq_server') - return rmq_server - -def get_rmq_queue(): - rmq_queue = cfg.get('RMQ', 'rmq_queue') - return rmq_queue - -def get_machine_name(): - machine_name = cfg.get('DEFAULT', 'machine') - return machine_name - -def get_server_name(): - server_name = cfg.get('DEFAULT', 'server') - return server_name - -def get_data_dir_path(): - data_dir_path = cfg.get('DEFAULT', 'data_dir') - return data_dir_path - -def get_engine_name(): - engine_name = cfg.get('PORTAL', 'engine_name') - return engine_name - -def get_username(): - username = cfg.get('PORTAL', 'username') - return username - -def get_password(): - password = cfg.get('PORTAL', 'password') - return password - -def get_host(): - host = cfg.get('PORTAL', 'host') - return host - -def get_port(): - port = cfg.get('PORTAL', 'port') - return port - -def get_xalt_engine(): - xalt_engine = cfg.get('XALT', 'xalt_engine') - return xalt_engine - -def get_xalt_name(): - xalt_name = cfg.get('XALT', 'xalt_name') - return xalt_name - -def get_xalt_user(): - xalt_user = cfg.get('XALT', 'xalt_user') - return xalt_user - -def get_xalt_password(): - xalt_password = cfg.get('XALT', 'xalt_password') - return xalt_password - -def get_xalt_host(): - xalt_host = cfg.get('XALT', 'xalt_host') - return xalt_host From 3219366e4c81c058b324df35b5711e05d51ae8e8 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:27:10 -0500 Subject: [PATCH 32/42] Update listend.py --- tacc_stats/listend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/listend.py b/tacc_stats/listend.py index 98f8472..81ae448 100755 --- a/tacc_stats/listend.py +++ b/tacc_stats/listend.py @@ -8,7 +8,7 @@ # Append your local repository path here: # sys.path.append("/home/sg99/tacc_stats") -import conf_parser as cfg +import tacc_stats.conf_parser as cfg def on_message(channel, method_frame, header_frame, body): From df3d34047408ccb508b2ca2f514ecf9b300b2ff0 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:27:47 -0500 Subject: [PATCH 33/42] Update sync_acct.py --- tacc_stats/dbload/sync_acct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/dbload/sync_acct.py b/tacc_stats/dbload/sync_acct.py index 7bf8207..bc28cee 100755 --- a/tacc_stats/dbload/sync_acct.py +++ b/tacc_stats/dbload/sync_acct.py @@ -16,7 +16,7 @@ from tacc_stats.analysis.gen.utils import read_sql -import conf_parser as cfg +import tacc_stats.conf_parser as cfg CONNECTION = cfg.get_db_connection_string() From fff2b4f28eb6eae0631916c889b40d5cf5fd7e56 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:28:13 -0500 Subject: [PATCH 34/42] Update sacct_gen.py --- tacc_stats/dbload/sacct_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/dbload/sacct_gen.py b/tacc_stats/dbload/sacct_gen.py index bbb4ab3..dd6bc91 100755 --- a/tacc_stats/dbload/sacct_gen.py +++ b/tacc_stats/dbload/sacct_gen.py @@ -3,7 +3,7 @@ # sys.path.append("/home/sg99/tacc_stats") from datetime import timedelta, date, datetime from dateutil.parser import parse -import conf_parser as cfg +import tacc_stats.conf_parser as cfg acct_path = cfg.get_accounting_path() From c016690404ad3180be3457df4aec77878c51c6ef Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:28:41 -0500 Subject: [PATCH 35/42] Update sync_timedb.py --- tacc_stats/dbload/sync_timedb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/dbload/sync_timedb.py b/tacc_stats/dbload/sync_timedb.py index 8e226a0..5e3e710 100755 --- a/tacc_stats/dbload/sync_timedb.py +++ b/tacc_stats/dbload/sync_timedb.py @@ -20,7 +20,7 @@ from tacc_stats.analysis.gen.utils import read_sql -import conf_parser as cfg +import tacc_stats.conf_parser as cfg # archive toggle should_archive = True From b8739a1f82bae6c41a4cc4d4f40801a9cc8cdb7b Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:29:41 -0500 Subject: [PATCH 36/42] Update settings.py --- tacc_stats/site/tacc_stats_site/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/site/tacc_stats_site/settings.py b/tacc_stats/site/tacc_stats_site/settings.py index a1615e1..496629f 100644 --- a/tacc_stats/site/tacc_stats_site/settings.py +++ b/tacc_stats/site/tacc_stats_site/settings.py @@ -3,7 +3,7 @@ import sys # Append your local repository path here: # sys.path.append("/home/sg99/tacc_stats") -import conf_parser as cfg +import tacc_stats.conf_parser as cfg import tacc_stats.site.tacc_stats_site as tacc_stats_site DIR = os.path.dirname(os.path.abspath(__file__)) From 825faaba00def800e45e803501b4c05ffcc34094 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:30:13 -0500 Subject: [PATCH 37/42] Update update_xalt.py --- tacc_stats/site/machine/update_xalt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/site/machine/update_xalt.py b/tacc_stats/site/machine/update_xalt.py index 17b0517..3956206 100755 --- a/tacc_stats/site/machine/update_xalt.py +++ b/tacc_stats/site/machine/update_xalt.py @@ -7,7 +7,7 @@ import django django.setup() from tacc_stats.site.machine import views -import conf_parser as cfg +import tacc_stats.conf_parser as cfg from tacc_stats.site.machine.models import Job, Libraries from tacc_stats.site.xalt.models import run, join_run_object, lib From a86929887c4fa4c1402df7ccc397892ce9ef5cdc Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:30:38 -0500 Subject: [PATCH 38/42] Update update_db.py --- tacc_stats/site/machine/update_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/site/machine/update_db.py b/tacc_stats/site/machine/update_db.py index e4930c9..355d530 100755 --- a/tacc_stats/site/machine/update_db.py +++ b/tacc_stats/site/machine/update_db.py @@ -11,7 +11,7 @@ from tacc_stats.site.machine.models import Job, Host, Libraries from tacc_stats.site.xalt.models import run, join_run_object, lib from tacc_stats.analysis.metrics import metrics -import conf_parser as cfg +import tacc_stats.conf_parser as cfg from tacc_stats.progress import progress from tacc_stats.daterange import daterange import pytz, calendar From ffcd09bff76f1e3aeabd47fea889267ab6bb8b45 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:31:12 -0500 Subject: [PATCH 39/42] Update update_metrics.py --- tacc_stats/analysis/metrics/update_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/analysis/metrics/update_metrics.py b/tacc_stats/analysis/metrics/update_metrics.py index 7dd8c93..a2db7c7 100755 --- a/tacc_stats/analysis/metrics/update_metrics.py +++ b/tacc_stats/analysis/metrics/update_metrics.py @@ -12,7 +12,7 @@ from tacc_stats.site.machine.models import job_data, metrics_data from tacc_stats.analysis.metrics import metrics -import conf_parser as cfg +import tacc_stats.conf_parser as cfg from tacc_stats.progress import progress CONNECTION = cfg.get_db_connection_string() From 444db6b6876582fed409826a79fd28dc063e9df6 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:32:15 -0500 Subject: [PATCH 40/42] Update views.py --- tacc_stats/site/machine/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/site/machine/views.py b/tacc_stats/site/machine/views.py index 8f68c92..c0a1e22 100644 --- a/tacc_stats/site/machine/views.py +++ b/tacc_stats/site/machine/views.py @@ -20,7 +20,7 @@ from tacc_stats.analysis.gen import jid_table from tacc_stats.site.machine.models import job_data, metrics_data from tacc_stats.site.xalt.models import run, join_run_object, lib -import conf_parser as cfg +import tacc_stats.conf_parser as cfg import tacc_stats.analysis.plot as plots #xalt From d7ac0107041f413fe6d58cd29617310b2f6414a0 Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:32:55 -0500 Subject: [PATCH 41/42] Update jid_table.py --- tacc_stats/analysis/gen/jid_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/analysis/gen/jid_table.py b/tacc_stats/analysis/gen/jid_table.py index 8a1bb03..74a665f 100644 --- a/tacc_stats/analysis/gen/jid_table.py +++ b/tacc_stats/analysis/gen/jid_table.py @@ -2,7 +2,7 @@ # Append your local repository path here: # sys.path.append("/home/sg99/tacc_stats") import psycopg2 -import conf_parser as cfg +import tacc_stats.conf_parser as cfg from tacc_stats.analysis.gen.utils import read_sql class jid_table: From 87dd9cc222bbc88fe88dacf8bf615f34deb9388b Mon Sep 17 00:00:00 2001 From: sanga1999 <153561744+sanga1999@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:58:52 -0500 Subject: [PATCH 42/42] Update conf_parser.py --- tacc_stats/conf_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tacc_stats/conf_parser.py b/tacc_stats/conf_parser.py index d05beed..438e9ff 100644 --- a/tacc_stats/conf_parser.py +++ b/tacc_stats/conf_parser.py @@ -12,7 +12,7 @@ cfg.read('/home/username/tacc_stats/tacc_stats.ini') def get_db_connection_string(): - temp_string = "dbname={0} "+cfg.get('PORTAL', 'username')+" "+cfg.get('PORTAL', 'password')+" "+cfg.get('PORTAL', 'port') + temp_string = "dbname={0} user="+cfg.get('PORTAL', 'username')+" password="+cfg.get('PORTAL', 'password')+" port="+cfg.get('PORTAL', 'port') connection_string = temp_string.format(cfg.get('PORTAL', 'dbname')) return connection_string