From ff2baf301425ece279b40e167c784c96fc217156 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 11:20:42 +0100 Subject: [PATCH 01/28] Refactor code to allow for multiple components to be accounted --- .../nova_extract_accounting.py | 0 .../nova_get_accounting_data.sql | 0 .../now-accounting.sh | 5 ++++- .../past-accounting.sh | 5 ++++- 4 files changed, 8 insertions(+), 2 deletions(-) rename OpenStack-nova-accounting/extract_accounting.py => OpenStack-accounting/nova_extract_accounting.py (100%) rename OpenStack-nova-accounting/get_accounting_data.sql => OpenStack-accounting/nova_get_accounting_data.sql (100%) rename {OpenStack-nova-accounting => OpenStack-accounting}/now-accounting.sh (71%) rename {OpenStack-nova-accounting => OpenStack-accounting}/past-accounting.sh (82%) diff --git a/OpenStack-nova-accounting/extract_accounting.py b/OpenStack-accounting/nova_extract_accounting.py similarity index 100% rename from OpenStack-nova-accounting/extract_accounting.py rename to OpenStack-accounting/nova_extract_accounting.py diff --git a/OpenStack-nova-accounting/get_accounting_data.sql b/OpenStack-accounting/nova_get_accounting_data.sql similarity index 100% rename from OpenStack-nova-accounting/get_accounting_data.sql rename to OpenStack-accounting/nova_get_accounting_data.sql diff --git a/OpenStack-nova-accounting/now-accounting.sh b/OpenStack-accounting/now-accounting.sh similarity index 71% rename from OpenStack-nova-accounting/now-accounting.sh rename to OpenStack-accounting/now-accounting.sh index b288e072..bfe43c28 100644 --- a/OpenStack-nova-accounting/now-accounting.sh +++ b/OpenStack-accounting/now-accounting.sh @@ -9,4 +9,7 @@ startdateepoch=`date -d "$startdate" +%s` echo $startdate echo $enddate -/usr/local/bin/extract_accounting.py "$startdate" "$enddate" +for extractor in $(ls /usr/local/bin/*extract_accounting.py); +do + $extractor "$startdate" "$enddate"; +done diff --git a/OpenStack-nova-accounting/past-accounting.sh b/OpenStack-accounting/past-accounting.sh similarity index 82% rename from OpenStack-nova-accounting/past-accounting.sh rename to OpenStack-accounting/past-accounting.sh index c9352015..458410cb 100644 --- a/OpenStack-nova-accounting/past-accounting.sh +++ b/OpenStack-accounting/past-accounting.sh @@ -19,6 +19,9 @@ do endtimeepoch=`date -d "$endtime" +%s` echo $starttime echo $endtime - /usr/local/bin/extract_accounting.py "$starttime" "$endtime" + for extractor in $(ls /usr/local/bin/*extract_accounting.py); + do + $extractor "$starttime" "$endtime"; + done starttime=$endtime done; From 5c2648fbc6e17c3a7b00fd6976d98d5d3db8ed10 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 11:57:46 +0100 Subject: [PATCH 02/28] Convert to python3 --- .../nova_extract_accounting.py | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/OpenStack-accounting/nova_extract_accounting.py b/OpenStack-accounting/nova_extract_accounting.py index 11057a56..757b5cbe 100644 --- a/OpenStack-accounting/nova_extract_accounting.py +++ b/OpenStack-accounting/nova_extract_accounting.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import time import datetime #from datetime import datetime,time @@ -8,37 +8,44 @@ import sqlalchemy from sqlalchemy.sql import select from sqlalchemy.orm import sessionmaker -from ConfigParser import SafeConfigParser +import configparser -#This config parsing needs rewriting to support python3 -#The parsing of the nova.conf seems to not work with the python3 version +def ifnull(var, val): + if var is None: + return val + return var # Read from config file -parser = SafeConfigParser() +influx_parser = configparser.SafeConfigParser() try: - parser.read('/etc/influxdb.conf') - host = parser.get('db', 'host') - database = parser.get('db', 'database') - username = parser.get('auth', 'username') - password = parser.get('auth', 'password') - instance = parser.get('cloud','instance') + influx_parser.read('/etc/influxdb.conf') + host = influx_parser.get('db', 'host') + database = influx_parser.get('db', 'database') + username = influx_parser.get('auth', 'username') + password = influx_parser.get('auth', 'password') + instance = influx_parser.get('cloud','instance') except: - raise RuntimeError('Unable to read from config file') + print('Unable to read from influx config file') sys.exit(1) url = 'http://'+host+'/write?db='+database +'&precision=s' nowtime = time.localtime() +nova_parser = configparser.RawConfigParser(strict=False) + +nova_parser.read('/etc/nova/nova.conf') try: - parser.read('/etc/nova/nova.conf') - connectionstring = parser.get('database','connection') + nova_parser.read('/etc/nova/nova.conf') + connectionstring = nova_parser.get('database','connection') except: - raise RuntimeError('Unable to read from config file') + print('Unable to read from nova config file') sys.exit(1) +starttime='2017-04-19 12:00' starttime=sys.argv[1] print(starttime) +endtime='2017-04-19 12:15' endtime=sys.argv[2] print(endtime) endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') @@ -50,7 +57,6 @@ sess = sessionmaker(bind=engine)() query = 'call get_accounting_data( "' + starttime +'","' + endtime + '")' - print(query) results = sess.execute(query, { 'p1': starttime, 'p2': endtime }) @@ -96,7 +102,8 @@ datastring += ",Root_GBs="+str(result["Root_GB"] * result["VMs"]) datastring += ",Ephemeral_GB_Seconds="+str(result["Ephemeral_GB"] * result['VM_Seconds']) datastring += ",Ephemeral_GBs="+str(result["Ephemeral_GB"] * result["VMs"]) - if result["GPU_Num"] > 0: + print(str(ifnull(result["GPU_Num"],0))) + if int( ifnull(result["GPU_Num"],0)) > 0: datastring += ",GPU_Seconds="+str(float(result["GPU_Num"]) * float(result['VM_Seconds'])) datastring += ",GPUs=" + str(float(result["GPU_Num"]) * float(result['VMs'])) print(result) @@ -106,12 +113,12 @@ datastring += ",GPUs=" + str(0) datastring += ",GPU_Seconds=" + str(0) - datastring += " "+str(long(endtimestamp)) - datastring += "\n" -print(datastring) + datastring += " "+str(int(endtimestamp)) + datastring += "\n" +print(datastring) r = requests.post(url,data=datastring,auth=(username,password)) print(r.text) print(r) From aab6b24e658b02d903e90e5c043ae368b42afa41 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 14:00:43 +0100 Subject: [PATCH 03/28] Neaten up formatting of sql script --- OpenStack-accounting/nova_get_accounting_data.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OpenStack-accounting/nova_get_accounting_data.sql b/OpenStack-accounting/nova_get_accounting_data.sql index 66663fb1..28b6bd62 100644 --- a/OpenStack-accounting/nova_get_accounting_data.sql +++ b/OpenStack-accounting/nova_get_accounting_data.sql @@ -1,4 +1,4 @@ -CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) +CREATE DEFINER=`accounting-db`@`host-172-16-101-160.nubes.stfc.ac.uk` PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) BEGIN SELECT IFNULL(i.availability_zone,'nova') as AvailabilityZone, @@ -34,9 +34,9 @@ SELECT it.swap AS Swap, it.root_gb AS Root_GB, it.ephemeral_gb AS Ephemeral_GB, - ifnull((select value from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like '%per_unit_cost%' and es.key like CONCAT('%',cast(YEAR(endtime) as NCHAR),'%')),0) as Per_Unit_Cost, + ifnull((select value from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like '%per_unit_cost%' and es.key like CONCAT('%',cast(YEAR(DATE_SUB(endtime, INTERVAL 3 MONTH)) as NCHAR),'%') limit 1),0) as Per_Unit_Cost, # for January to March pull in the previous year's price so we work to financial years ifnull((select value from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like 'accounting:unit%' ), "core") as Charge_Unit, - (select IFNULL(value,0) from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like '%gpu_num%' ) as GPU_Num + (select IFNULL(value,0) from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like 'accounting:gpu_num%' ) as GPU_Num FROM nova.instances i JOIN From 23e5930551a27b13a90f263b9fda3c38d523ebe9 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 15:14:06 +0100 Subject: [PATCH 04/28] Refactor to use functions and to use a dedicated accounting db account --- OpenStack-accounting/accountinglib.py | 54 +++++++++++++++++++ .../etc/thecount/thecount.conf.example | 2 + .../nova_extract_accounting.py | 53 +++--------------- OpenStack-accounting/readme.md | 7 +++ 4 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 OpenStack-accounting/accountinglib.py create mode 100644 OpenStack-accounting/etc/thecount/thecount.conf.example create mode 100644 OpenStack-accounting/readme.md diff --git a/OpenStack-accounting/accountinglib.py b/OpenStack-accounting/accountinglib.py new file mode 100644 index 00000000..9454c0a2 --- /dev/null +++ b/OpenStack-accounting/accountinglib.py @@ -0,0 +1,54 @@ +#!/usr/bin/python3 +import time +import datetime +#from datetime import datetime,time +import json +import requests +import sys +import sqlalchemy +from sqlalchemy.sql import select +from sqlalchemy.orm import sessionmaker +import configparser + +def ifnull(var, val): + if var is None: + return val + return var + +def send_to_influx(datastring): + # Read from config file + influx_parser = configparser.SafeConfigParser() + try: + influx_parser.read('/etc/influxdb.conf') + host = influx_parser.get('db', 'host') + database = influx_parser.get('db', 'database') + username = influx_parser.get('auth', 'username') + password = influx_parser.get('auth', 'password') + instance = influx_parser.get('cloud','instance') + except: + print('Unable to read from influx config file') + sys.exit(1) + finaldatastring = datastring.replace("Accounting,","Accounting,instance="+instance+",") + print(finaldatastring) + url = 'http://'+host+'/write?db='+database +'&precision=s' + response = requests.post(url,data=finaldatastring,auth=(username,password)) + return response + +def get_accounting_data(database,starttime,endtime): + thecount_parser = configparser.RawConfigParser(strict=False) + + try: + thecount_parser.read('/etc/thecount/thecount.conf') + connectionstring = thecount_parser.get('database','connection') + '/' + database + except: + print('Unable to read from thecount config file') + sys.exit(1) + + engine = sqlalchemy.create_engine(connectionstring, encoding='utf-8') + connection = engine.connect() + sess = sessionmaker(bind=engine)() + query = 'call get_accounting_data( "' + starttime +'","' + endtime + '")' + + print(query) + results = sess.execute(query, { 'p1': starttime, 'p2': endtime }) + return results diff --git a/OpenStack-accounting/etc/thecount/thecount.conf.example b/OpenStack-accounting/etc/thecount/thecount.conf.example new file mode 100644 index 00000000..a0dd7141 --- /dev/null +++ b/OpenStack-accounting/etc/thecount/thecount.conf.example @@ -0,0 +1,2 @@ +[database] +connection =mysql+pymysql://:@:3306 diff --git a/OpenStack-accounting/nova_extract_accounting.py b/OpenStack-accounting/nova_extract_accounting.py index 757b5cbe..9ac508f2 100644 --- a/OpenStack-accounting/nova_extract_accounting.py +++ b/OpenStack-accounting/nova_extract_accounting.py @@ -1,7 +1,8 @@ #!/usr/bin/python3 +import accountinglib + import time import datetime -#from datetime import datetime,time import json import requests import sys @@ -10,38 +11,8 @@ from sqlalchemy.orm import sessionmaker import configparser -def ifnull(var, val): - if var is None: - return val - return var - -# Read from config file -influx_parser = configparser.SafeConfigParser() -try: - influx_parser.read('/etc/influxdb.conf') - host = influx_parser.get('db', 'host') - database = influx_parser.get('db', 'database') - username = influx_parser.get('auth', 'username') - password = influx_parser.get('auth', 'password') - instance = influx_parser.get('cloud','instance') -except: - print('Unable to read from influx config file') - sys.exit(1) - -url = 'http://'+host+'/write?db='+database +'&precision=s' - nowtime = time.localtime() -nova_parser = configparser.RawConfigParser(strict=False) - -nova_parser.read('/etc/nova/nova.conf') -try: - nova_parser.read('/etc/nova/nova.conf') - connectionstring = nova_parser.get('database','connection') -except: - print('Unable to read from nova config file') - sys.exit(1) - starttime='2017-04-19 12:00' starttime=sys.argv[1] print(starttime) @@ -52,15 +23,8 @@ def ifnull(var, val): endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) print(endtimestamp) -engine = sqlalchemy.create_engine(connectionstring, encoding='utf-8') -connection = engine.connect() -sess = sessionmaker(bind=engine)() -query = 'call get_accounting_data( "' + starttime +'","' + endtime + '")' - -print(query) -results = sess.execute(query, { 'p1': starttime, 'p2': endtime }) - - +results = accountinglib.get_accounting_data("nova",starttime,endtime) +print(results) datastring = '' for result in results: print(result) @@ -81,7 +45,7 @@ def ifnull(var, val): datastring += "Accounting" - datastring += ",instance="+instance + datastring += ",AvailabilityZone="+result["AvailabilityZone"] datastring += ",Project="+result["Project"].replace(' ','\ ') datastring += ",Department="+department.replace(' ','\ ') @@ -102,8 +66,8 @@ def ifnull(var, val): datastring += ",Root_GBs="+str(result["Root_GB"] * result["VMs"]) datastring += ",Ephemeral_GB_Seconds="+str(result["Ephemeral_GB"] * result['VM_Seconds']) datastring += ",Ephemeral_GBs="+str(result["Ephemeral_GB"] * result["VMs"]) - print(str(ifnull(result["GPU_Num"],0))) - if int( ifnull(result["GPU_Num"],0)) > 0: + print(str(accountinglib.ifnull(result["GPU_Num"],0))) + if int( accountinglib.ifnull(result["GPU_Num"],0)) > 0: datastring += ",GPU_Seconds="+str(float(result["GPU_Num"]) * float(result['VM_Seconds'])) datastring += ",GPUs=" + str(float(result["GPU_Num"]) * float(result['VMs'])) print(result) @@ -117,8 +81,7 @@ def ifnull(var, val): datastring += " "+str(int(endtimestamp)) datastring += "\n" +r = accountinglib.send_to_influx(datastring) -print(datastring) -r = requests.post(url,data=datastring,auth=(username,password)) print(r.text) print(r) diff --git a/OpenStack-accounting/readme.md b/OpenStack-accounting/readme.md new file mode 100644 index 00000000..88f8b4ab --- /dev/null +++ b/OpenStack-accounting/readme.md @@ -0,0 +1,7 @@ +Prerequisites +python-ConfigParser +python3-requests +python36-sqlalchemy +python36-PyMySQL + +Installation From c33022b66cffefba3ba19f522fd53da25969325d Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 15:22:51 +0100 Subject: [PATCH 05/28] Reorganise directory structure and add readme --- OpenStack-accounting/readme.md | 17 +++++++++++++---- .../{ => sql}/nova_get_accounting_data.sql | 0 .../{ => usr/local/sbin}/accountinglib.py | 0 .../local/sbin}/nova_extract_accounting.py | 0 .../{ => usr/local/sbin}/now-accounting.sh | 0 .../{ => usr/local/sbin}/past-accounting.sh | 0 6 files changed, 13 insertions(+), 4 deletions(-) rename OpenStack-accounting/{ => sql}/nova_get_accounting_data.sql (100%) rename OpenStack-accounting/{ => usr/local/sbin}/accountinglib.py (100%) rename OpenStack-accounting/{ => usr/local/sbin}/nova_extract_accounting.py (100%) rename OpenStack-accounting/{ => usr/local/sbin}/now-accounting.sh (100%) rename OpenStack-accounting/{ => usr/local/sbin}/past-accounting.sh (100%) diff --git a/OpenStack-accounting/readme.md b/OpenStack-accounting/readme.md index 88f8b4ab..79573274 100644 --- a/OpenStack-accounting/readme.md +++ b/OpenStack-accounting/readme.md @@ -1,7 +1,16 @@ Prerequisites -python-ConfigParser -python3-requests -python36-sqlalchemy -python36-PyMySQL +The following packages are required: + - python-ConfigParser + - python3-requests + - python36-sqlalchemy + - python36-PyMySQL Installation +Copy the scripts into `/usr/local/sbin` as shown in this repo +Create a config file with a database connection string in the format shown in `/etc/thecount/thecount.conf.example` in `/etc/thecount/thecount.conf` +Create the stored procedures in the `sql` directory appropriate db. + +Use +`now-accounting.sh` generates accounting for the last 24 hours - recommend setting up a cron to run this at midnight +`past-accounting.sh` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for past usage where possible +`*-extract_accounting.py` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for that component diff --git a/OpenStack-accounting/nova_get_accounting_data.sql b/OpenStack-accounting/sql/nova_get_accounting_data.sql similarity index 100% rename from OpenStack-accounting/nova_get_accounting_data.sql rename to OpenStack-accounting/sql/nova_get_accounting_data.sql diff --git a/OpenStack-accounting/accountinglib.py b/OpenStack-accounting/usr/local/sbin/accountinglib.py similarity index 100% rename from OpenStack-accounting/accountinglib.py rename to OpenStack-accounting/usr/local/sbin/accountinglib.py diff --git a/OpenStack-accounting/nova_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py similarity index 100% rename from OpenStack-accounting/nova_extract_accounting.py rename to OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py diff --git a/OpenStack-accounting/now-accounting.sh b/OpenStack-accounting/usr/local/sbin/now-accounting.sh similarity index 100% rename from OpenStack-accounting/now-accounting.sh rename to OpenStack-accounting/usr/local/sbin/now-accounting.sh diff --git a/OpenStack-accounting/past-accounting.sh b/OpenStack-accounting/usr/local/sbin/past-accounting.sh similarity index 100% rename from OpenStack-accounting/past-accounting.sh rename to OpenStack-accounting/usr/local/sbin/past-accounting.sh From 0d8a421d7dfa778d09d5230ba250fd102276716c Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 15:26:29 +0100 Subject: [PATCH 06/28] Tidy up code --- .../usr/local/sbin/nova_extract_accounting.py | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py index 9ac508f2..b9afa70f 100644 --- a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py @@ -3,32 +3,22 @@ import time import datetime -import json -import requests -import sys -import sqlalchemy -from sqlalchemy.sql import select -from sqlalchemy.orm import sessionmaker -import configparser + nowtime = time.localtime() -starttime='2017-04-19 12:00' +print("Nova Accounting run start") starttime=sys.argv[1] -print(starttime) -endtime='2017-04-19 12:15' +print("Start Time = " + starttime) endtime=sys.argv[2] -print(endtime) +print("End Time = " + endtime) endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) -print(endtimestamp) + results = accountinglib.get_accounting_data("nova",starttime,endtime) -print(results) datastring = '' for result in results: - print(result) - try: if "rally" in result["Project"] : department="STFC Cloud" @@ -66,7 +56,6 @@ datastring += ",Root_GBs="+str(result["Root_GB"] * result["VMs"]) datastring += ",Ephemeral_GB_Seconds="+str(result["Ephemeral_GB"] * result['VM_Seconds']) datastring += ",Ephemeral_GBs="+str(result["Ephemeral_GB"] * result["VMs"]) - print(str(accountinglib.ifnull(result["GPU_Num"],0))) if int( accountinglib.ifnull(result["GPU_Num"],0)) > 0: datastring += ",GPU_Seconds="+str(float(result["GPU_Num"]) * float(result['VM_Seconds'])) datastring += ",GPUs=" + str(float(result["GPU_Num"]) * float(result['VMs'])) From 211f3bc4b8849b3d716282059a54d106595a8bf5 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 15:38:04 +0100 Subject: [PATCH 07/28] Add back in needed import --- OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py index b9afa70f..2a4e1c9d 100644 --- a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import accountinglib +import sys import time import datetime From f5a78dc1a824c4da9d4379a67427c5375f1a9709 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 4 Oct 2023 16:20:58 +0100 Subject: [PATCH 08/28] Add functionality for manila and cinder accounting --- .../sql/cinder_get_accounting_data.sql | 41 +++++++++++++ .../sql/manila_get_accounting_data.sql | 48 ++++++++++++++++ .../local/sbin/cinder_extract_accounting.py | 56 ++++++++++++++++++ .../local/sbin/manila_extract_accounting.py | 57 +++++++++++++++++++ 4 files changed, 202 insertions(+) create mode 100644 OpenStack-accounting/sql/cinder_get_accounting_data.sql create mode 100644 OpenStack-accounting/sql/manila_get_accounting_data.sql create mode 100644 OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py create mode 100644 OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py diff --git a/OpenStack-accounting/sql/cinder_get_accounting_data.sql b/OpenStack-accounting/sql/cinder_get_accounting_data.sql new file mode 100644 index 00000000..104376c3 --- /dev/null +++ b/OpenStack-accounting/sql/cinder_get_accounting_data.sql @@ -0,0 +1,41 @@ +CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) +BEGIN +SELECT + IFNULL(v.availability_zone, 'nova') AS AvailabilityZone, + p.name AS Project, + pp.name AS Department, + COUNT(v.id) AS Volumes, + "Volume" as CinderType, + @VolumeSeconds:=SUM(IF(v.created_at <= starttime + AND (v.deleted_at >= endtime + OR ISNULL(v.deleted_at)), + TIMESTAMPDIFF(SECOND, + starttime, + endtime), + IF(v.created_at <= starttime + AND v.deleted_at < endtime, + TIMESTAMPDIFF(SECOND, + starttime, + v.deleted_at), + IF(v.created_at > starttime + AND (v.deleted_at >= endtime + OR ISNULL(v.deleted_at)), + TIMESTAMPDIFF(SECOND, + v.created_at, + endtime), + TIMESTAMPDIFF(SECOND, + v.created_at, + v.deleted_at))))) AS Volume_Seconds, + v.size AS Volume_GB +FROM + cinder.volumes v + JOIN + keystone.project p ON v.project_id = p.id + JOIN + keystone.project pp ON p.parent_id = pp.id +WHERE + v.created_at <= endtime + AND (v.deleted_at >= starttime + OR ISNULL(v.deleted_at)) +GROUP BY v.availability_zone , v.size , p.name , pp.name; +END diff --git a/OpenStack-accounting/sql/manila_get_accounting_data.sql b/OpenStack-accounting/sql/manila_get_accounting_data.sql new file mode 100644 index 00000000..8e2c0440 --- /dev/null +++ b/OpenStack-accounting/sql/manila_get_accounting_data.sql @@ -0,0 +1,48 @@ +CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) +BEGIN +SELECT + p.name AS Project, + pp.name AS Department, + maz.name AS Availability_zone, + st.name AS Share_type, + COUNT(m.id) AS Shares, + "Share" as ManilaType, + @ShareSeconds:=SUM(IF(m.created_at <= starttime + AND (m.deleted_at >= endtime + OR ISNULL(m.deleted_at)), + TIMESTAMPDIFF(SECOND, + starttime, + endtime), + IF(m.created_at <= starttime + AND m.deleted_at < endtime, + TIMESTAMPDIFF(SECOND, + starttime, + m.deleted_at), + IF(m.created_at > starttime + AND (m.deleted_at >= endtime + OR ISNULL(m.deleted_at)), + TIMESTAMPDIFF(SECOND, + m.created_at, + endtime), + TIMESTAMPDIFF(SECOND, + m.created_at, + m.deleted_at))))) AS Share_Seconds, + m.size AS Share_GB +FROM + manila.shares m + JOIN + manila.share_instances si ON m.id = si.share_id + JOIN + manila.share_types st ON si.share_type_id = st.id + JOIN + manila.availability_zones maz ON maz.id = si.availability_zone_id + JOIN + keystone.project p ON m.project_id = p.id + JOIN + keystone.project pp ON p.parent_id = pp.id +WHERE + m.created_at <= endtime + AND (m.deleted_at >= starttime + OR ISNULL(m.deleted_at)) +GROUP BY m.size , p.name , pp.name , maz.name , st.name; +END diff --git a/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py new file mode 100644 index 00000000..2c31cc56 --- /dev/null +++ b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py @@ -0,0 +1,56 @@ +#!/usr/bin/python3 +import accountinglib + +import sys +import time +import datetime + + +nowtime = time.localtime() + +print("Cinder Accounting run start") +starttime=sys.argv[1] +print("Start Time = " + starttime) +endtime=sys.argv[2] +print("End Time = " + endtime) +endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') +endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) + + +results = accountinglib.get_accounting_data("cinder",starttime,endtime) +datastring = '' +print(results) +for result in results: + print(result) + try: + if "rally" in result["Project"] : + department="STFC Cloud" + else: + if "default" in result["Department"]: + department=result["Project"] + else: + department=result["Department"] + + except: + department="UNKNOWN" + + datastring += "Accounting" + + datastring += ",AvailabilityZone="+result["AvailabilityZone"] + datastring += ",Project="+result["Project"].replace(' ','\ ') + datastring += ",Department="+department.replace(' ','\ ') + datastring += ",CinderType="+result["CinderType"] + datastring += ",YYYY-MM="+ endyyyymm + datastring += " Volumes="+str(result["Volumes"]) + datastring += ",Volume_Seconds="+str(result["Volume_Seconds"]) + datastring += ",CinderGBs="+str(result["Volume_GB"] * result['Volume_Seconds']) + + + + datastring += " "+str(int(endtimestamp)) + datastring += "\n" + +r = accountinglib.send_to_influx(datastring) + +print(r.text) +print(r) diff --git a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py new file mode 100644 index 00000000..138fb0ad --- /dev/null +++ b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 +import accountinglib + +import sys +import time +import datetime + + +nowtime = time.localtime() + +print("Cinder Accounting run start") +starttime=sys.argv[1] +print("Start Time = " + starttime) +endtime=sys.argv[2] +print("End Time = " + endtime) +endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') +endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) + + +results = accountinglib.get_accounting_data("manila",starttime,endtime) +datastring = '' +print(results) +for result in results: + print(result) + try: + if "rally" in result["Project"] : + department="STFC Cloud" + else: + if "default" in result["Department"]: + department=result["Project"] + else: + department=result["Department"] + + except: + department="UNKNOWN" + + datastring += "Accounting" + + datastring += ",AvailabilityZone="+result["Availability_zone"] + datastring += ",Project="+result["Project"].replace(' ','\ ') + datastring += ",Department="+department.replace(' ','\ ') + datastring += ",ManilaType="+result["ManilaType"] + datastring += ",ManilaShareType="+result["Share_type"] + datastring += ",YYYY-MM="+ endyyyymm + datastring += " Shares="+str(result["Shares"]) + datastring += ",Share_Seconds="+str(result["Share_Seconds"]) + datastring += ",ManilaGBs="+str(result["Share_GB"] * result['Share_Seconds']) + + + + datastring += " "+str(int(endtimestamp)) + datastring += "\n" + +r = accountinglib.send_to_influx(datastring) + +print(r.text) +print(r) From b7855aac6cd5580406a6f3275810e9cbc1901909 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 09:19:19 +0100 Subject: [PATCH 09/28] Add accounting for glance objects --- .../sql/glance_get_accounting_data.sql | 45 +++++++++++++++ .../local/sbin/glance_extract_accounting.py | 55 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 OpenStack-accounting/sql/glance_get_accounting_data.sql create mode 100644 OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py diff --git a/OpenStack-accounting/sql/glance_get_accounting_data.sql b/OpenStack-accounting/sql/glance_get_accounting_data.sql new file mode 100644 index 00000000..c9851c9f --- /dev/null +++ b/OpenStack-accounting/sql/glance_get_accounting_data.sql @@ -0,0 +1,45 @@ +CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) +BEGIN +SELECT + p.name AS Project, + pp.name AS Department, + COUNT(g.id) AS Images, + ip.value as GlanceType, + if(il.value like "%rbd%", "RBD" ,if(il.value like "%swift%","OBJECT","UNKNONWN")) as StorageBackend, + @VolumeSeconds:=SUM(IF(g.created_at <= starttime + AND (g.deleted_at >= endtime + OR ISNULL(g.deleted_at)), + TIMESTAMPDIFF(SECOND, + starttime, + endtime), + IF(g.created_at <= starttime + AND g.deleted_at < endtime, + TIMESTAMPDIFF(SECOND, + starttime, + g.deleted_at), + IF(g.created_at > starttime + AND (g.deleted_at >= endtime + OR ISNULL(g.deleted_at)), + TIMESTAMPDIFF(SECOND, + g.created_at, + endtime), + TIMESTAMPDIFF(SECOND, + g.created_at, + g.deleted_at))))) AS Image_Seconds, + g.size/(1024 * 1024 * 1024) AS Glance_GB +FROM + glance.images g + join + glance.image_properties ip on g.id = ip.image_id and ip.name = "image_type" + join + glance.image_locations il on g.id = il.image_id + JOIN + keystone.project p ON g.owner = p.id + JOIN + keystone.project pp ON p.parent_id = pp.id +WHERE + g.created_at <= endtime + AND (g.deleted_at >= starttime + OR g.deleted_at is null ) +GROUP BY ip.value, g.size , p.name , pp.name,if(il.value like "%rbd%", "SIRIUS" ,if(il.value like "%swift%","ECHO","UNKNONWN")); +END diff --git a/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py new file mode 100644 index 00000000..011f1b33 --- /dev/null +++ b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py @@ -0,0 +1,55 @@ +#!/usr/bin/python3 +import accountinglib + +import sys +import time +import datetime + + +nowtime = time.localtime() + +print("Glance Accounting run start") +starttime=sys.argv[1] +print("Start Time = " + starttime) +endtime=sys.argv[2] +print("End Time = " + endtime) +endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') +endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) + + +results = accountinglib.get_accounting_data("glance",starttime,endtime) +datastring = '' +print(results) +for result in results: + print(result) + try: + if "rally" in result["Project"] : + department="STFC Cloud" + else: + if "efault" in result["Department"]: + department=result["Project"] + else: + department=result["Department"] + + except: + department="UNKNOWN" + + datastring += "Accounting" + + datastring += ",Project="+result["Project"].replace(' ','\ ') + datastring += ",Department="+department.replace(' ','\ ') + datastring += ",GlanceType="+result["GlanceType"] + datastring += ",YYYY-MM="+ endyyyymm + datastring += " Images="+str(result["Images"]) + datastring += ",Image_Seconds="+str(result["Image_Seconds"]) + datastring += ",GlanceGBSeconds="+str(result["Glance_GB"] * result['Image_Seconds'] * result['Images']) + + + + datastring += " "+str(int(endtimestamp)) + datastring += "\n" + +r = accountinglib.send_to_influx(datastring) + +print(r.text) +print(r) From 447c75a52160efc70e6b89b668be83a26da65913 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 09:21:21 +0100 Subject: [PATCH 10/28] Fix issue with Volumes and Shares being counted fractionally --- .../usr/local/sbin/cinder_extract_accounting.py | 2 +- .../usr/local/sbin/manila_extract_accounting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py index 2c31cc56..0d815768 100644 --- a/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py @@ -43,7 +43,7 @@ datastring += ",YYYY-MM="+ endyyyymm datastring += " Volumes="+str(result["Volumes"]) datastring += ",Volume_Seconds="+str(result["Volume_Seconds"]) - datastring += ",CinderGBs="+str(result["Volume_GB"] * result['Volume_Seconds']) + datastring += ",CinderGBs="+str(result["Volume_GB"] * result['Volume_Seconds'] * result["Volumes"]) diff --git a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py index 138fb0ad..ddc76207 100644 --- a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py @@ -44,7 +44,7 @@ datastring += ",YYYY-MM="+ endyyyymm datastring += " Shares="+str(result["Shares"]) datastring += ",Share_Seconds="+str(result["Share_Seconds"]) - datastring += ",ManilaGBs="+str(result["Share_GB"] * result['Share_Seconds']) + datastring += ",ManilaGBs="+str(result["Share_GB"] * result['Share_Seconds'] * result["Shares"]) From 037309543b5184044232ca6ed58c6620952d0b8f Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 09:25:28 +0100 Subject: [PATCH 11/28] Tweaks and formatting for the readme --- OpenStack-accounting/readme.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/OpenStack-accounting/readme.md b/OpenStack-accounting/readme.md index 79573274..e8a170c6 100644 --- a/OpenStack-accounting/readme.md +++ b/OpenStack-accounting/readme.md @@ -1,16 +1,24 @@ -Prerequisites +# TheCount + + +## Prerequisites + The following packages are required: - python-ConfigParser - python3-requests - python36-sqlalchemy - python36-PyMySQL -Installation +A database user with read access to the relevant databases for each OpenStack component you are accounting for + +## Installation + Copy the scripts into `/usr/local/sbin` as shown in this repo Create a config file with a database connection string in the format shown in `/etc/thecount/thecount.conf.example` in `/etc/thecount/thecount.conf` Create the stored procedures in the `sql` directory appropriate db. -Use +## Use + `now-accounting.sh` generates accounting for the last 24 hours - recommend setting up a cron to run this at midnight `past-accounting.sh` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for past usage where possible `*-extract_accounting.py` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for that component From 260768cce530b4f125311864a7eb33500286e2a3 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 13:04:56 +0100 Subject: [PATCH 12/28] Remove this dependency as it is not needed with Python3 --- OpenStack-accounting/readme.md | 1 - 1 file changed, 1 deletion(-) diff --git a/OpenStack-accounting/readme.md b/OpenStack-accounting/readme.md index e8a170c6..50e5931a 100644 --- a/OpenStack-accounting/readme.md +++ b/OpenStack-accounting/readme.md @@ -4,7 +4,6 @@ ## Prerequisites The following packages are required: - - python-ConfigParser - python3-requests - python36-sqlalchemy - python36-PyMySQL From 543fc5c9c4eee78b46845fd6e9718675d0f4545a Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 13:05:22 +0100 Subject: [PATCH 13/28] Fix typo in log message --- .../usr/local/sbin/manila_extract_accounting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py index ddc76207..8cf54f97 100644 --- a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py @@ -8,7 +8,7 @@ nowtime = time.localtime() -print("Cinder Accounting run start") +print("Manila Accounting run start") starttime=sys.argv[1] print("Start Time = " + starttime) endtime=sys.argv[2] From 1664acee6048a7407c4f5a1da94ed90bcdb680fb Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 13:07:52 +0100 Subject: [PATCH 14/28] Remove unneeded definer --- OpenStack-accounting/sql/nova_get_accounting_data.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenStack-accounting/sql/nova_get_accounting_data.sql b/OpenStack-accounting/sql/nova_get_accounting_data.sql index 28b6bd62..6bbf50df 100644 --- a/OpenStack-accounting/sql/nova_get_accounting_data.sql +++ b/OpenStack-accounting/sql/nova_get_accounting_data.sql @@ -1,4 +1,4 @@ -CREATE DEFINER=`accounting-db`@`host-172-16-101-160.nubes.stfc.ac.uk` PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) +CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) BEGIN SELECT IFNULL(i.availability_zone,'nova') as AvailabilityZone, From f4b854929fc81fb7d368201429e4cee34d6d9c5b Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 13:09:42 +0100 Subject: [PATCH 15/28] Add requirements file --- OpenStack-accounting/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 OpenStack-accounting/requirements.txt diff --git a/OpenStack-accounting/requirements.txt b/OpenStack-accounting/requirements.txt new file mode 100644 index 00000000..86ff0290 --- /dev/null +++ b/OpenStack-accounting/requirements.txt @@ -0,0 +1,3 @@ +python3-requests +python36-sqlalchemy +python36-PyMySQL From 9cdc36553e3ba5e24d2050586acab5b903c2d7ae Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 13:16:01 +0100 Subject: [PATCH 16/28] Add docstrings --- OpenStack-accounting/usr/local/sbin/accountinglib.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OpenStack-accounting/usr/local/sbin/accountinglib.py b/OpenStack-accounting/usr/local/sbin/accountinglib.py index 9454c0a2..a820fcc8 100644 --- a/OpenStack-accounting/usr/local/sbin/accountinglib.py +++ b/OpenStack-accounting/usr/local/sbin/accountinglib.py @@ -11,11 +11,13 @@ import configparser def ifnull(var, val): + ```Returns the second argument if the first argument is Null/None``` if var is None: return val return var def send_to_influx(datastring): + ```Takes a datastring formatted to send to InfluxDBs rest api. Loads necessary config, sends and returns the response``` # Read from config file influx_parser = configparser.SafeConfigParser() try: @@ -35,6 +37,7 @@ def send_to_influx(datastring): return response def get_accounting_data(database,starttime,endtime): + ```Takes a db name and a start and end time as arguments. Loads db config, creates a db connection and runs a stored procedure. Returns the results of the stored procedure``` thecount_parser = configparser.RawConfigParser(strict=False) try: From bedee2caf43aabf42b0fdf91508bda24a0d6287a Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 14:12:41 +0100 Subject: [PATCH 17/28] Generate approximate concurrency data --- .../sql/nova_get_accounting_data.sql | 128 +++++++++++++++--- 1 file changed, 106 insertions(+), 22 deletions(-) diff --git a/OpenStack-accounting/sql/nova_get_accounting_data.sql b/OpenStack-accounting/sql/nova_get_accounting_data.sql index 6bbf50df..ccb414e1 100644 --- a/OpenStack-accounting/sql/nova_get_accounting_data.sql +++ b/OpenStack-accounting/sql/nova_get_accounting_data.sql @@ -1,24 +1,67 @@ CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) BEGIN +/* + Generates accounting data from the Nova database on various resources used + A temporary table is set up to run incremental checks to try and establish concurrency stats - This is dropped at the end of the proc + Some further comments below for the trickier bits + */ +create table if not exists thecount_concurrency_stats ( +start_time datetime, +end_time datetime, +project_id varchar(255), +flavor_id int , +concVMs int, +concVCPUs int, +concMem int, +concSwap int, +concRoot int, +concEph int +); +set @looptime = starttime; +while @looptime < endtime do /* loop which iterates through intervals within the time period to get approximate concurrency */ +set @loopend = DATE_ADD(@looptime, INTERVAL 10 MINUTE); +insert into thecount_concurrency_stats SELECT - IFNULL(i.availability_zone,'nova') as AvailabilityZone, - p.name as Project, - pp.name as Department, - #i.power_state as Power_State, - it.name as Flavor, - count(i.uuid) as VMs, - @VMSeconds:=SUM(IF(i.created_at <= starttime + @looptime AS start_time, + @loopend AS end_time, + i.project_id AS project_id, + i.instance_type_id AS flavor_id, + COUNT(uuid) AS concVMs, + SUM(it.vcpus) AS concVCPUs, + SUM(it.memory_mb) AS concMem, + SUM(it.swap) AS concSwap, + SUM(it.root_gb) AS concRoot, + SUM(it.ephemeral_gb) AS concEph + +FROM + nova.instances i + JOIN + nova_api.flavors it ON i.instance_type_id = it.id +WHERE + i.created_at <= @loopend + AND (i.deleted_at >= @looptime + OR ISNULL(i.deleted_at)) +GROUP BY instance_type_id , i.project_id ; +set @looptime = @loopend; +end while; +SELECT + IFNULL(i.availability_zone, 'nova') AS AvailabilityZone, + p.name AS Project, + pp.name AS Department, + it.name AS Flavor, + MAX(ts.concVMs) AS VMs, + @VMSeconds:=SUM(IF(i.created_at <= starttime /* Captures VMs which were created outside of the period deleted out of the period */ AND (i.deleted_at >= endtime OR ISNULL(i.deleted_at)), TIMESTAMPDIFF(SECOND, starttime, endtime), - IF(i.created_at <= starttime + IF(i.created_at <= starttime /* Captures VMs which were created before the period and deleted during the period */ AND i.deleted_at < endtime, TIMESTAMPDIFF(SECOND, starttime, i.deleted_at), - IF(i.created_at > starttime + IF(i.created_at > starttime /* Captures VMs which were created during the period and deleted outside the period */ AND (i.deleted_at >= endtime OR ISNULL(i.deleted_at)), TIMESTAMPDIFF(SECOND, @@ -26,29 +69,70 @@ SELECT endtime), TIMESTAMPDIFF(SECOND, i.created_at, - i.deleted_at))))) AS VM_Seconds, - #"x"+CAST(SUM(it.memory_mb * @VMSeconds) AS CHAR(100))+"z" AS Memory_MB_Seconds, + i.deleted_at))))) AS VM_Seconds, /* Generates a count of seconds VMs were running */ 'something' AS testingstuff, - it.memory_mb AS Memory_MB, - it.vcpus AS VCPU, - it.swap AS Swap, - it.root_gb AS Root_GB, - it.ephemeral_gb AS Ephemeral_GB, - ifnull((select value from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like '%per_unit_cost%' and es.key like CONCAT('%',cast(YEAR(DATE_SUB(endtime, INTERVAL 3 MONTH)) as NCHAR),'%') limit 1),0) as Per_Unit_Cost, # for January to March pull in the previous year's price so we work to financial years - ifnull((select value from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like 'accounting:unit%' ), "core") as Charge_Unit, - (select IFNULL(value,0) from nova_api.flavor_extra_specs es where flavor_id = it.id and es.key like 'accounting:gpu_num%' ) as GPU_Num + it.memory_mb AS Memory_MB, + it.vcpus AS VCPU, + it.swap AS Swap, + it.root_gb AS Root_GB, + it.ephemeral_gb AS Ephemeral_GB, + MAX(ts.concMem) AS Max_Conc_Memory_MB, + MAX(ts.concVCPUs) AS Max_Conc_VCPU, + MAX(ts.concSwap) AS Max_Conc_Swap, + MAX(ts.concRoot) AS Max_Conc_Root_GB, + MAX(ts.concEph) AS Max_Conc_Ephemeral_GB, + MIN(ts.concMem) AS Min_Conc_Memory_MB, + MIN(ts.concVCPUs) AS Min_Conc_VCPU, + MIN(ts.concSwap) AS Min_Conc_Swap, + MIN(ts.concRoot) AS Min_Conc_Root_GB, + MIN(ts.concEph) AS Min_Conc_Ephemeral_GB, + /* The below section extracts metadata to be used for accounting */ + IFNULL((SELECT + value + FROM + nova_api.flavor_extra_specs es + WHERE + flavor_id = it.id + AND es.key LIKE '%per_unit_cost%' + AND es.key LIKE CONCAT('%', + CAST(YEAR(DATE_SUB(endtime, INTERVAL 3 MONTH)) + AS NCHAR), + '%') + LIMIT 1), + 0) AS Per_Unit_Cost, + IFNULL((SELECT + value + FROM + nova_api.flavor_extra_specs es + WHERE + flavor_id = it.id + AND es.key LIKE 'accounting:unit%'), + 'core') AS Charge_Unit, + (SELECT + IFNULL(value, 0) + FROM + nova_api.flavor_extra_specs es + WHERE + flavor_id = it.id + AND es.key LIKE 'accounting:gpu_num%') AS GPU_Num FROM nova.instances i JOIN + thecount_concurrency_stats ts ON i.instance_type_id = ts.flavor_id + AND i.project_id = ts.project_id + JOIN nova_api.flavors it ON i.instance_type_id = it.id JOIN keystone.project p ON i.project_id = p.id - join - keystone.project pp on p.parent_id = pp.id + JOIN + keystone.project pp ON p.parent_id = pp.id WHERE i.created_at <= endtime AND (i.deleted_at >= starttime OR ISNULL(i.deleted_at)) -GROUP BY i.availability_zone , p.name , it.name,SUBSTRING(it.name,1,LOCATE('.',it.name)) +GROUP BY i.availability_zone , p.name , it.name , SUBSTRING(it.name, + 1, + LOCATE('.', it.name)) ; +drop table thecount_concurrency_stats; END From a8a18fe0a0d0e8e6f1d41f5797f3c2b567a8fa4d Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 14:18:16 +0100 Subject: [PATCH 18/28] Add commenting to SQL scripts --- .../sql/cinder_get_accounting_data.sql | 11 +++++++---- .../sql/glance_get_accounting_data.sql | 11 +++++++---- .../sql/manila_get_accounting_data.sql | 11 +++++++---- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/OpenStack-accounting/sql/cinder_get_accounting_data.sql b/OpenStack-accounting/sql/cinder_get_accounting_data.sql index 104376c3..04855d90 100644 --- a/OpenStack-accounting/sql/cinder_get_accounting_data.sql +++ b/OpenStack-accounting/sql/cinder_get_accounting_data.sql @@ -1,23 +1,26 @@ CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) BEGIN +/* + This procedure generates accounting data for cinder +*/ SELECT IFNULL(v.availability_zone, 'nova') AS AvailabilityZone, p.name AS Project, pp.name AS Department, COUNT(v.id) AS Volumes, "Volume" as CinderType, - @VolumeSeconds:=SUM(IF(v.created_at <= starttime + @VolumeSeconds:=SUM(IF(v.created_at <= starttime /* Captures Volumes which were created outside of the period deleted out of the period */ AND (v.deleted_at >= endtime OR ISNULL(v.deleted_at)), TIMESTAMPDIFF(SECOND, starttime, endtime), - IF(v.created_at <= starttime + IF(v.created_at <= starttime /* Captures Volumes which were created before the period and deleted during the period */ AND v.deleted_at < endtime, TIMESTAMPDIFF(SECOND, starttime, v.deleted_at), - IF(v.created_at > starttime + IF(v.created_at > starttime /* Captures Volumes which were created during the period and deleted outside the period */ AND (v.deleted_at >= endtime OR ISNULL(v.deleted_at)), TIMESTAMPDIFF(SECOND, @@ -25,7 +28,7 @@ SELECT endtime), TIMESTAMPDIFF(SECOND, v.created_at, - v.deleted_at))))) AS Volume_Seconds, + v.deleted_at))))) AS Volume_Seconds, /* Generates a count of seconds Volumes were running */ v.size AS Volume_GB FROM cinder.volumes v diff --git a/OpenStack-accounting/sql/glance_get_accounting_data.sql b/OpenStack-accounting/sql/glance_get_accounting_data.sql index c9851c9f..bfaaba2c 100644 --- a/OpenStack-accounting/sql/glance_get_accounting_data.sql +++ b/OpenStack-accounting/sql/glance_get_accounting_data.sql @@ -1,23 +1,26 @@ CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) BEGIN +/* + This procedure generates accounting data for glance +*/ SELECT p.name AS Project, pp.name AS Department, COUNT(g.id) AS Images, ip.value as GlanceType, if(il.value like "%rbd%", "RBD" ,if(il.value like "%swift%","OBJECT","UNKNONWN")) as StorageBackend, - @VolumeSeconds:=SUM(IF(g.created_at <= starttime + @VolumeSeconds:=SUM(IF(g.created_at <= starttime /* Captures Images which were created outside of the period deleted out of the period */ AND (g.deleted_at >= endtime OR ISNULL(g.deleted_at)), TIMESTAMPDIFF(SECOND, starttime, endtime), - IF(g.created_at <= starttime + IF(g.created_at <= starttime /* Captures Images which were created before the period and deleted during the period */ AND g.deleted_at < endtime, TIMESTAMPDIFF(SECOND, starttime, g.deleted_at), - IF(g.created_at > starttime + IF(g.created_at > starttime /* Captures Images which were created during the period and deleted outside the period */ AND (g.deleted_at >= endtime OR ISNULL(g.deleted_at)), TIMESTAMPDIFF(SECOND, @@ -25,7 +28,7 @@ SELECT endtime), TIMESTAMPDIFF(SECOND, g.created_at, - g.deleted_at))))) AS Image_Seconds, + g.deleted_at))))) AS Image_Seconds, /* Generates a count of seconds Images were running */ g.size/(1024 * 1024 * 1024) AS Glance_GB FROM glance.images g diff --git a/OpenStack-accounting/sql/manila_get_accounting_data.sql b/OpenStack-accounting/sql/manila_get_accounting_data.sql index 8e2c0440..0686c7c2 100644 --- a/OpenStack-accounting/sql/manila_get_accounting_data.sql +++ b/OpenStack-accounting/sql/manila_get_accounting_data.sql @@ -1,5 +1,8 @@ CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime) BEGIN +/* + This procedure generates accounting data for manila +*/ SELECT p.name AS Project, pp.name AS Department, @@ -7,18 +10,18 @@ SELECT st.name AS Share_type, COUNT(m.id) AS Shares, "Share" as ManilaType, - @ShareSeconds:=SUM(IF(m.created_at <= starttime + @ShareSeconds:=SUM(IF(m.created_at <= starttime /* Captures Shares which were created outside of the period deleted out of the period */ AND (m.deleted_at >= endtime OR ISNULL(m.deleted_at)), TIMESTAMPDIFF(SECOND, starttime, endtime), - IF(m.created_at <= starttime + IF(m.created_at <= starttime /* Captures Shares which were created before the period and deleted during the period */ AND m.deleted_at < endtime, TIMESTAMPDIFF(SECOND, starttime, m.deleted_at), - IF(m.created_at > starttime + IF(m.created_at > starttime /* Captures Shares which were created during the period and deleted outside the period */ AND (m.deleted_at >= endtime OR ISNULL(m.deleted_at)), TIMESTAMPDIFF(SECOND, @@ -26,7 +29,7 @@ SELECT endtime), TIMESTAMPDIFF(SECOND, m.created_at, - m.deleted_at))))) AS Share_Seconds, + m.deleted_at))))) AS Share_Seconds, /* Generates a count of seconds Shares were running */ m.size AS Share_GB FROM manila.shares m From cf15b7406efaa1e283d0f095bd4a472029af764b Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 15:29:56 +0100 Subject: [PATCH 19/28] Abandon calculating concurrency for now --- .../sql/nova_get_accounting_data.sql | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/OpenStack-accounting/sql/nova_get_accounting_data.sql b/OpenStack-accounting/sql/nova_get_accounting_data.sql index ccb414e1..31d98b08 100644 --- a/OpenStack-accounting/sql/nova_get_accounting_data.sql +++ b/OpenStack-accounting/sql/nova_get_accounting_data.sql @@ -2,54 +2,13 @@ CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetim BEGIN /* Generates accounting data from the Nova database on various resources used - A temporary table is set up to run incremental checks to try and establish concurrency stats - This is dropped at the end of the proc Some further comments below for the trickier bits */ -create table if not exists thecount_concurrency_stats ( -start_time datetime, -end_time datetime, -project_id varchar(255), -flavor_id int , -concVMs int, -concVCPUs int, -concMem int, -concSwap int, -concRoot int, -concEph int -); -set @looptime = starttime; -while @looptime < endtime do /* loop which iterates through intervals within the time period to get approximate concurrency */ -set @loopend = DATE_ADD(@looptime, INTERVAL 10 MINUTE); -insert into thecount_concurrency_stats -SELECT - @looptime AS start_time, - @loopend AS end_time, - i.project_id AS project_id, - i.instance_type_id AS flavor_id, - COUNT(uuid) AS concVMs, - SUM(it.vcpus) AS concVCPUs, - SUM(it.memory_mb) AS concMem, - SUM(it.swap) AS concSwap, - SUM(it.root_gb) AS concRoot, - SUM(it.ephemeral_gb) AS concEph - -FROM - nova.instances i - JOIN - nova_api.flavors it ON i.instance_type_id = it.id -WHERE - i.created_at <= @loopend - AND (i.deleted_at >= @looptime - OR ISNULL(i.deleted_at)) -GROUP BY instance_type_id , i.project_id ; -set @looptime = @loopend; -end while; SELECT IFNULL(i.availability_zone, 'nova') AS AvailabilityZone, p.name AS Project, pp.name AS Department, it.name AS Flavor, - MAX(ts.concVMs) AS VMs, @VMSeconds:=SUM(IF(i.created_at <= starttime /* Captures VMs which were created outside of the period deleted out of the period */ AND (i.deleted_at >= endtime OR ISNULL(i.deleted_at)), @@ -76,16 +35,6 @@ SELECT it.swap AS Swap, it.root_gb AS Root_GB, it.ephemeral_gb AS Ephemeral_GB, - MAX(ts.concMem) AS Max_Conc_Memory_MB, - MAX(ts.concVCPUs) AS Max_Conc_VCPU, - MAX(ts.concSwap) AS Max_Conc_Swap, - MAX(ts.concRoot) AS Max_Conc_Root_GB, - MAX(ts.concEph) AS Max_Conc_Ephemeral_GB, - MIN(ts.concMem) AS Min_Conc_Memory_MB, - MIN(ts.concVCPUs) AS Min_Conc_VCPU, - MIN(ts.concSwap) AS Min_Conc_Swap, - MIN(ts.concRoot) AS Min_Conc_Root_GB, - MIN(ts.concEph) AS Min_Conc_Ephemeral_GB, /* The below section extracts metadata to be used for accounting */ IFNULL((SELECT value @@ -118,9 +67,6 @@ SELECT FROM nova.instances i JOIN - thecount_concurrency_stats ts ON i.instance_type_id = ts.flavor_id - AND i.project_id = ts.project_id - JOIN nova_api.flavors it ON i.instance_type_id = it.id JOIN keystone.project p ON i.project_id = p.id @@ -134,5 +80,4 @@ GROUP BY i.availability_zone , p.name , it.name , SUBSTRING(it.name, 1, LOCATE('.', it.name)) ; -drop table thecount_concurrency_stats; END From 1454f8d5996b6e50b41cd9465fd0b9786b377ae7 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 15:53:02 +0100 Subject: [PATCH 20/28] Add back in field lost while trying to get concurrency working --- OpenStack-accounting/sql/nova_get_accounting_data.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/OpenStack-accounting/sql/nova_get_accounting_data.sql b/OpenStack-accounting/sql/nova_get_accounting_data.sql index 31d98b08..29339708 100644 --- a/OpenStack-accounting/sql/nova_get_accounting_data.sql +++ b/OpenStack-accounting/sql/nova_get_accounting_data.sql @@ -9,6 +9,7 @@ SELECT p.name AS Project, pp.name AS Department, it.name AS Flavor, + COUNT(i.uuid) as VMs, @VMSeconds:=SUM(IF(i.created_at <= starttime /* Captures VMs which were created outside of the period deleted out of the period */ AND (i.deleted_at >= endtime OR ISNULL(i.deleted_at)), From 1e1aeed549c235e86c289fe944b97a8e6db2c8ec Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 16:37:55 +0100 Subject: [PATCH 21/28] Anish-ifying - addressing Anish's comments --- .../usr/local/sbin/accountinglib.py | 82 ++++++++----- .../local/sbin/cinder_extract_accounting.py | 70 ++++++----- .../local/sbin/glance_extract_accounting.py | 69 +++++------ .../local/sbin/manila_extract_accounting.py | 72 ++++++------ .../usr/local/sbin/nova_extract_accounting.py | 111 +++++++++--------- 5 files changed, 207 insertions(+), 197 deletions(-) diff --git a/OpenStack-accounting/usr/local/sbin/accountinglib.py b/OpenStack-accounting/usr/local/sbin/accountinglib.py index a820fcc8..41e94e89 100644 --- a/OpenStack-accounting/usr/local/sbin/accountinglib.py +++ b/OpenStack-accounting/usr/local/sbin/accountinglib.py @@ -6,52 +6,80 @@ import requests import sys import sqlalchemy +import logging from sqlalchemy.sql import select from sqlalchemy.orm import sessionmaker import configparser +def get_logger(component): + logging.basicConfig(filename="/var/log/thecount.log", + format=f'%(asctime)s {component} %(message)s', + filemode='a') + logger = logging.getLogger() + logger.setLevel(logging.INFO) + return logger + def ifnull(var, val): - ```Returns the second argument if the first argument is Null/None``` + '''Returns the second argument if the first argument is Null/None''' if var is None: return val return var -def send_to_influx(datastring): - ```Takes a datastring formatted to send to InfluxDBs rest api. Loads necessary config, sends and returns the response``` - # Read from config file - influx_parser = configparser.SafeConfigParser() - try: - influx_parser.read('/etc/influxdb.conf') - host = influx_parser.get('db', 'host') - database = influx_parser.get('db', 'database') - username = influx_parser.get('auth', 'username') - password = influx_parser.get('auth', 'password') - instance = influx_parser.get('cloud','instance') - except: - print('Unable to read from influx config file') - sys.exit(1) - finaldatastring = datastring.replace("Accounting,","Accounting,instance="+instance+",") - print(finaldatastring) - url = 'http://'+host+'/write?db='+database +'&precision=s' - response = requests.post(url,data=finaldatastring,auth=(username,password)) - return response - -def get_accounting_data(database,starttime,endtime): - ```Takes a db name and a start and end time as arguments. Loads db config, creates a db connection and runs a stored procedure. Returns the results of the stored procedure``` +def project_to_department(result): + '''Returns an appropriate department for a project''' + if "rally" in result["Project"]: + department = "STFC Cloud" + elif "efault" in result["Department"]: + department = result["Project"] + else: + department = result["Department"] + return department + +def send_to_influx(datastring, logger): + '''Takes a datastring formatted to send to InfluxDBs rest api. Loads necessary config, sends and returns the response''' + # Read from config file + influx_parser = configparser.SafeConfigParser() + try: + influx_parser.read('/etc/influxdb.conf') + except Exceptions as exp: + logger.info(f'Unable to read from influx config file - {str(exp)}') + sys.exit(1) + try: + host = influx_parser.get('db', 'host') + database = influx_parser.get('db', 'database') + username = influx_parser.get('auth', 'username') + password = influx_parser.get('auth', 'password') + instance = influx_parser.get('cloud','instance') + except Exceptions as exp: + logger.info(f'Unable to parse influx config file - {str(exp)}') + sys.exit(1) + finaldatastring = datastring.replace("Accounting,","Accounting,instance="+instance+",") + logger.info(finaldatastring) + url = f'http://{host}/write?db={database}&precision=s' + response = requests.post(url,data=finaldatastring,auth=(username,password)) + return response + +def get_accounting_data(database,starttime,endtime, logger): + '''Takes a db name and a start and end time as arguments. Loads db config, creates a db connection and runs a stored procedure. Returns the results of the stored procedure''' thecount_parser = configparser.RawConfigParser(strict=False) try: thecount_parser.read('/etc/thecount/thecount.conf') + except Exceptions as exp: + logger.info(f'Unable to read from thecount config file - {str(exp)}') + sys.exit(1) + + try: connectionstring = thecount_parser.get('database','connection') + '/' + database - except: - print('Unable to read from thecount config file') + except Exceptions as exp: + logger.info(f'Unable to parse thecount config file - {str(exp)}') sys.exit(1) engine = sqlalchemy.create_engine(connectionstring, encoding='utf-8') connection = engine.connect() sess = sessionmaker(bind=engine)() - query = 'call get_accounting_data( "' + starttime +'","' + endtime + '")' + query = f'call get_accounting_data( "{starttime}","{endtime}")' - print(query) + logger.info(query) results = sess.execute(query, { 'p1': starttime, 'p2': endtime }) return results diff --git a/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py index 0d815768..9dcf9d92 100644 --- a/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py @@ -4,53 +4,49 @@ import sys import time import datetime +import logging +def main(): + nowtime = time.localtime() + logger = accountinglib.get_logger("cinder") -nowtime = time.localtime() + logger.info("Cinder Accounting run start") + starttime=sys.argv[1] + logger.info("Start Time = " + starttime) + endtime=sys.argv[2] + logger.info("End Time = " + endtime) + endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') + endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) -print("Cinder Accounting run start") -starttime=sys.argv[1] -print("Start Time = " + starttime) -endtime=sys.argv[2] -print("End Time = " + endtime) -endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') -endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) + results = accountinglib.get_accounting_data("cinder", starttime, endtime, logger) + datastring = '' + logger.info(results) + for result in results: + logger.info(result) + department = accountinglib.project_to_department(result) -results = accountinglib.get_accounting_data("cinder",starttime,endtime) -datastring = '' -print(results) -for result in results: - print(result) - try: - if "rally" in result["Project"] : - department="STFC Cloud" - else: - if "default" in result["Department"]: - department=result["Project"] - else: - department=result["Department"] - except: - department="UNKNOWN" + datastring += "Accounting" - datastring += "Accounting" + datastring += ",AvailabilityZone="+result["AvailabilityZone"] + datastring += ",Project="+result["Project"].replace(' ','\ ') + datastring += ",Department="+department.replace(' ','\ ') + datastring += ",CinderType="+result["CinderType"] + datastring += ",YYYY-MM="+ endyyyymm + datastring += " Volumes="+str(result["Volumes"]) + datastring += ",Volume_Seconds="+str(result["Volume_Seconds"]) + datastring += ",CinderGBs="+str(result["Volume_GB"] * result['Volume_Seconds'] * result["Volumes"]) - datastring += ",AvailabilityZone="+result["AvailabilityZone"] - datastring += ",Project="+result["Project"].replace(' ','\ ') - datastring += ",Department="+department.replace(' ','\ ') - datastring += ",CinderType="+result["CinderType"] - datastring += ",YYYY-MM="+ endyyyymm - datastring += " Volumes="+str(result["Volumes"]) - datastring += ",Volume_Seconds="+str(result["Volume_Seconds"]) - datastring += ",CinderGBs="+str(result["Volume_GB"] * result['Volume_Seconds'] * result["Volumes"]) + datastring += " "+str(int(endtimestamp)) + datastring += "\n" - datastring += " "+str(int(endtimestamp)) - datastring += "\n" + r = accountinglib.send_to_influx(datastring, logger) -r = accountinglib.send_to_influx(datastring) + logger.info(r.text) + logger.info(r) -print(r.text) -print(r) +if __name__ == "__main__": + main() diff --git a/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py index 011f1b33..3f4bff02 100644 --- a/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py @@ -4,52 +4,47 @@ import sys import time import datetime +import logging +def main(): + nowtime = time.localtime() + logger = accountinglib.get_logger("glance") -nowtime = time.localtime() + logger.info("Glance Accounting run start") + starttime=sys.argv[1] + logger.info("Start Time = " + starttime) + endtime=sys.argv[2] + logger.info("End Time = " + endtime) + endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') + endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) -print("Glance Accounting run start") -starttime=sys.argv[1] -print("Start Time = " + starttime) -endtime=sys.argv[2] -print("End Time = " + endtime) -endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') -endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) + results = accountinglib.get_accounting_data("glance", starttime, endtime, logger) + datastring = '' + logger.info(results) + for result in results: + logger.info(result) + department = accountinglib.project_to_department(result) -results = accountinglib.get_accounting_data("glance",starttime,endtime) -datastring = '' -print(results) -for result in results: - print(result) - try: - if "rally" in result["Project"] : - department="STFC Cloud" - else: - if "efault" in result["Department"]: - department=result["Project"] - else: - department=result["Department"] + datastring += "Accounting" - except: - department="UNKNOWN" + datastring += ",Project="+result["Project"].replace(' ','\ ') + datastring += ",Department="+department.replace(' ','\ ') + datastring += ",GlanceType="+result["GlanceType"] + datastring += ",YYYY-MM="+ endyyyymm + datastring += " Images="+str(result["Images"]) + datastring += ",Image_Seconds="+str(result["Image_Seconds"]) + datastring += ",GlanceGBSeconds="+str(result["Glance_GB"] * result['Image_Seconds'] * result['Images']) - datastring += "Accounting" - datastring += ",Project="+result["Project"].replace(' ','\ ') - datastring += ",Department="+department.replace(' ','\ ') - datastring += ",GlanceType="+result["GlanceType"] - datastring += ",YYYY-MM="+ endyyyymm - datastring += " Images="+str(result["Images"]) - datastring += ",Image_Seconds="+str(result["Image_Seconds"]) - datastring += ",GlanceGBSeconds="+str(result["Glance_GB"] * result['Image_Seconds'] * result['Images']) + datastring += " "+str(int(endtimestamp)) + datastring += "\n" + r = accountinglib.send_to_influx(datastring, logger) - datastring += " "+str(int(endtimestamp)) - datastring += "\n" + logger.info(r.text) + logger.info(r) -r = accountinglib.send_to_influx(datastring) - -print(r.text) -print(r) +if __name__ == "__main__": + main() diff --git a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py index 8cf54f97..531b58ca 100644 --- a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py @@ -4,54 +4,50 @@ import sys import time import datetime +import logging +def main(): + nowtime = time.localtime() + logger = accountinglib.get_logger("manila") -nowtime = time.localtime() + logger.info("Manila Accounting run start") + starttime=sys.argv[1] + logger.info("Start Time = " + starttime) + endtime=sys.argv[2] + logger.info("End Time = " + endtime) + endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') + endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) -print("Manila Accounting run start") -starttime=sys.argv[1] -print("Start Time = " + starttime) -endtime=sys.argv[2] -print("End Time = " + endtime) -endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') -endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) + results = accountinglib.get_accounting_data("manila", starttime, endtime, logger) + datastring = '' + logger.info(results) + for result in results: + logger.info(result) + department = accountinglib.project_to_department(result) -results = accountinglib.get_accounting_data("manila",starttime,endtime) -datastring = '' -print(results) -for result in results: - print(result) - try: - if "rally" in result["Project"] : - department="STFC Cloud" - else: - if "default" in result["Department"]: - department=result["Project"] - else: - department=result["Department"] + datastring += "Accounting" - except: - department="UNKNOWN" + datastring += ",AvailabilityZone="+result["Availability_zone"] + datastring += ",Project="+result["Project"].replace(' ','\ ') + datastring += ",Department="+department.replace(' ','\ ') + datastring += ",ManilaType="+result["ManilaType"] + datastring += ",ManilaShareType="+result["Share_type"] + datastring += ",YYYY-MM="+ endyyyymm + datastring += " Shares="+str(result["Shares"]) + datastring += ",Share_Seconds="+str(result["Share_Seconds"]) + datastring += ",ManilaGBs="+str(result["Share_GB"] * result['Share_Seconds'] * result["Shares"]) - datastring += "Accounting" - datastring += ",AvailabilityZone="+result["Availability_zone"] - datastring += ",Project="+result["Project"].replace(' ','\ ') - datastring += ",Department="+department.replace(' ','\ ') - datastring += ",ManilaType="+result["ManilaType"] - datastring += ",ManilaShareType="+result["Share_type"] - datastring += ",YYYY-MM="+ endyyyymm - datastring += " Shares="+str(result["Shares"]) - datastring += ",Share_Seconds="+str(result["Share_Seconds"]) - datastring += ",ManilaGBs="+str(result["Share_GB"] * result['Share_Seconds'] * result["Shares"]) + datastring += " "+str(int(endtimestamp)) + datastring += "\n" + r = accountinglib.send_to_influx(datastring, logger) - datastring += " "+str(int(endtimestamp)) - datastring += "\n" + logger.info(r.text) + logger.info(r) -r = accountinglib.send_to_influx(datastring) -print(r.text) -print(r) +if __name__ == "__main__": + main() diff --git a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py index 2a4e1c9d..14e05eec 100644 --- a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py @@ -4,74 +4,69 @@ import sys import time import datetime +import logging +def main(): + nowtime = time.localtime() + logger = accountinglib.get_logger("nova") -nowtime = time.localtime() + logger.info("Nova Accounting run start") + starttime=sys.argv[1] + logger.info("Start Time = " + starttime) + endtime=sys.argv[2] + logger.info("End Time = " + endtime) + endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') + endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) -print("Nova Accounting run start") -starttime=sys.argv[1] -print("Start Time = " + starttime) -endtime=sys.argv[2] -print("End Time = " + endtime) -endyyyymm=datetime.datetime.strptime(endtime,"%Y-%m-%d %H:%M").strftime('%Y-%m') -endtimestamp = time.mktime(datetime.datetime.strptime(endtime, "%Y-%m-%d %H:%M").timetuple()) + results = accountinglib.get_accounting_data("nova", starttime, endtime, logger) + datastring = '' + for result in results: + department = accountinglib.project_to_department(result) -results = accountinglib.get_accounting_data("nova",starttime,endtime) -datastring = '' -for result in results: - try: - if "rally" in result["Project"] : - department="STFC Cloud" - else: - if "default" in result["Department"]: - department=result["Project"] - else: - department=result["Department"] - - except: - department="UNKNOWN" + instancetype=result['Charge_Unit'] - instancetype=result['Charge_Unit'] + datastring += "Accounting" - datastring += "Accounting" + datastring += ",AvailabilityZone="+result["AvailabilityZone"] + datastring += ",Project="+result["Project"].replace(' ','\ ') + datastring += ",Department="+department.replace(' ','\ ') + datastring += ",Flavor="+result["Flavor"].replace('.','_') + datastring += ",FlavorPrefix="+result["Flavor"].split('.')[0] + datastring += ",InstanceType="+instancetype + datastring += ",YYYY-MM="+ endyyyymm + datastring += ",Charge_Unit="+ result["Charge_Unit"] + datastring += " VMs="+str(result["VMs"]) + datastring += ",VM_Seconds="+str(result["VM_Seconds"]) + datastring += ",Memory_MB_Seconds="+str(result["Memory_MB"] * result['VM_Seconds']) + datastring += ",Memory_MBs="+str(result["Memory_MB"] * result["VMs"]) + datastring += ",VCPU_Seconds="+str(result["VCPU"] * result['VM_Seconds']) + datastring += ",VCPUs="+str(result["VCPU"] * result["VMs"]) + datastring += ",Swap_Seconds="+str(result["Swap"] * result['VM_Seconds']) + datastring += ",Swaps="+str(result["Swap"] * result["VMs"]) + datastring += ",Root_GB_Seconds="+str(result["Root_GB"] * result['VM_Seconds']) + datastring += ",Root_GBs="+str(result["Root_GB"] * result["VMs"]) + datastring += ",Ephemeral_GB_Seconds="+str(result["Ephemeral_GB"] * result['VM_Seconds']) + datastring += ",Ephemeral_GBs="+str(result["Ephemeral_GB"] * result["VMs"]) + if int( accountinglib.ifnull(result["GPU_Num"],0)) > 0: + datastring += ",GPU_Seconds="+str(float(result["GPU_Num"]) * float(result['VM_Seconds'])) + datastring += ",GPUs=" + str(float(result["GPU_Num"]) * float(result['VMs'])) + logger.info(result) + datastring += ",COST=" + str(float(result["GPU_Num"]) * float(result['VM_Seconds']) * float(result["Per_Unit_Cost"]) / float(3600)) + else: + datastring += ",COST=" + str(float(result["VCPU"]) * float(result['VM_Seconds']) * float(result["Per_Unit_Cost"]) / float(3600)) + datastring += ",GPUs=" + str(0) + datastring += ",GPU_Seconds=" + str(0) - datastring += ",AvailabilityZone="+result["AvailabilityZone"] - datastring += ",Project="+result["Project"].replace(' ','\ ') - datastring += ",Department="+department.replace(' ','\ ') - datastring += ",Flavor="+result["Flavor"].replace('.','_') - datastring += ",FlavorPrefix="+result["Flavor"].split('.')[0] - datastring += ",InstanceType="+instancetype - datastring += ",YYYY-MM="+ endyyyymm - datastring += ",Charge_Unit="+ result["Charge_Unit"] - datastring += " VMs="+str(result["VMs"]) - datastring += ",VM_Seconds="+str(result["VM_Seconds"]) - datastring += ",Memory_MB_Seconds="+str(result["Memory_MB"] * result['VM_Seconds']) - datastring += ",Memory_MBs="+str(result["Memory_MB"] * result["VMs"]) - datastring += ",VCPU_Seconds="+str(result["VCPU"] * result['VM_Seconds']) - datastring += ",VCPUs="+str(result["VCPU"] * result["VMs"]) - datastring += ",Swap_Seconds="+str(result["Swap"] * result['VM_Seconds']) - datastring += ",Swaps="+str(result["Swap"] * result["VMs"]) - datastring += ",Root_GB_Seconds="+str(result["Root_GB"] * result['VM_Seconds']) - datastring += ",Root_GBs="+str(result["Root_GB"] * result["VMs"]) - datastring += ",Ephemeral_GB_Seconds="+str(result["Ephemeral_GB"] * result['VM_Seconds']) - datastring += ",Ephemeral_GBs="+str(result["Ephemeral_GB"] * result["VMs"]) - if int( accountinglib.ifnull(result["GPU_Num"],0)) > 0: - datastring += ",GPU_Seconds="+str(float(result["GPU_Num"]) * float(result['VM_Seconds'])) - datastring += ",GPUs=" + str(float(result["GPU_Num"]) * float(result['VMs'])) - print(result) - datastring += ",COST=" + str(float(result["GPU_Num"]) * float(result['VM_Seconds']) * float(result["Per_Unit_Cost"]) / float(3600)) - else: - datastring += ",COST=" + str(float(result["VCPU"]) * float(result['VM_Seconds']) * float(result["Per_Unit_Cost"]) / float(3600)) - datastring += ",GPUs=" + str(0) - datastring += ",GPU_Seconds=" + str(0) + datastring += " "+str(int(endtimestamp)) + datastring += "\n" - datastring += " "+str(int(endtimestamp)) - datastring += "\n" + r = accountinglib.send_to_influx(datastring, logger) -r = accountinglib.send_to_influx(datastring) + logger.info(r.text) + logger.info(r) -print(r.text) -print(r) +if __name__ == "__main__": + main() From 3fd0eef052fe5629d6ac2a27822c930ea5dbc9ed Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Thu, 5 Oct 2023 16:56:15 +0100 Subject: [PATCH 22/28] Fix past accounting script --- OpenStack-accounting/usr/local/sbin/past-accounting.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenStack-accounting/usr/local/sbin/past-accounting.sh b/OpenStack-accounting/usr/local/sbin/past-accounting.sh index 458410cb..dc1b3569 100644 --- a/OpenStack-accounting/usr/local/sbin/past-accounting.sh +++ b/OpenStack-accounting/usr/local/sbin/past-accounting.sh @@ -9,9 +9,9 @@ startdateepoch=`date -d "$startdate" +%s` endtime=$startdate starttime=`date -d "$startdate" +"%Y-%m-%d %H:%M"` +endtimeepoch=`date -d "$endtime" +%s` - -while [ $endtimeepoch -lt $enddateepoch ]; +while [][ $endtimeepoch -lt $enddateepoch ]]; do starttime=`date -d "$starttime" +"%Y-%m-%d %H:%M"` starttimeepoch=`date -d "$starttime" +%s` From a3f278ffbb1218e731089495ad6692d3eb852ab0 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Mon, 23 Oct 2023 08:34:25 +0100 Subject: [PATCH 23/28] More Docstrings --- OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py | 1 + OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py | 1 + OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py | 1 + OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py | 1 + 4 files changed, 4 insertions(+) diff --git a/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py index 9dcf9d92..61d94c37 100644 --- a/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/cinder_extract_accounting.py @@ -7,6 +7,7 @@ import logging def main(): + '''Processes accounting from Cinder''' nowtime = time.localtime() logger = accountinglib.get_logger("cinder") diff --git a/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py index 3f4bff02..5cedd4ff 100644 --- a/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py @@ -7,6 +7,7 @@ import logging def main(): + '''Processes accounting from Glance''' nowtime = time.localtime() logger = accountinglib.get_logger("glance") diff --git a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py index 531b58ca..df4752c1 100644 --- a/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/manila_extract_accounting.py @@ -7,6 +7,7 @@ import logging def main(): + '''Processes accounting from Manila''' nowtime = time.localtime() logger = accountinglib.get_logger("manila") diff --git a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py index 14e05eec..6e535151 100644 --- a/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/nova_extract_accounting.py @@ -7,6 +7,7 @@ import logging def main(): + '''Processes accounting from Nova''' nowtime = time.localtime() logger = accountinglib.get_logger("nova") From 14d5eb4eb2558a40e1e98004e383dae2144a2be3 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Mon, 23 Oct 2023 08:42:33 +0100 Subject: [PATCH 24/28] Deal with archived instances in the SQL --- .../sql/nova_get_accounting_data.sql | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/OpenStack-accounting/sql/nova_get_accounting_data.sql b/OpenStack-accounting/sql/nova_get_accounting_data.sql index 29339708..d655d295 100644 --- a/OpenStack-accounting/sql/nova_get_accounting_data.sql +++ b/OpenStack-accounting/sql/nova_get_accounting_data.sql @@ -80,5 +80,82 @@ WHERE GROUP BY i.availability_zone , p.name , it.name , SUBSTRING(it.name, 1, LOCATE('.', it.name)) + UNION + SELECT + IFNULL(i.availability_zone, 'nova') AS AvailabilityZone, + p.name AS Project, + pp.name AS Department, + it.name AS Flavor, + COUNT(i.uuid) as VMs, + @VMSeconds:=SUM(IF(i.created_at <= starttime /* Captures VMs which were created outside of the period deleted out of the period */ + AND (i.deleted_at >= endtime + OR ISNULL(i.deleted_at)), + TIMESTAMPDIFF(SECOND, + starttime, + endtime), + IF(i.created_at <= starttime /* Captures VMs which were created before the period and deleted during the period */ + AND i.deleted_at < endtime, + TIMESTAMPDIFF(SECOND, + starttime, + i.deleted_at), + IF(i.created_at > starttime /* Captures VMs which were created during the period and deleted outside the period */ + AND (i.deleted_at >= endtime + OR ISNULL(i.deleted_at)), + TIMESTAMPDIFF(SECOND, + i.created_at, + endtime), + TIMESTAMPDIFF(SECOND, + i.created_at, + i.deleted_at))))) AS VM_Seconds, /* Generates a count of seconds VMs were running */ + 'something' AS testingstuff, + it.memory_mb AS Memory_MB, + it.vcpus AS VCPU, + it.swap AS Swap, + it.root_gb AS Root_GB, + it.ephemeral_gb AS Ephemeral_GB, + /* The below section extracts metadata to be used for accounting */ + IFNULL((SELECT + value + FROM + nova_api.flavor_extra_specs es + WHERE + flavor_id = it.id + AND es.key LIKE '%per_unit_cost%' + AND es.key LIKE CONCAT('%', + CAST(YEAR(DATE_SUB(endtime, INTERVAL 3 MONTH)) + AS NCHAR), + '%') + LIMIT 1), + 0) AS Per_Unit_Cost, + IFNULL((SELECT + value + FROM + nova_api.flavor_extra_specs es + WHERE + flavor_id = it.id + AND es.key LIKE 'accounting:unit%'), + 'core') AS Charge_Unit, + (SELECT + IFNULL(value, 0) + FROM + nova_api.flavor_extra_specs es + WHERE + flavor_id = it.id + AND es.key LIKE 'accounting:gpu_num%') AS GPU_Num + FROM + nova.shadow_instances i + JOIN + nova_api.flavors it ON i.instance_type_id = it.id + JOIN + keystone.project p ON i.project_id = p.id + JOIN + keystone.project pp ON p.parent_id = pp.id + WHERE + i.created_at <= endtime + AND (i.deleted_at >= starttime + OR ISNULL(i.deleted_at)) + GROUP BY i.availability_zone , p.name , it.name , SUBSTRING(it.name, + 1, + LOCATE('.', it.name)) ; END From be40495b441a75d26f9903473e9602641654520d Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Mon, 23 Oct 2023 09:09:33 +0100 Subject: [PATCH 25/28] Remove debugging line --- OpenStack-accounting/sql/nova_get_accounting_data.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/OpenStack-accounting/sql/nova_get_accounting_data.sql b/OpenStack-accounting/sql/nova_get_accounting_data.sql index d655d295..6d9ba972 100644 --- a/OpenStack-accounting/sql/nova_get_accounting_data.sql +++ b/OpenStack-accounting/sql/nova_get_accounting_data.sql @@ -30,7 +30,6 @@ SELECT TIMESTAMPDIFF(SECOND, i.created_at, i.deleted_at))))) AS VM_Seconds, /* Generates a count of seconds VMs were running */ - 'something' AS testingstuff, it.memory_mb AS Memory_MB, it.vcpus AS VCPU, it.swap AS Swap, @@ -107,7 +106,6 @@ GROUP BY i.availability_zone , p.name , it.name , SUBSTRING(it.name, TIMESTAMPDIFF(SECOND, i.created_at, i.deleted_at))))) AS VM_Seconds, /* Generates a count of seconds VMs were running */ - 'something' AS testingstuff, it.memory_mb AS Memory_MB, it.vcpus AS VCPU, it.swap AS Swap, From 3fdb8c4b1937f03d85437e7992ed093f8bb2d1fd Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Mon, 23 Oct 2023 10:37:15 +0100 Subject: [PATCH 26/28] Add in storage backend --- OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py index 5cedd4ff..83ac2102 100644 --- a/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py +++ b/OpenStack-accounting/usr/local/sbin/glance_extract_accounting.py @@ -31,6 +31,7 @@ def main(): datastring += ",Project="+result["Project"].replace(' ','\ ') datastring += ",Department="+department.replace(' ','\ ') + datastring += ",StorageBackend="+result["StorageBackend"] datastring += ",GlanceType="+result["GlanceType"] datastring += ",YYYY-MM="+ endyyyymm datastring += " Images="+str(result["Images"]) From ce05bf05d0603d887480e43a7b81d9fa42dbc964 Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Mon, 23 Oct 2023 10:39:23 +0100 Subject: [PATCH 27/28] Fix variable name --- OpenStack-accounting/sql/glance_get_accounting_data.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenStack-accounting/sql/glance_get_accounting_data.sql b/OpenStack-accounting/sql/glance_get_accounting_data.sql index bfaaba2c..4913ed4e 100644 --- a/OpenStack-accounting/sql/glance_get_accounting_data.sql +++ b/OpenStack-accounting/sql/glance_get_accounting_data.sql @@ -9,7 +9,7 @@ SELECT COUNT(g.id) AS Images, ip.value as GlanceType, if(il.value like "%rbd%", "RBD" ,if(il.value like "%swift%","OBJECT","UNKNONWN")) as StorageBackend, - @VolumeSeconds:=SUM(IF(g.created_at <= starttime /* Captures Images which were created outside of the period deleted out of the period */ + @ImageSeconds:=SUM(IF(g.created_at <= starttime /* Captures Images which were created outside of the period deleted out of the period */ AND (g.deleted_at >= endtime OR ISNULL(g.deleted_at)), TIMESTAMPDIFF(SECOND, From d0be5372b37de0068b7766ffbd771201c807a10e Mon Sep 17 00:00:00 2001 From: "apdibbo@clearskies.it" Date: Wed, 1 Nov 2023 08:28:40 +0000 Subject: [PATCH 28/28] Fix atomism --- OpenStack-accounting/usr/local/sbin/past-accounting.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenStack-accounting/usr/local/sbin/past-accounting.sh b/OpenStack-accounting/usr/local/sbin/past-accounting.sh index dc1b3569..64e62b3c 100644 --- a/OpenStack-accounting/usr/local/sbin/past-accounting.sh +++ b/OpenStack-accounting/usr/local/sbin/past-accounting.sh @@ -11,7 +11,7 @@ endtime=$startdate starttime=`date -d "$startdate" +"%Y-%m-%d %H:%M"` endtimeepoch=`date -d "$endtime" +%s` -while [][ $endtimeepoch -lt $enddateepoch ]]; +while [[ $endtimeepoch -lt $enddateepoch ]]; do starttime=`date -d "$starttime" +"%Y-%m-%d %H:%M"` starttimeepoch=`date -d "$starttime" +%s`