Skip to content

Commit

Permalink
updates to constants code and display
Browse files Browse the repository at this point in the history
  • Loading branch information
@stuchalk
stuchalk committed Jun 20, 2024
1 parent 707c8ad commit 9cf7813
Show file tree
Hide file tree
Showing 6 changed files with 133 additions and 39 deletions.
84 changes: 66 additions & 18 deletions constants/ingest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,24 @@
import django
import csv
import re
import json

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "umisconfig.settings")
django.setup()

from constants.models import *
from django.db.models import Count
from umisconfig.settings import STATIC_URL, BASE_DIR
from decimal import Decimal
from os.path import exists
from units.functions import *
from math import *


def procvalue(val, proc):
proc.orig_value = val
valuestr = val.replace(' ', '')
valuestr = valuestr.replace('…', '')
valuestr = valuestr.replace('...', '')
proc.value_str = valuestr
if val.find('e') != -1:
valueacc = valuestr.index('e') - 1
Expand All @@ -38,7 +41,8 @@ def procvalue(val, proc):


def procuncert(unc, proc):
if unc == 'null':
# calculate uncertainties and add to proc
if unc == 'null' or unc == '(exact)':
proc.uncert_str = None
proc.uncert_num = None
proc.uncert_man = None
Expand All @@ -52,34 +56,53 @@ def procuncert(unc, proc):
tzero = '0'
else:
tzero = ''
floatval = float(uncertstr)
floatstr = str(floatval)
if '(exact)' in uncertstr:
floatstr = None
else:
floatval = float(uncertstr)
floatstr = str(floatval)
if tzero == '0':
uncertnum = floatstr.replace('e', '.0e')
else:
uncertnum = floatstr
proc.uncert_num = uncertnum

if uncertnum.find('e') != -1:
if uncertnum is None:
parts = [None, None]
elif uncertnum.find('e') != -1:
parts = uncertnum.split("e")
elif float(uncertnum) < 1: # float returns decimal value not value in sci notation
ulen = len(uncertnum)
exp = -1
for i in range(2, ulen-1):
if uncertnum[i] == '0':
exp = exp - 1
expn = -1
for j in range(2, ulen-1):
if uncertnum[j] == '0':
expn = expn - 1
acc = len(uncertnum) - 2
man = str(float(uncertnum) * pow(10, -1 * exp))[0:acc+1]
parts = [man, str(exp)]
man = str(float(uncertnum) * pow(10, -1 * expn))[0:acc+1]
parts = [man, str(expn)]
else:
parts = [uncertnum, 0]
proc.uncert_man = parts[0]
proc.uncert_exp = int(parts[1])
proc.uncert_acc = len(parts[0]) - 1 # lose one for decimal point
if parts[1]:
proc.uncert_exp = int(parts[1])
else:
proc.uncert_exp = None
if parts[0]:
proc.uncert_acc = len(parts[0]) - 1 # lose one for decimal point
else:
proc.uncert_acc = None

# # calculate the relative uncertainty
# reluncert = float(proc.uncert_num)/float(proc.value_num)
# relacc = proc.uncert_acc
# uparts = str(reluncert).split("e")
# proc.reluncert_man = uparts[0][0:relacc+1]
# proc.reluncert_exp = int(uparts[1])

return


years = ['1998']
years = ['2022']
for year in years:
path = str(BASE_DIR) + '/' + STATIC_URL + 'imports/allascii_' + year + '.txt'

Expand All @@ -90,12 +113,20 @@ def procuncert(unc, proc):
reader = csv.reader(f, delimiter="\t")
consts = list(reader)

ntmp = Constants.objects.all().values_list('allnames', flat=True)
names = []
for nam in ntmp:
nlist = json.loads(nam)
for nitem in nlist:
names.append(nitem)

for const in consts:
print(const)
# print(const)
name = const[0]
# check if constant has been added
check = Constantvalues.objects.filter(orig_name=name, year=year)
if check.count() == 1:
print("found " + name)
continue
data = {}
# check if its three columns of data (uncert at end) or four (just process)
Expand Down Expand Up @@ -124,8 +155,9 @@ def procuncert(unc, proc):
conval.orig_name = name
conval.year = year
# check for ellipsis
if const[1].find('…') != -1:
if const[1].find('…') != -1 or const[1].find('...') != -1:
ell = 1
const[1] = const[1].replace('…', '').replace('...', '')
else:
ell = 0
conval.ellipsis = ell
Expand All @@ -134,14 +166,26 @@ def procuncert(unc, proc):
# process the uncertainty
procuncert(const[2], conval)
# add the unit
conval.orig_unit = const[3]
if const[3] == '':
conval.orig_unit = '1'
else:
conval.orig_unit = const[3]

# find constant and add id
con = Constants.objects.filter(allnames__contains=name)[0]
if name in names:
con = Constants.objects.filter(allnames__contains=name)[0]
else:
print("new constant - " + name)
exit()
conval.constant_id = con.id
# save constantvalue
conval.updated = getds()
conval.save()

# update constant for year based version
match year:
case '2022':
con.is_2022 = 1
case '2018':
con.is_2018 = 1
case '2014':
Expand All @@ -154,4 +198,8 @@ def procuncert(unc, proc):
con.is_2002 = 1
case '1998':
con.is_1998 = 1

# update constant table
con.updated = getds()
con.save()
print("added " + name)
1 change: 1 addition & 0 deletions constants/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ class Constants(models.Model):
qudt_id = models.CharField(max_length=256, blank=True, null=True)
is_qudt = models.IntegerField(blank=True, null=True)
is_codata = models.IntegerField(blank=True, null=True)
is_2022 = models.IntegerField(blank=True, null=True)
is_2018 = models.IntegerField(blank=True, null=True)
is_2014 = models.IntegerField(blank=True, null=True)
is_2010 = models.IntegerField(blank=True, null=True)
Expand Down
55 changes: 53 additions & 2 deletions constants/validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@
import django
import math
import requests
import re

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "umisconfig.settings")
django.setup()

from constants.models import *
from bs4 import BeautifulSoup

run = "calcrel"
run = "getreluncert"


if run == "idsval":
Expand Down Expand Up @@ -78,7 +79,7 @@
continue # invalid page
title = link.string.strip()
# find constant in DB
found = Constants.objects.filter(allnames__contains="'" + title + "'")
found = Constants.objects.filter(allnames__contains='"' + title + '"')
hit = None
if found.count() == 1:
# save page to DB entry
Expand Down Expand Up @@ -144,3 +145,53 @@
upd.comments = cmt
upd.save()
print("Constant '" + val['orig_name'] + ":" + str(val['year']) + "'")

if run == 'getreluncert':
path = "https://physics.nist.gov"
url = path + "/cgi-bin/cuu/Category?view=html&All+values.x=0"
page = requests.get(url)
soup = BeautifulSoup(page.content, "html.parser")
cnt = 0
year = '2022'
for link in soup.find_all('a'):
if link.get('href').find('/cuu/Value') == -1:
continue # not found
else:
tmp = link.get('href').split('|')
href = path + tmp[0]
if href == 'https://physics.nist.gov/cgi-bin/cuu/Value?am':
continue # invalid page
title = link.string.strip()
found = Constants.objects.filter(allnames__contains='"' + title + '"')
if found.count() == 1:
con = found[0]
# get constant value for constant and year
conid = con.id
val = Constantvalues.objects.get(constant_id=conid, year=year)
if val.reluncert_man is not None:
print(con.name + ' already added')
continue
# go to the page and get the relative uncertainty
page = requests.get(href)
const = BeautifulSoup(page.content, "html.parser")
rows = const.findAll("td", attrs={"align": "right", "bgcolor": "#cce2f3"})
for row in rows:
if row.text.strip() == 'Relative standard uncertainty':
uncert = row.parent.findAll('b')[0].text.strip()
if uncert == '(exact)':
val.reluncert_exp = None
val.reluncert_man = None
else:
parts = re.split("\xa0x\xa010", uncert)
val.reluncert_man = parts[0]
val.reluncert_exp = parts[1]
val.save()
print('added ' + con.name)
elif found.count() > 1:
print("multiple hits!")
print(found.values('id', 'name'))
exit()
else:
print("not found!")
print(title)
exit()
6 changes: 3 additions & 3 deletions constants/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def index(request):

def view(request, cnid):
constant = Constants.objects.get(id=cnid)
values = constant.constantvalues_set.all()
values = constant.constantvalues_set.all().order_by('-year')
allvals, allaccs, allexps = [], [], []
for i, value in enumerate(values):
# reformat field value to display correctly
Expand Down Expand Up @@ -49,7 +49,7 @@ def view(request, cnid):
def jsonout(request, cnid):
constant = dict(Constants.objects.values('identifier', 'name', 'units_si', 'nistpage', 'symbol').get(id=cnid))
values = Constantvalues.objects.filter(constant_id=cnid).\
values('year', 'value_num', 'uncert_num', 'orig_unit', 'reluncert_man', 'reluncert_exp')
values('year', 'value_num', 'uncert_num', 'orig_unit', 'reluncert_man', 'reluncert_exp').order_by('year')
constant.update({'values': []})
for value in values:
constant['values'].append(dict(value))
Expand All @@ -60,7 +60,7 @@ def jsonout(request, cnid):
def alldata(request, cnid):
constant = Constants.objects.get(id=cnid)
condata = {'datasets': [{'label': constant.name, 'data': []}]}
values = constant.constantvalues_set.all()
values = constant.constantvalues_set.all().order_by('year')
data = []
for value in values:
xy = {'x': int(value.year), 'y': float(value.value_num)}
Expand Down
2 changes: 1 addition & 1 deletion templates/constants/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<div class="card-header">
<div class="row">
<div class="col-5">
<h5 class="card-title mb-0">Fundamental Constants</h5>
<h5 class="card-title mb-0">Fundamental Constants ({{ constants|length }})</h5>
</div>
<div class="col-5 offset-2 px-0">
<label for="listsrc"></label>
Expand Down
24 changes: 9 additions & 15 deletions templates/constants/view.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
locale: 'en-EN',
scales: {
x: {
min: {{ 1995 }},
max: {{ 2025 }},
type: 'linear',
display: 'true',
offset: 'true',
Expand All @@ -19,22 +21,14 @@
grid: { display: false },
time: { unit: 'year' },
ticks: {
callback: (value) => {
return value;
}
autoSkip: true,
maxTicksLimit: 7
}
},
y: {
min: {{ ymin }},
max: {{ ymax }},
title: { display: true, text: 'Value' },
ticks: {
callback: (value) => {
return value;
{#return new Intl.NumberFormat('en-EN', {#}
{# maximumSignificantDigits: {{ acc }} }).format(value)#}
}
} // removes ',' from year values
title: { display: true, text: 'Value' }
}
},
plugins: {
Expand All @@ -59,15 +53,15 @@
<div class="card">
<div class="card-header text-bg-light">
<a type="button" href="/constants/json/{{ constant.id }}" class="btn btn-xs btn-info text-white float-end">JSON</a>
<h5 class="card-title mb-0">Constant: {{ constant.name }} ({{ constant.symbol }})</h5>
<h5 class="card-title mb-0">Fundmental Constant: {{ constant.name }} ({{ constant.symbol }})</h5>
</div>
<div class="card-body">
<div class="row">
<div id="accordionValues" class="accordion col-12 col-md-6">
<div id="accordionValues" class="accordion col-12 col-lg-5">
{% for value in values %}
<div class="accordion-item">
<div id="heading{{ forloop.counter }}" class="accordion-header">
<button class="accordion-button" type="button" data-bs-toggle="collapse"
<button class="accordion-button p-2" type="button" data-bs-toggle="collapse"
data-bs-target="#collapse{{ forloop.counter }}"
aria-expanded="{% if forloop.first %}true{% else %}false{% endif %}"
aria-controls="collapse{{ forloop.counter }}">
Expand Down Expand Up @@ -111,7 +105,7 @@ <h5 class="card-title mb-0">Constant: {{ constant.name }} ({{ constant.symbol }}
</div>
{% endfor %}
</div>
<div class="col-12 col-md-6">
<div class="col-12 col-lg-7">
<canvas id="chart"></canvas>
</div>
</div>
Expand Down

0 comments on commit 9cf7813

Please sign in to comment.