diff --git a/constants/ingest.py b/constants/ingest.py index 8be3e10..5712ac3 100644 --- a/constants/ingest.py +++ b/constants/ingest.py @@ -3,21 +3,24 @@ import django import csv import re +import json os.environ.setdefault("DJANGO_SETTINGS_MODULE", "umisconfig.settings") django.setup() from constants.models import * -from django.db.models import Count from umisconfig.settings import STATIC_URL, BASE_DIR from decimal import Decimal from os.path import exists +from units.functions import * +from math import * def procvalue(val, proc): proc.orig_value = val valuestr = val.replace(' ', '') valuestr = valuestr.replace('…', '') + valuestr = valuestr.replace('...', '') proc.value_str = valuestr if val.find('e') != -1: valueacc = valuestr.index('e') - 1 @@ -38,7 +41,8 @@ def procvalue(val, proc): def procuncert(unc, proc): - if unc == 'null': + # calculate uncertainties and add to proc + if unc == 'null' or unc == '(exact)': proc.uncert_str = None proc.uncert_num = None proc.uncert_man = None @@ -52,34 +56,53 @@ def procuncert(unc, proc): tzero = '0' else: tzero = '' - floatval = float(uncertstr) - floatstr = str(floatval) + if '(exact)' in uncertstr: + floatstr = None + else: + floatval = float(uncertstr) + floatstr = str(floatval) if tzero == '0': uncertnum = floatstr.replace('e', '.0e') else: uncertnum = floatstr proc.uncert_num = uncertnum - if uncertnum.find('e') != -1: + if uncertnum is None: + parts = [None, None] + elif uncertnum.find('e') != -1: parts = uncertnum.split("e") elif float(uncertnum) < 1: # float returns decimal value not value in sci notation ulen = len(uncertnum) - exp = -1 - for i in range(2, ulen-1): - if uncertnum[i] == '0': - exp = exp - 1 + expn = -1 + for j in range(2, ulen-1): + if uncertnum[j] == '0': + expn = expn - 1 acc = len(uncertnum) - 2 - man = str(float(uncertnum) * pow(10, -1 * exp))[0:acc+1] - parts = [man, str(exp)] + man = str(float(uncertnum) * pow(10, -1 * expn))[0:acc+1] + parts = [man, str(expn)] else: parts = [uncertnum, 0] proc.uncert_man = parts[0] - proc.uncert_exp = int(parts[1]) - proc.uncert_acc = len(parts[0]) - 1 # lose one for decimal point + if parts[1]: + proc.uncert_exp = int(parts[1]) + else: + proc.uncert_exp = None + if parts[0]: + proc.uncert_acc = len(parts[0]) - 1 # lose one for decimal point + else: + proc.uncert_acc = None + + # # calculate the relative uncertainty + # reluncert = float(proc.uncert_num)/float(proc.value_num) + # relacc = proc.uncert_acc + # uparts = str(reluncert).split("e") + # proc.reluncert_man = uparts[0][0:relacc+1] + # proc.reluncert_exp = int(uparts[1]) + return -years = ['1998'] +years = ['2022'] for year in years: path = str(BASE_DIR) + '/' + STATIC_URL + 'imports/allascii_' + year + '.txt' @@ -90,12 +113,20 @@ def procuncert(unc, proc): reader = csv.reader(f, delimiter="\t") consts = list(reader) + ntmp = Constants.objects.all().values_list('allnames', flat=True) + names = [] + for nam in ntmp: + nlist = json.loads(nam) + for nitem in nlist: + names.append(nitem) + for const in consts: - print(const) + # print(const) name = const[0] # check if constant has been added check = Constantvalues.objects.filter(orig_name=name, year=year) if check.count() == 1: + print("found " + name) continue data = {} # check if its three columns of data (uncert at end) or four (just process) @@ -124,8 +155,9 @@ def procuncert(unc, proc): conval.orig_name = name conval.year = year # check for ellipsis - if const[1].find('…') != -1: + if const[1].find('…') != -1 or const[1].find('...') != -1: ell = 1 + const[1] = const[1].replace('…', '').replace('...', '') else: ell = 0 conval.ellipsis = ell @@ -134,14 +166,26 @@ def procuncert(unc, proc): # process the uncertainty procuncert(const[2], conval) # add the unit - conval.orig_unit = const[3] + if const[3] == '': + conval.orig_unit = '1' + else: + conval.orig_unit = const[3] + # find constant and add id - con = Constants.objects.filter(allnames__contains=name)[0] + if name in names: + con = Constants.objects.filter(allnames__contains=name)[0] + else: + print("new constant - " + name) + exit() conval.constant_id = con.id # save constantvalue + conval.updated = getds() conval.save() + # update constant for year based version match year: + case '2022': + con.is_2022 = 1 case '2018': con.is_2018 = 1 case '2014': @@ -154,4 +198,8 @@ def procuncert(unc, proc): con.is_2002 = 1 case '1998': con.is_1998 = 1 + + # update constant table + con.updated = getds() con.save() + print("added " + name) diff --git a/constants/models.py b/constants/models.py index 1f31040..b7efe17 100644 --- a/constants/models.py +++ b/constants/models.py @@ -18,6 +18,7 @@ class Constants(models.Model): qudt_id = models.CharField(max_length=256, blank=True, null=True) is_qudt = models.IntegerField(blank=True, null=True) is_codata = models.IntegerField(blank=True, null=True) + is_2022 = models.IntegerField(blank=True, null=True) is_2018 = models.IntegerField(blank=True, null=True) is_2014 = models.IntegerField(blank=True, null=True) is_2010 = models.IntegerField(blank=True, null=True) diff --git a/constants/validate.py b/constants/validate.py index 29314d9..5d813db 100644 --- a/constants/validate.py +++ b/constants/validate.py @@ -3,6 +3,7 @@ import django import math import requests +import re os.environ.setdefault("DJANGO_SETTINGS_MODULE", "umisconfig.settings") django.setup() @@ -10,7 +11,7 @@ from constants.models import * from bs4 import BeautifulSoup -run = "calcrel" +run = "getreluncert" if run == "idsval": @@ -78,7 +79,7 @@ continue # invalid page title = link.string.strip() # find constant in DB - found = Constants.objects.filter(allnames__contains="'" + title + "'") + found = Constants.objects.filter(allnames__contains='"' + title + '"') hit = None if found.count() == 1: # save page to DB entry @@ -144,3 +145,53 @@ upd.comments = cmt upd.save() print("Constant '" + val['orig_name'] + ":" + str(val['year']) + "'") + +if run == 'getreluncert': + path = "https://physics.nist.gov" + url = path + "/cgi-bin/cuu/Category?view=html&All+values.x=0" + page = requests.get(url) + soup = BeautifulSoup(page.content, "html.parser") + cnt = 0 + year = '2022' + for link in soup.find_all('a'): + if link.get('href').find('/cuu/Value') == -1: + continue # not found + else: + tmp = link.get('href').split('|') + href = path + tmp[0] + if href == 'https://physics.nist.gov/cgi-bin/cuu/Value?am': + continue # invalid page + title = link.string.strip() + found = Constants.objects.filter(allnames__contains='"' + title + '"') + if found.count() == 1: + con = found[0] + # get constant value for constant and year + conid = con.id + val = Constantvalues.objects.get(constant_id=conid, year=year) + if val.reluncert_man is not None: + print(con.name + ' already added') + continue + # go to the page and get the relative uncertainty + page = requests.get(href) + const = BeautifulSoup(page.content, "html.parser") + rows = const.findAll("td", attrs={"align": "right", "bgcolor": "#cce2f3"}) + for row in rows: + if row.text.strip() == 'Relative standard uncertainty': + uncert = row.parent.findAll('b')[0].text.strip() + if uncert == '(exact)': + val.reluncert_exp = None + val.reluncert_man = None + else: + parts = re.split("\xa0x\xa010", uncert) + val.reluncert_man = parts[0] + val.reluncert_exp = parts[1] + val.save() + print('added ' + con.name) + elif found.count() > 1: + print("multiple hits!") + print(found.values('id', 'name')) + exit() + else: + print("not found!") + print(title) + exit() diff --git a/constants/views.py b/constants/views.py index c12f3f6..e6e98b9 100644 --- a/constants/views.py +++ b/constants/views.py @@ -14,7 +14,7 @@ def index(request): def view(request, cnid): constant = Constants.objects.get(id=cnid) - values = constant.constantvalues_set.all() + values = constant.constantvalues_set.all().order_by('-year') allvals, allaccs, allexps = [], [], [] for i, value in enumerate(values): # reformat field value to display correctly @@ -49,7 +49,7 @@ def view(request, cnid): def jsonout(request, cnid): constant = dict(Constants.objects.values('identifier', 'name', 'units_si', 'nistpage', 'symbol').get(id=cnid)) values = Constantvalues.objects.filter(constant_id=cnid).\ - values('year', 'value_num', 'uncert_num', 'orig_unit', 'reluncert_man', 'reluncert_exp') + values('year', 'value_num', 'uncert_num', 'orig_unit', 'reluncert_man', 'reluncert_exp').order_by('year') constant.update({'values': []}) for value in values: constant['values'].append(dict(value)) @@ -60,7 +60,7 @@ def jsonout(request, cnid): def alldata(request, cnid): constant = Constants.objects.get(id=cnid) condata = {'datasets': [{'label': constant.name, 'data': []}]} - values = constant.constantvalues_set.all() + values = constant.constantvalues_set.all().order_by('year') data = [] for value in values: xy = {'x': int(value.year), 'y': float(value.value_num)} diff --git a/templates/constants/index.html b/templates/constants/index.html index 5bd6b50..18afe88 100644 --- a/templates/constants/index.html +++ b/templates/constants/index.html @@ -8,7 +8,7 @@
-
Fundamental Constants
+
Fundamental Constants ({{ constants|length }})
diff --git a/templates/constants/view.html b/templates/constants/view.html index 74a40d2..61f37e7 100644 --- a/templates/constants/view.html +++ b/templates/constants/view.html @@ -11,6 +11,8 @@ locale: 'en-EN', scales: { x: { + min: {{ 1995 }}, + max: {{ 2025 }}, type: 'linear', display: 'true', offset: 'true', @@ -19,22 +21,14 @@ grid: { display: false }, time: { unit: 'year' }, ticks: { - callback: (value) => { - return value; - } + autoSkip: true, + maxTicksLimit: 7 } }, y: { min: {{ ymin }}, max: {{ ymax }}, - title: { display: true, text: 'Value' }, - ticks: { - callback: (value) => { - return value; - {#return new Intl.NumberFormat('en-EN', {#} - {# maximumSignificantDigits: {{ acc }} }).format(value)#} - } - } // removes ',' from year values + title: { display: true, text: 'Value' } } }, plugins: { @@ -59,15 +53,15 @@
JSON -
Constant: {{ constant.name }} ({{ constant.symbol }})
+
Fundmental Constant: {{ constant.name }} ({{ constant.symbol }})
-
+
{% for value in values %}
-
{% endfor %}
-
+