From 14edb8c5d26fe95d31698491a88beb3ce3b390b1 Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Wed, 25 Sep 2024 11:09:41 -0500 Subject: [PATCH 01/14] refactor: add omniring to device settings db (cherry picked from commit 7c59e4c8fe5d9e5384ca225fd9ec32c7c7102623) --- .../0128_devicesettings_omniring_and_more.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 database/migrations/0128_devicesettings_omniring_and_more.py diff --git a/database/migrations/0128_devicesettings_omniring_and_more.py b/database/migrations/0128_devicesettings_omniring_and_more.py new file mode 100644 index 000000000..ac1153dd9 --- /dev/null +++ b/database/migrations/0128_devicesettings_omniring_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.15 on 2024-09-24 20:11 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('database', '0127_study_end_date_study_manually_stopped'), + ] + + operations = [ + migrations.AddField( + model_name='devicesettings', + name='omniring', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='devicesettings', + name='omniring_global_offset_seconds', + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name='devicesettings', + name='omniring_on_duration_seconds', + field=models.PositiveIntegerField(default=60, validators=[django.core.validators.MinValueValidator(1)]), + ), + migrations.AddField( + model_name='devicesettings', + name='omniring_total_duration_seconds', + field=models.PositiveIntegerField(default=300, validators=[django.core.validators.MinValueValidator(1)]), + ), + ] From 7ced5b5ba41e560e56e16fcdd11f30dd44940f9d Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Wed, 25 Sep 2024 11:10:10 -0500 Subject: [PATCH 02/14] feat: add omniring FE config (cherry picked from commit a8cfed9f42a0adfbb42750af0b5ed0b774295294) --- frontend/templates/study_device_settings.html | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/frontend/templates/study_device_settings.html b/frontend/templates/study_device_settings.html index af528a44b..a55caa353 100755 --- a/frontend/templates/study_device_settings.html +++ b/frontend/templates/study_device_settings.html @@ -334,6 +334,52 @@

Data Stream Settings

value="{{ settings['bluetooth_global_offset_seconds'] }}" {{ "disabled" if readonly }}> + + {# OmniRing #} +
+ The OmniRing data stream is synchonized across all devices in your study. It scans at a specific time + within a defined period. An On duration of 5 minutes with a Total duration of 1 hour + and a global offset of 15 minutes will trigger a bluetooth scan for 5 minutes once an hour, + every hour, at 15 minutes past the hour. +
+
+
+
+ +
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
{% if not readonly %} From 1ea0067639c67638e63cf3eedb49bab21a6a48da Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Wed, 25 Sep 2024 11:10:20 -0500 Subject: [PATCH 03/14] fix: running in docker env var (cherry picked from commit 00a4c821085dbb93b0c0ed8d1020ebab01c0b618) --- docker_management/dev.docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker_management/dev.docker-compose.yml b/docker_management/dev.docker-compose.yml index 16121e2ad..e5727c2eb 100644 --- a/docker_management/dev.docker-compose.yml +++ b/docker_management/dev.docker-compose.yml @@ -69,6 +69,7 @@ services: context: ${PWD} dockerfile: docker_management/backend/dev.Dockerfile environment: + - RUNNING_IN_DOCKER=True - DOMAIN_NAME=${DOMAIN_NAME} - FLASK_SECRET_KEY=${FLASK_SECRET_KEY} - S3_BUCKET=${S3_BUCKET} From 62ddc1263d7621f1d55164e087fdb0bf04817d7a Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Wed, 25 Sep 2024 11:10:34 -0500 Subject: [PATCH 04/14] refactor: add omniring to device settings db (cherry picked from commit ca1516b1d2e21d99f9c92adbe7512e411d436e8c) --- database/study_models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/database/study_models.py b/database/study_models.py index 332040a07..a4e27810d 100644 --- a/database/study_models.py +++ b/database/study_models.py @@ -14,14 +14,13 @@ from constants.data_stream_constants import ALL_DATA_STREAMS from constants.message_strings import DEFAULT_HEARTBEAT_MESSAGE from constants.study_constants import (ABOUT_PAGE_TEXT, CONSENT_FORM_TEXT, - DEFAULT_CONSENT_SECTIONS_JSON, SURVEY_SUBMIT_SUCCESS_TOAST_TEXT) + DEFAULT_CONSENT_SECTIONS_JSON, SURVEY_SUBMIT_SUCCESS_TOAST_TEXT) from constants.user_constants import ResearcherRole from database.common_models import ObjectIDModel, UtilityModel from database.models import JSONTextField, TimestampedModel from database.validators import LengthValidator from libs.utils.date_utils import date_is_in_the_past - # this is an import hack to improve IDE assistance try: from database.models import (ChunkRegistry, DashboardColorSetting, FileToProcess, Intervention, @@ -202,6 +201,7 @@ def export(self) -> Dict[str, Any]: texts = models.BooleanField(default=True) wifi = models.BooleanField(default=True) bluetooth = models.BooleanField(default=False) + omniring = models.BooleanField(default=False) power_state = models.BooleanField(default=True) use_anonymized_hashing = models.BooleanField(default=True) use_gps_fuzzing = models.BooleanField(default=False) @@ -230,6 +230,9 @@ def export(self) -> Dict[str, Any]: bluetooth_on_duration_seconds = models.PositiveIntegerField(default=60, validators=[MinValueValidator(1)]) bluetooth_total_duration_seconds = models.PositiveIntegerField(default=300, validators=[MinValueValidator(1)]) bluetooth_global_offset_seconds = models.PositiveIntegerField(default=0) + omniring_on_duration_seconds = models.PositiveIntegerField(default=60, validators=[MinValueValidator(1)]) + omniring_total_duration_seconds = models.PositiveIntegerField(default=300, validators=[MinValueValidator(1)]) + omniring_global_offset_seconds = models.PositiveIntegerField(default=0) check_for_new_surveys_frequency_seconds = models.PositiveIntegerField(default=3600, validators=[MinValueValidator(30)]) create_new_data_files_frequency_seconds = models.PositiveIntegerField(default=15 * 60, validators=[MinValueValidator(30)]) gps_off_duration_seconds = models.PositiveIntegerField(default=600, validators=[MinValueValidator(1)]) From c544da80df71faf6e30b450e65a8fee195bc71d5 Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Wed, 25 Sep 2024 11:10:48 -0500 Subject: [PATCH 05/14] refactor: add omniring constants (cherry picked from commit 824be169c6db70a8f22e3cbf9d075fc4047b587e) --- constants/data_stream_constants.py | 7 +++++++ constants/study_constants.py | 4 ++++ data_access_api_reference/download_data.py | 1 + 3 files changed, 12 insertions(+) diff --git a/constants/data_stream_constants.py b/constants/data_stream_constants.py index 6cd31414a..f825a6d77 100644 --- a/constants/data_stream_constants.py +++ b/constants/data_stream_constants.py @@ -3,6 +3,7 @@ AMBIENT_AUDIO = "ambient_audio" ANDROID_LOG_FILE = "app_log" BLUETOOTH = "bluetooth" +OMNIRING = "omniring" CALL_LOG = "calls" DEVICEMOTION = "devicemotion" GPS = "gps" @@ -25,6 +26,7 @@ AMBIENT_AUDIO, ANDROID_LOG_FILE, BLUETOOTH, + OMNIRING, CALL_LOG, DEVICEMOTION, GPS, @@ -47,6 +49,7 @@ UPLOAD_FILE_TYPE_MAPPING = { "accel": ACCELEROMETER, "bluetoothLog": BLUETOOTH, + "omniring": OMNIRING, "callLog": CALL_LOG, "devicemotion": DEVICEMOTION, "gps": GPS, @@ -73,6 +76,7 @@ DATA_STREAM_TO_S3_FILE_NAME_STRING = { ACCELEROMETER: "accel", BLUETOOTH: "bluetoothLog", + OMNIRING: "omniring", CALL_LOG: "callLog", GPS: "gps", IDENTIFIERS: "identifiers", @@ -95,6 +99,7 @@ CHUNKABLE_FILES = { ACCELEROMETER, BLUETOOTH, + OMNIRING, CALL_LOG, GPS, IDENTIFIERS, @@ -123,6 +128,7 @@ AMBIENT_AUDIO, ANDROID_LOG_FILE, BLUETOOTH, + OMNIRING, CALL_LOG, DEVICEMOTION, GPS, @@ -145,6 +151,7 @@ AMBIENT_AUDIO: "Ambient Audio Recording (bytes)", ANDROID_LOG_FILE: "Android Log File (bytes)", BLUETOOTH: "Bluetooth (bytes)", + OMNIRING: "Omniring (bytes)", CALL_LOG: "Call Log (bytes)", DEVICEMOTION: "Device Motion (bytes)", GPS: "GPS (bytes)", diff --git a/constants/study_constants.py b/constants/study_constants.py index 702c8cb6b..6fd68187e 100644 --- a/constants/study_constants.py +++ b/constants/study_constants.py @@ -60,6 +60,7 @@ "texts", "wifi", "bluetooth", + "omniring", "power_state", "proximity", "gyro", @@ -80,6 +81,9 @@ "bluetooth_on_duration_seconds", "bluetooth_total_duration_seconds", "bluetooth_global_offset_seconds", + "omniring_on_duration_seconds", + "omniring_total_duration_seconds", + "omniring_global_offset_seconds", "check_for_new_surveys_frequency_seconds", "create_new_data_files_frequency_seconds", "gps_off_duration_seconds", diff --git a/data_access_api_reference/download_data.py b/data_access_api_reference/download_data.py index 52079354b..ef930e040 100644 --- a/data_access_api_reference/download_data.py +++ b/data_access_api_reference/download_data.py @@ -26,6 +26,7 @@ # Data Streams ACCELEROMETER = "accelerometer" BLUETOOTH = "bluetooth" +OMNIRING = "omniring" CALL_LOG = "calls" GPS = "gps" IDENTIFIERS = "identifiers" From f4af2ccdd91682575c262f21295a10b118e1f59f Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Tue, 26 Nov 2024 11:06:30 -0600 Subject: [PATCH 06/14] refactor: add omniring upload headers --- constants/data_processing_constants.py | 85 ++++++++++++++------------ 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/constants/data_processing_constants.py b/constants/data_processing_constants.py index f3a5f9b22..81a288d1f 100644 --- a/constants/data_processing_constants.py +++ b/constants/data_processing_constants.py @@ -1,11 +1,11 @@ ## Chunks # This value is in seconds, it sets the time period that chunked files will be sliced into. -from constants.data_stream_constants import (ACCELEROMETER, ANDROID_LOG_FILE, BLUETOOTH, CALL_LOG, - DEVICE_IDENTIFIERS_HEADER, DEVICEMOTION, GPS, GYRO, IDENTIFIERS, IOS_LOG_FILE, MAGNETOMETER, - POWER_STATE, PROXIMITY, REACHABILITY, SURVEY_TIMINGS, TEXTS_LOG, WIFI) +from constants.data_stream_constants import (ACCELEROMETER, ANDROID_LOG_FILE, BLUETOOTH, OMNIRING, CALL_LOG, + DEVICE_IDENTIFIERS_HEADER, DEVICEMOTION, GPS, GYRO, IDENTIFIERS, + IOS_LOG_FILE, MAGNETOMETER, + POWER_STATE, PROXIMITY, REACHABILITY, SURVEY_TIMINGS, TEXTS_LOG, WIFI) from constants.user_constants import ANDROID_API, IOS_API - CHUNK_TIMESLICE_QUANTUM = 3600 # These reference dicts contain the output headers that should exist for each data stream, per-os. @@ -19,55 +19,56 @@ REFERENCE_CHUNKREGISTRY_HEADERS = { ACCELEROMETER: { ANDROID_API: b'timestamp,UTC time,accuracy,x,y,z', - IOS_API: b'timestamp,UTC time,accuracy,x,y,z', + IOS_API: b'timestamp,UTC time,accuracy,x,y,z', }, ANDROID_LOG_FILE: { ANDROID_API: b'timestamp,UTC time,event', - IOS_API: b'timestamp,UTC time,event', + IOS_API: b'timestamp,UTC time,event', }, BLUETOOTH: { ANDROID_API: b'timestamp,UTC time,hashed MAC,RSSI', - IOS_API: b'timestamp,UTC time,hashed MAC,RSSI', # android-only data stream + IOS_API: b'timestamp,UTC time,hashed MAC,RSSI', # android-only data stream }, CALL_LOG: { ANDROID_API: b'timestamp,UTC time,hashed phone number,call type,duration in seconds', - IOS_API: b'timestamp,UTC time,hashed phone number,call type,duration in seconds', # android-only data stream + IOS_API: b'timestamp,UTC time,hashed phone number,call type,duration in seconds', # android-only data stream }, DEVICEMOTION: { - ANDROID_API: b'timestamp,UTC time,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', # ios-only data stream - IOS_API: b'timestamp,UTC time,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', + ANDROID_API: b'timestamp,UTC time,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', + # ios-only data stream + IOS_API: b'timestamp,UTC time,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', }, GPS: { ANDROID_API: b'timestamp,UTC time,latitude,longitude,altitude,accuracy', - IOS_API: b'timestamp,UTC time,latitude,longitude,altitude,accuracy', + IOS_API: b'timestamp,UTC time,latitude,longitude,altitude,accuracy', }, GYRO: { ANDROID_API: b'timestamp,UTC time,accuracy,x,y,z', - IOS_API: b'timestamp,UTC time,x,y,z', + IOS_API: b'timestamp,UTC time,x,y,z', }, IDENTIFIERS: { ANDROID_API: b'timestamp,UTC time,patient_id,MAC,phone_number,device_id,device_os,os_version,product,brand,hardware_id,manufacturer,model,beiwe_version', - IOS_API: b'timestamp,UTC time,patient_id,MAC,phone_number,device_id,device_os,os_version,product,brand,hardware_id,manufacturer,model,beiwe_version', + IOS_API: b'timestamp,UTC time,patient_id,MAC,phone_number,device_id,device_os,os_version,product,brand,hardware_id,manufacturer,model,beiwe_version', }, IOS_LOG_FILE: { ANDROID_API: b'timestamp,UTC time,launchId,memory,battery,event,msg,d1,d2,d3,d4', # ios-only datastream - IOS_API: b'timestamp,UTC time,launchId,memory,battery,event,msg,d1,d2,d3,d4', + IOS_API: b'timestamp,UTC time,launchId,memory,battery,event,msg,d1,d2,d3,d4', }, MAGNETOMETER: { ANDROID_API: b'timestamp,UTC time,x,y,z', # ios-only datastream - IOS_API: b'timestamp,UTC time,x,y,z', + IOS_API: b'timestamp,UTC time,x,y,z', }, POWER_STATE: { ANDROID_API: b'timestamp,UTC time,event', - IOS_API: b'timestamp,UTC time,event,level', + IOS_API: b'timestamp,UTC time,event,level', }, PROXIMITY: { ANDROID_API: b'timestamp,UTC time,event', # ios-only datastream - IOS_API: b'timestamp,UTC time,event', + IOS_API: b'timestamp,UTC time,event', }, REACHABILITY: { ANDROID_API: b'timestamp,UTC time,event', # ios-only datastream - IOS_API: b'timestamp,UTC time,event', + IOS_API: b'timestamp,UTC time,event', }, # SURVEY_ANSWERS: { # we don't chunk survey answers... # ANDROID_API: b'question id,question type,question text,question answer options,answer', @@ -75,47 +76,52 @@ # }, SURVEY_TIMINGS: { ANDROID_API: b'timestamp,UTC time,question id,survey id,question type,question text,question answer options,answer', - IOS_API: b'timestamp,UTC time,question id,survey id,question type,question text,question answer options,answer,event', + IOS_API: b'timestamp,UTC time,question id,survey id,question type,question text,question answer options,answer,event', }, TEXTS_LOG: { ANDROID_API: b'timestamp,UTC time,hashed phone number,sent vs received,message length,time sent', - IOS_API: b'timestamp,UTC time,hashed phone number,sent vs received,message length,time sent', # android-only datastream + IOS_API: b'timestamp,UTC time,hashed phone number,sent vs received,message length,time sent', + # android-only datastream }, WIFI: { ANDROID_API: b'timestamp,UTC time,hashed MAC,frequency,RSSI', - IOS_API: b'timestamp,UTC time,hashed MAC,frequency,RSSI', + IOS_API: b'timestamp,UTC time,hashed MAC,frequency,RSSI', } } - REFERENCE_UPLOAD_HEADERS = { ACCELEROMETER: { ANDROID_API: b'timestamp,accuracy,x,y,z', - IOS_API: b'timestamp,accuracy,x,y,z', + IOS_API: b'timestamp,accuracy,x,y,z', }, ANDROID_LOG_FILE: { ANDROID_API: b'THIS LINE IS A LOG FILE HEADER', - IOS_API: b'THIS LINE IS A LOG FILE HEADER', # android-only datastream + IOS_API: b'THIS LINE IS A LOG FILE HEADER', # android-only datastream }, BLUETOOTH: { ANDROID_API: b'timestamp, hashed MAC, RSSI', - IOS_API: b'timestamp, hashed MAC, RSSI', # android-only datastream + IOS_API: b'timestamp, hashed MAC, RSSI', # android-only datastream + }, + OMNIRING: { + ANDROID_API: b'PPG_red,PPG_IR,PPG_Green,IMU_Accel_x,IMU_Accel_y,IMU_Accel_z,IMU_Gyro_x,IMU_Gyro_y,IMU_Gyro_z,IMU_Mag_x,IMU_Mag_y,IMU_Mag_z,temperature,timestamp', + IOS_API: b'PPG_red,PPG_IR,PPG_Green,IMU_Accel_x,IMU_Accel_y,IMU_Accel_z,IMU_Gyro_x,IMU_Gyro_y,IMU_Gyro_z,IMU_Mag_x,IMU_Mag_y,IMU_Mag_z,temperature,timestamp', }, CALL_LOG: { ANDROID_API: b'hashed phone number,call type,timestamp,duration in seconds', - IOS_API: b'hashed phone number,call type,timestamp,duration in seconds', # android-only datastream + IOS_API: b'hashed phone number,call type,timestamp,duration in seconds', # android-only datastream }, DEVICEMOTION: { - ANDROID_API: b'timestamp,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', # ios-only datastream - IOS_API: b'timestamp,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', + ANDROID_API: b'timestamp,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', + # ios-only datastream + IOS_API: b'timestamp,roll,pitch,yaw,rotation_rate_x,rotation_rate_y,rotation_rate_z,gravity_x,gravity_y,gravity_z,user_accel_x,user_accel_y,user_accel_z,magnetic_field_calibration_accuracy,magnetic_field_x,magnetic_field_y,magnetic_field_z', }, GPS: { ANDROID_API: b'timestamp, latitude, longitude, altitude, accuracy', - IOS_API: b'timestamp,latitude,longitude,altitude,accuracy', + IOS_API: b'timestamp,latitude,longitude,altitude,accuracy', }, GYRO: { ANDROID_API: b'timestamp,accuracy,x,y,z', - IOS_API: b'timestamp,x,y,z', + IOS_API: b'timestamp,x,y,z', }, IDENTIFIERS: { ANDROID_API: DEVICE_IDENTIFIERS_HEADER, @@ -123,23 +129,23 @@ }, IOS_LOG_FILE: { ANDROID_API: b'timestamp,launchId,memory,battery,event,msg,d1,d2,d3,d4', # ios-only datastream - IOS_API: b'timestamp,launchId,memory,battery,event,msg,d1,d2,d3,d4', + IOS_API: b'timestamp,launchId,memory,battery,event,msg,d1,d2,d3,d4', }, MAGNETOMETER: { ANDROID_API: b'timestamp,x,y,z', # ios-only datastream - IOS_API: b'timestamp,x,y,z', + IOS_API: b'timestamp,x,y,z', }, POWER_STATE: { ANDROID_API: b'timestamp, event', - IOS_API: b'timestamp,event,level', + IOS_API: b'timestamp,event,level', }, PROXIMITY: { ANDROID_API: b'timestamp,event', # ios-only datastream - IOS_API: b'timestamp,event', + IOS_API: b'timestamp,event', }, REACHABILITY: { ANDROID_API: b'timestamp,event', # ios-only datastream - IOS_API: b'timestamp,event', + IOS_API: b'timestamp,event', }, # SURVEY_ANSWERS: { # we don't chunk survey answers... # ANDROID_API: b'question id,question type,question text,question answer options,answer', @@ -147,17 +153,16 @@ # }, SURVEY_TIMINGS: { ANDROID_API: b'timestamp,question id,question type,question text,question answer options,answer', - IOS_API: b'timestamp,question id,question type,question text,question answer options,answer,event', + IOS_API: b'timestamp,question id,question type,question text,question answer options,answer,event', }, TEXTS_LOG: { ANDROID_API: b'timestamp,hashed phone number,sent vs received,message length,time sent', - IOS_API: None, # ios-only datastream + IOS_API: None, # ios-only datastream }, WIFI: { ANDROID_API: b'hashed MAC, frequency, RSSI', - IOS_API: b'hashed MAC, frequency, RSSI', # android-only datastream + IOS_API: b'hashed MAC, frequency, RSSI', # android-only datastream } } - -CHUNK_EXISTS_CASE = "chunk_exists_case" \ No newline at end of file +CHUNK_EXISTS_CASE = "chunk_exists_case" From 07538328aea8e2abf5b667ec48dee25ee786ff2f Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Tue, 26 Nov 2024 13:57:24 -0600 Subject: [PATCH 07/14] fix: database migration merge --- database/migrations/0133_merge_20241126_1949.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 database/migrations/0133_merge_20241126_1949.py diff --git a/database/migrations/0133_merge_20241126_1949.py b/database/migrations/0133_merge_20241126_1949.py new file mode 100644 index 000000000..9a0dbeed0 --- /dev/null +++ b/database/migrations/0133_merge_20241126_1949.py @@ -0,0 +1,14 @@ +# Generated by Django 4.2.15 on 2024-11-26 19:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('database', '0128_devicesettings_omniring_and_more'), + ('database', '0132_participant_last_known_surveys_available_and_more'), + ] + + operations = [ + ] From 23dcb35ce03ee0491726049743ebe3127c7fc473 Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Tue, 26 Nov 2024 14:27:17 -0600 Subject: [PATCH 08/14] fix: omniring forest constants --- constants/forest_constants.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/constants/forest_constants.py b/constants/forest_constants.py index f6aa3fab2..ee985cba4 100644 --- a/constants/forest_constants.py +++ b/constants/forest_constants.py @@ -1,7 +1,7 @@ # trunk-ignore-all(ruff/F405) from django.db.models.fields import (BooleanField, CharField, DateField, DateTimeField, FloatField, - IntegerField, TextField) + IntegerField, TextField) # trunk-ignore(ruff/F403) from constants.data_stream_constants import * @@ -9,7 +9,6 @@ from forest.constants import Frequency - # the canonical location where any files are allocated for forest tasks. ROOT_FOREST_TASK_PATH = "/tmp/forest/" @@ -91,7 +90,6 @@ class ForestTree(DjangoDropdown): }, } - # special tree parameters PARAMETER_ALL_BV_SET = "all_bv_set" PARAMETER_ALL_MEMORY_DICT = "all_memory_dict" @@ -108,7 +106,6 @@ class ForestTree(DjangoDropdown): PARAMETER_INTERVENTIONS_FILEPATH, ] - # documented at https://forest.beiwe.org/en/latest/#forest-trees # Don't forget about FOREST_TREE_TO_SERIALIZABLE_FIELD_NAMES in tableau_api_constants.py FOREST_TREE_REQUIRED_DATA_STREAMS = { @@ -119,7 +116,6 @@ class ForestTree(DjangoDropdown): ForestTree.willow: [CALL_LOG, TEXTS_LOG], } - ## The following dictionary is a mapping of output CSV fields from various Forest Trees to their # summary statistic names. Note that this data structure is imported and used in tableau constants. @@ -185,7 +181,6 @@ class ForestTree(DjangoDropdown): "cadence": "oak_cadence", } - # Metadata SUMMARY_METADATA_FIELD_NAMES = [ "date", @@ -204,6 +199,7 @@ class ForestTree(DjangoDropdown): AMBIENT_AUDIO: "beiwe_ambient_audio_bytes", ANDROID_LOG_FILE: "beiwe_app_log_bytes", BLUETOOTH: "beiwe_bluetooth_bytes", + OMNIRING: "beiwe_omniring_bytes", CALL_LOG: "beiwe_calls_bytes", DEVICEMOTION: "beiwe_devicemotion_bytes", GPS: "beiwe_gps_bytes", @@ -252,7 +248,6 @@ class ForestTree(DjangoDropdown): name.replace("jasmine_", "").replace("_", " ").title() for name in JASMINE_FIELDS ] - WILLOW_FIELDS = [ # Willow, Texts "willow_incoming_text_count", @@ -308,12 +303,11 @@ class ForestTree(DjangoDropdown): ] SERIALIZABLE_FIELD_NAMES = SUMMARY_METADATA_FIELD_NAMES + DATA_QUANTITY_FIELD_NAMES \ - + JASMINE_FIELDS + WILLOW_FIELDS + SYCAMORE_FIELDS + OAK_FIELDS + + JASMINE_FIELDS + WILLOW_FIELDS + SYCAMORE_FIELDS + OAK_FIELDS # SERIALIZABLE_FIELD_NAMES.extend(TREE_COLUMN_NAMES_TO_SUMMARY_STATISTICS.values()) NICE_SERIALIZABLE_FIELD_NAMES = NICE_SUMMARY_METADATA_FIELD_NAMES + NICE_BEIWE_DATA_QUANTITY_FIELD_NAMES \ - + NICE_JASMINE_FIELDS + NICE_WILLOW_FIELDS + NICE_SYCAMORE_FIELDS + NICE_OAK_FIELDS - + + NICE_JASMINE_FIELDS + NICE_WILLOW_FIELDS + NICE_SYCAMORE_FIELDS + NICE_OAK_FIELDS FOREST_TREE_TO_SERIALIZABLE_FIELD_NAMES = { ForestTree.jasmine: JASMINE_FIELDS, @@ -322,7 +316,6 @@ class ForestTree(DjangoDropdown): ForestTree.oak: OAK_FIELDS, } - SERIALIZABLE_FIELD_NAMES_DROPDOWN = [(f, f) for f in SERIALIZABLE_FIELD_NAMES] VALID_QUERY_PARAMETERS = [ From fbc82d50298fd53ee894d1cb556f3902417b3a37 Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Tue, 26 Nov 2024 15:02:40 -0600 Subject: [PATCH 09/14] fix: omniring forest model --- database/forest_models.py | 22 ++++++++++++------- ... 0133_devicesettings_omniring_and_more.py} | 10 +++++++-- .../migrations/0133_merge_20241126_1949.py | 14 ------------ 3 files changed, 22 insertions(+), 24 deletions(-) rename database/migrations/{0128_devicesettings_omniring_and_more.py => 0133_devicesettings_omniring_and_more.py} (75%) delete mode 100644 database/migrations/0133_merge_20241126_1949.py diff --git a/database/forest_models.py b/database/forest_models.py index 3430caec4..a2bc43bd3 100644 --- a/database/forest_models.py +++ b/database/forest_models.py @@ -12,9 +12,11 @@ from config.settings import DOMAIN_NAME from constants.celery_constants import ForestTaskStatus from constants.forest_constants import (DEFAULT_FOREST_PARAMETERS, FOREST_PICKLING_ERROR, - ForestTree, NON_PICKLED_PARAMETERS, OAK_DATE_FORMAT_PARAMETER, PARAMETER_ALL_BV_SET, - PARAMETER_ALL_MEMORY_DICT, PARAMETER_CONFIG_PATH, PARAMETER_INTERVENTIONS_FILEPATH, - ROOT_FOREST_TASK_PATH, SYCAMORE_DATE_FORMAT) + ForestTree, NON_PICKLED_PARAMETERS, OAK_DATE_FORMAT_PARAMETER, + PARAMETER_ALL_BV_SET, + PARAMETER_ALL_MEMORY_DICT, PARAMETER_CONFIG_PATH, + PARAMETER_INTERVENTIONS_FILEPATH, + ROOT_FOREST_TASK_PATH, SYCAMORE_DATE_FORMAT) from database.common_models import TimestampedModel from database.user_models_participant import Participant from libs.utils.date_utils import datetime_to_list @@ -97,7 +99,6 @@ def sentry_tags(self) -> Dict[str, str]: # "all_memory_dict_s3_key": self.all_memory_dict_s3_key, } - # ## forest tree parameters # @@ -289,6 +290,7 @@ class SummaryStatisticDaily(TimestampedModel): beiwe_ambient_audio_bytes = models.PositiveBigIntegerField(null=True, blank=True) beiwe_app_log_bytes = models.PositiveBigIntegerField(null=True, blank=True) beiwe_bluetooth_bytes = models.PositiveBigIntegerField(null=True, blank=True) + beiwe_omniring_bytes = models.PositiveBigIntegerField(null=True, blank=True) beiwe_calls_bytes = models.PositiveBigIntegerField(null=True, blank=True) beiwe_devicemotion_bytes = models.PositiveBigIntegerField(null=True, blank=True) beiwe_gps_bytes = models.PositiveBigIntegerField(null=True, blank=True) @@ -364,10 +366,14 @@ class SummaryStatisticDaily(TimestampedModel): oak_cadence = models.FloatField(null=True, blank=True) # points to the task that populated this data set. () - jasmine_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, related_name="jasmine_summary_statistics") - willow_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, related_name="willow_summary_statistics") - sycamore_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, related_name="sycamore_summary_statistics") - oak_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, related_name="oak_summary_statistics") + jasmine_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, + related_name="jasmine_summary_statistics") + willow_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, + related_name="willow_summary_statistics") + sycamore_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, + related_name="sycamore_summary_statistics") + oak_task: ForestTask = models.ForeignKey(ForestTask, blank=True, null=True, on_delete=models.PROTECT, + related_name="oak_summary_statistics") class Meta: constraints = [ diff --git a/database/migrations/0128_devicesettings_omniring_and_more.py b/database/migrations/0133_devicesettings_omniring_and_more.py similarity index 75% rename from database/migrations/0128_devicesettings_omniring_and_more.py rename to database/migrations/0133_devicesettings_omniring_and_more.py index ac1153dd9..78334a743 100644 --- a/database/migrations/0128_devicesettings_omniring_and_more.py +++ b/database/migrations/0133_devicesettings_omniring_and_more.py @@ -1,12 +1,13 @@ -# Generated by Django 4.2.15 on 2024-09-24 20:11 +# Generated by Django 4.2.15 on 2024-11-26 20:40 import django.core.validators from django.db import migrations, models class Migration(migrations.Migration): + dependencies = [ - ('database', '0127_study_end_date_study_manually_stopped'), + ('database', '0132_participant_last_known_surveys_available_and_more'), ] operations = [ @@ -30,4 +31,9 @@ class Migration(migrations.Migration): name='omniring_total_duration_seconds', field=models.PositiveIntegerField(default=300, validators=[django.core.validators.MinValueValidator(1)]), ), + migrations.AddField( + model_name='summarystatisticdaily', + name='beiwe_omniring_bytes', + field=models.PositiveBigIntegerField(blank=True, null=True), + ), ] diff --git a/database/migrations/0133_merge_20241126_1949.py b/database/migrations/0133_merge_20241126_1949.py deleted file mode 100644 index 9a0dbeed0..000000000 --- a/database/migrations/0133_merge_20241126_1949.py +++ /dev/null @@ -1,14 +0,0 @@ -# Generated by Django 4.2.15 on 2024-11-26 19:49 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('database', '0128_devicesettings_omniring_and_more'), - ('database', '0132_participant_last_known_surveys_available_and_more'), - ] - - operations = [ - ] From d01d4e89db8f8dd781d3d53b99ad599f1613c851 Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Tue, 26 Nov 2024 15:27:41 -0600 Subject: [PATCH 10/14] fix: omniring data constant --- constants/data_stream_constants.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/constants/data_stream_constants.py b/constants/data_stream_constants.py index f825a6d77..26ef8dd98 100644 --- a/constants/data_stream_constants.py +++ b/constants/data_stream_constants.py @@ -20,7 +20,6 @@ VOICE_RECORDING = "audio_recordings" WIFI = "wifi" - ALL_DATA_STREAMS = [ ACCELEROMETER, AMBIENT_AUDIO, @@ -49,7 +48,7 @@ UPLOAD_FILE_TYPE_MAPPING = { "accel": ACCELEROMETER, "bluetoothLog": BLUETOOTH, - "omniring": OMNIRING, + "omniRingLog": OMNIRING, "callLog": CALL_LOG, "devicemotion": DEVICEMOTION, "gps": GPS, @@ -120,7 +119,6 @@ DEVICE_IDENTIFIERS_HEADER = \ "patient_id,MAC,phone_number,device_id,device_os,os_version,product,brand,hardware_id,manufacturer,model,beiwe_version\n" - ## Dashboard constants DASHBOARD_DATA_STREAMS = [ @@ -167,4 +165,4 @@ TEXTS_LOG: "Text Log (bytes)", VOICE_RECORDING: "Audio Recordings (bytes)", WIFI: "Wifi (bytes)", -} \ No newline at end of file +} From 6345f467a04c3b34dda1fc58beb959358143a0c7 Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Tue, 26 Nov 2024 15:50:51 -0600 Subject: [PATCH 11/14] fix: omniring data constant --- constants/data_processing_constants.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/constants/data_processing_constants.py b/constants/data_processing_constants.py index 81a288d1f..566762878 100644 --- a/constants/data_processing_constants.py +++ b/constants/data_processing_constants.py @@ -29,6 +29,10 @@ ANDROID_API: b'timestamp,UTC time,hashed MAC,RSSI', IOS_API: b'timestamp,UTC time,hashed MAC,RSSI', # android-only data stream }, + OMNIRING: { + ANDROID_API: b'timestamp,UTC time,PPG_red,PPG_IR,PPG_Green,IMU_Accel_x,IMU_Accel_y,IMU_Accel_z,IMU_Gyro_x,IMU_Gyro_y,IMU_Gyro_z,IMU_Mag_x,IMU_Mag_y,IMU_Mag_z,temperature,timestamp', + IOS_API: b'timestamp,UTC time,PPG_red,PPG_IR,PPG_Green,IMU_Accel_x,IMU_Accel_y,IMU_Accel_z,IMU_Gyro_x,IMU_Gyro_y,IMU_Gyro_z,IMU_Mag_x,IMU_Mag_y,IMU_Mag_z,temperature,timestamp', + }, CALL_LOG: { ANDROID_API: b'timestamp,UTC time,hashed phone number,call type,duration in seconds', IOS_API: b'timestamp,UTC time,hashed phone number,call type,duration in seconds', # android-only data stream From 6ce36d84aa5276fd2a62205bd463983673f1d28d Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Mon, 2 Dec 2024 11:53:14 -0600 Subject: [PATCH 12/14] refactor: omniring model design --- .../0133_devicesettings_omniring_and_more.py | 11 +++----- database/study_models.py | 25 +++++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/database/migrations/0133_devicesettings_omniring_and_more.py b/database/migrations/0133_devicesettings_omniring_and_more.py index 78334a743..723f549c3 100644 --- a/database/migrations/0133_devicesettings_omniring_and_more.py +++ b/database/migrations/0133_devicesettings_omniring_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.15 on 2024-11-26 20:40 +# Generated by Django 4.2.15 on 2024-12-02 17:36 import django.core.validators from django.db import migrations, models @@ -18,19 +18,14 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='devicesettings', - name='omniring_global_offset_seconds', - field=models.PositiveIntegerField(default=0), + name='omniring_off_duration_seconds', + field=models.PositiveIntegerField(default=600, validators=[django.core.validators.MinValueValidator(1)]), ), migrations.AddField( model_name='devicesettings', name='omniring_on_duration_seconds', field=models.PositiveIntegerField(default=60, validators=[django.core.validators.MinValueValidator(1)]), ), - migrations.AddField( - model_name='devicesettings', - name='omniring_total_duration_seconds', - field=models.PositiveIntegerField(default=300, validators=[django.core.validators.MinValueValidator(1)]), - ), migrations.AddField( model_name='summarystatisticdaily', name='beiwe_omniring_bytes', diff --git a/database/study_models.py b/database/study_models.py index a4e27810d..f01d03fae 100644 --- a/database/study_models.py +++ b/database/study_models.py @@ -24,7 +24,7 @@ # this is an import hack to improve IDE assistance try: from database.models import (ChunkRegistry, DashboardColorSetting, FileToProcess, Intervention, - Participant, ParticipantFieldValue, Researcher, StudyRelation, Survey) + Participant, ParticipantFieldValue, Researcher, StudyRelation, Survey) except ImportError: pass @@ -61,9 +61,11 @@ class Study(TimestampedModel, ObjectIDModel): easy_enrollment = models.BooleanField(default=False) # Researcher security settings - password_minimum_length = models.PositiveIntegerField(default=8, validators=[MinValueValidator(8), MaxValueValidator(20)]) + password_minimum_length = models.PositiveIntegerField(default=8, + validators=[MinValueValidator(8), MaxValueValidator(20)]) password_max_age_enabled = models.BooleanField(default=False) - password_max_age_days = models.PositiveIntegerField(default=365, validators=[MinValueValidator(30), MaxValueValidator(365)]) + password_max_age_days = models.PositiveIntegerField(default=365, + validators=[MinValueValidator(30), MaxValueValidator(365)]) mfa_required = models.BooleanField(default=False) # related field typings (IDE halp) @@ -111,9 +113,9 @@ def get_all_studies_by_name(cls) -> QuerySet[Study]: @classmethod def _get_administered_studies_by_name(cls, researcher) -> QuerySet[Study]: return cls.get_all_studies_by_name().filter( - study_relations__researcher=researcher, - study_relations__relationship=ResearcherRole.study_admin, - ) + study_relations__researcher=researcher, + study_relations__relationship=ResearcherRole.study_admin, + ) @classmethod def get_researcher_studies_by_name(cls, researcher) -> QuerySet[Study]: @@ -223,18 +225,21 @@ def export(self) -> Dict[str, Any]: accelerometer_off_duration_seconds = models.PositiveIntegerField(default=10, validators=[MinValueValidator(1)]) accelerometer_on_duration_seconds = models.PositiveIntegerField(default=10, validators=[MinValueValidator(1)]) accelerometer_frequency = models.PositiveIntegerField(default=10, validators=[MinValueValidator(1)]) - ambient_audio_off_duration_seconds = models.PositiveIntegerField(default=10*60, validators=[MinValueValidator(1)]) - ambient_audio_on_duration_seconds = models.PositiveIntegerField(default=10*60, validators=[MinValueValidator(1)]) + ambient_audio_off_duration_seconds = models.PositiveIntegerField(default=10 * 60, validators=[MinValueValidator(1)]) + ambient_audio_on_duration_seconds = models.PositiveIntegerField(default=10 * 60, validators=[MinValueValidator(1)]) ambient_audio_bitrate = models.PositiveIntegerField(default=24000, validators=[MinValueValidator(16000)]) ambient_audio_sampling_rate = models.PositiveIntegerField(default=44100, validators=[MinValueValidator(16000)]) bluetooth_on_duration_seconds = models.PositiveIntegerField(default=60, validators=[MinValueValidator(1)]) bluetooth_total_duration_seconds = models.PositiveIntegerField(default=300, validators=[MinValueValidator(1)]) bluetooth_global_offset_seconds = models.PositiveIntegerField(default=0) + omniring_off_duration_seconds = models.PositiveIntegerField(default=600, validators=[MinValueValidator(1)]) omniring_on_duration_seconds = models.PositiveIntegerField(default=60, validators=[MinValueValidator(1)]) omniring_total_duration_seconds = models.PositiveIntegerField(default=300, validators=[MinValueValidator(1)]) omniring_global_offset_seconds = models.PositiveIntegerField(default=0) - check_for_new_surveys_frequency_seconds = models.PositiveIntegerField(default=3600, validators=[MinValueValidator(30)]) - create_new_data_files_frequency_seconds = models.PositiveIntegerField(default=15 * 60, validators=[MinValueValidator(30)]) + check_for_new_surveys_frequency_seconds = models.PositiveIntegerField(default=3600, + validators=[MinValueValidator(30)]) + create_new_data_files_frequency_seconds = models.PositiveIntegerField(default=15 * 60, + validators=[MinValueValidator(30)]) gps_off_duration_seconds = models.PositiveIntegerField(default=600, validators=[MinValueValidator(1)]) gps_on_duration_seconds = models.PositiveIntegerField(default=60, validators=[MinValueValidator(1)]) seconds_before_auto_logout = models.PositiveIntegerField(default=600, validators=[MinValueValidator(1)]) From 7185fca53f6ba169a925de5210fb9caaef5d7d79 Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Mon, 2 Dec 2024 11:53:23 -0600 Subject: [PATCH 13/14] fix: omniring front end design --- frontend/templates/study_device_settings.html | 63 ++++++------------- 1 file changed, 20 insertions(+), 43 deletions(-) diff --git a/frontend/templates/study_device_settings.html b/frontend/templates/study_device_settings.html index a55caa353..eedcb2130 100755 --- a/frontend/templates/study_device_settings.html +++ b/frontend/templates/study_device_settings.html @@ -334,52 +334,29 @@

Data Stream Settings

value="{{ settings['bluetooth_global_offset_seconds'] }}" {{ "disabled" if readonly }}> - - {# OmniRing #} -
- The OmniRing data stream is synchonized across all devices in your study. It scans at a specific time - within a defined period. An On duration of 5 minutes with a Total duration of 1 hour - and a global offset of 15 minutes will trigger a bluetooth scan for 5 minutes once an hour, - every hour, at 15 minutes past the hour. + + {# OmniRing #} +
+
+ + +
-
-
+
+
+
+ +
- -
-
- - -
-
-
-
- - -
-
-
-
- - -
-
-
-
- - -
+
+
+
+ +
+
{% if not readonly %} From 4deed21f933a46ded17bc4074a9313d63598f37b Mon Sep 17 00:00:00 2001 From: Reyva Babtista Date: Mon, 2 Dec 2024 12:24:17 -0600 Subject: [PATCH 14/14] fix: omniring model design --- constants/study_constants.py | 4 +--- database/study_models.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/constants/study_constants.py b/constants/study_constants.py index 6fd68187e..808981a57 100644 --- a/constants/study_constants.py +++ b/constants/study_constants.py @@ -81,9 +81,8 @@ "bluetooth_on_duration_seconds", "bluetooth_total_duration_seconds", "bluetooth_global_offset_seconds", + "omniring_off_duration_seconds", "omniring_on_duration_seconds", - "omniring_total_duration_seconds", - "omniring_global_offset_seconds", "check_for_new_surveys_frequency_seconds", "create_new_data_files_frequency_seconds", "gps_off_duration_seconds", @@ -101,7 +100,6 @@ "heartbeat_timer_minutes", ] - # Surveys have several types of questions and some special symbols # Survey Question Types diff --git a/database/study_models.py b/database/study_models.py index f01d03fae..54122ac88 100644 --- a/database/study_models.py +++ b/database/study_models.py @@ -234,8 +234,6 @@ def export(self) -> Dict[str, Any]: bluetooth_global_offset_seconds = models.PositiveIntegerField(default=0) omniring_off_duration_seconds = models.PositiveIntegerField(default=600, validators=[MinValueValidator(1)]) omniring_on_duration_seconds = models.PositiveIntegerField(default=60, validators=[MinValueValidator(1)]) - omniring_total_duration_seconds = models.PositiveIntegerField(default=300, validators=[MinValueValidator(1)]) - omniring_global_offset_seconds = models.PositiveIntegerField(default=0) check_for_new_surveys_frequency_seconds = models.PositiveIntegerField(default=3600, validators=[MinValueValidator(30)]) create_new_data_files_frequency_seconds = models.PositiveIntegerField(default=15 * 60,