hacky temp table

Northeastern-Electric-Racing · Jan 7, 2025 · b8a5865 · b8a5865
1 parent 53aea54
commit b8a5865
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 12 deletions.
diff --git a/scylla-server/migrations/2024-11-10-031516_create_all/down.sql b/scylla-server/migrations/2024-11-10-031516_create_all/down.sql
@@ -6,3 +6,4 @@ ALTER TABLE "data" DROP CONSTRAINT "data_dataTypeName_fkey";
 DROP TABLE "data";
 DROP TABLE "dataType";
 DROP TABLE "run";
+DROP TABLE "data_temp";
diff --git a/scylla-server/migrations/2024-11-10-031516_create_all/up.sql b/scylla-server/migrations/2024-11-10-031516_create_all/up.sql
@@ -25,6 +25,15 @@ CREATE TABLE "data" (
 -- SELECT * FROM create_hypertable("data", by_range("time"));
 -- SELECT * FROM add_dimension("data", by_hash("dataTypeNmae", 4));
 
+-- CreateTable
+CREATE TABLE "data_temp" (
+    "values" REAL [] NOT NULL check ("values" <> '{}' AND array_position("values", NULL) IS NULL),
+    "dataTypeName" TEXT NOT NULL,
+    "time" TIMESTAMPTZ NOT NULL,
+    "runId" INTEGER NOT NULL,
+
+    PRIMARY KEY("time", "dataTypeName")
+);
 
 -- CreateTable
 CREATE TABLE "dataType" (

diff --git a/scylla-server/src/models.rs b/scylla-server/src/models.rs
@@ -22,6 +22,7 @@ pub struct Data {
 /// but the overhead of mapping Vec<f32> to Vec<Option<f32>> is non-negligible.
 #[derive(Insertable)]
 #[diesel(table_name = crate::schema::data)]
+#[diesel(table_name = crate::schema::data_temp)]
 #[diesel(belongs_to(DataType, foreign_key = dataTypeName))]
 #[diesel(treat_none_as_default_value = false)]
 #[diesel(check_for_backend(diesel::pg::Pg))]

diff --git a/scylla-server/src/schema.rs b/scylla-server/src/schema.rs
@@ -17,6 +17,15 @@ diesel::table! {
     }
 }
 
+diesel::table! {
+    data_temp (time, dataTypeName) {
+        values -> Array<Nullable<Float4>>,
+        dataTypeName -> Text,
+        time -> Timestamptz,
+        runId -> Int4,
+    }
+}
+
 diesel::table! {
     run (id) {
         id -> Int4,
@@ -32,4 +41,9 @@ diesel::table! {
 diesel::joinable!(data -> dataType (dataTypeName));
 diesel::joinable!(data -> run (runId));
 
-diesel::allow_tables_to_appear_in_same_query!(data, dataType, run,);
+diesel::allow_tables_to_appear_in_same_query!(
+    data,
+    dataType,
+    data_temp,
+    run,
+);
diff --git a/scylla-server/src/services/data_service.rs b/scylla-server/src/services/data_service.rs
@@ -1,10 +1,9 @@
-use diesel::prelude::*;
-
 use crate::{
     models::{Data, DataInsert},
-    schema::data::dsl::*,
     ClientData, Database,
 };
+use diesel::prelude::*;
+use tracing::debug;
 
 use super::DbError;
 
@@ -20,6 +19,7 @@ pub async fn get_data(
 ) -> Result<Vec<Data>, DbError> {
     Ok(db
         .interact(move |conn| {
+            use crate::schema::data::dsl::*;
             data.filter(runId.eq(run_id).and(dataTypeName.eq(data_type_name)))
                 .load::<Data>(conn)
         })
@@ -36,6 +36,7 @@ pub async fn get_data(
 pub async fn add_data(db: Database, client_data: ClientData) -> Result<Data, DbError> {
     Ok(db
         .interact(move |conn| {
+            use crate::schema::data::dsl::*;
             diesel::insert_into(data)
                 .values(Into::<DataInsert>::into(client_data))
                 .get_result(conn)
@@ -46,6 +47,7 @@ pub async fn add_data(db: Database, client_data: ClientData) -> Result<Data, DbE
 pub async fn add_many(db: Database, client_data: Vec<ClientData>) -> Result<usize, DbError> {
     Ok(db
         .interact(move |conn| {
+            use crate::schema::data::dsl::*;
             diesel::insert_into(data)
                 .values(
                     client_data
@@ -62,14 +64,36 @@ pub async fn add_many(db: Database, client_data: Vec<ClientData>) -> Result<usiz
 pub async fn copy_many(db: Database, client_data: Vec<ClientData>) -> Result<usize, DbError> {
     Ok(db
         .interact(move |conn| {
-            diesel::copy_from(data)
-                .from_insertable(
-                    client_data
-                        .into_iter()
-                        .map(Into::<DataInsert>::into)
-                        .collect::<Vec<DataInsert>>(),
-                )
-                .execute(conn)
+            let mut res;
+            let select;
+            {
+                use crate::schema::data_temp::dsl::*;
+                res = diesel::copy_from(data_temp)
+                    .from_insertable(
+                        client_data
+                            .into_iter()
+                            .map(Into::<DataInsert>::into)
+                            .collect::<Vec<DataInsert>>(),
+                    )
+                    .execute(conn)?;
+                select = data_temp.select(data_temp::all_columns());
+            }
+            debug!("Copied {} to temp table", res);
+            {
+                use crate::schema::data::dsl::*;
+                res = diesel::insert_into(data)
+                    .values(select)
+                    .on_conflict_do_nothing()
+                    .execute(conn)?;
+                debug!("Inserted {} to data table", res);
+            }
+            {
+                use crate::schema::data_temp::dsl::*;
+                diesel::delete(data_temp).execute(conn)?;
+                debug!("Cleared temporary table!");
+            }
+
+            Ok::<usize, diesel::result::Error>(res)
         })
         .await??)
 }