Merge pull request #279 from Qrlew/upgrade_deps

Upgrade dependencies
Qrlew · May 27, 2024 · 897e14d · 897e14d
2 parents f4bfbc3 + d6b49a6
commit 897e14d
Show file tree

Hide file tree

Showing 10 changed files with 325 additions and 371 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - use a column present in the schema as privacy unit weight
 - added flag to PrivacyUnit to deactivate the md5 of the privacy_unit field
 - added new rewriting rules: PUP -> PUP for the Reduce and (Pub, PUP) -> PUP, (PUP, Pub) -> PUP for the Join 
+- upgrade dependencies
 
 ## [0.9.17] - 2024-03-28
 ### Fixed

diff --git a/Cargo.toml b/Cargo.toml
@@ -19,34 +19,34 @@ rand = "0.8"
 log = "0.4"
 env_logger = "0.11"
 colored = "2.0"
-itertools = "0.12"
-paste = "1.0.7"
+itertools = "0.13"
+paste = "1.0.15"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 chrono = { version = "0.4", features = ["serde"] }
-sqlparser = "0.43"
+sqlparser = "0.46"
 dot = "0.1"
-base64 = "0.21"
-rusqlite = { version = "0.30", features = ["chrono"], optional = true }
+base64 = "0.22"
+rusqlite = { version = "0.31", features = ["chrono"], optional = true }
 postgres = { version = "0.19", features = ["with-chrono-0_4"] }
 r2d2 = "0.8"
 r2d2_postgres = "0.18"
-rust_decimal = { version = "1.29", features = [ "tokio-pg" ] }
+rust_decimal = { version = "1.35", features = [ "tokio-pg" ] }
 statrs = "0.16.0"
 sqlx = { version = "0.6.3", features = ["mssql", "runtime-tokio-native-tls", "offline", "any"], optional = true }
 tokio = { version = "1", features = ["full"], optional = true }
 
 # bigquery dependencies
-gcp-bigquery-client = { version = "=0.18.0", optional = true }
-wiremock = { version = "0.5.19", optional = true }
+gcp-bigquery-client = { version = "=0.20.0", optional = true }
+wiremock = { version = "0.6", optional = true }
 tempfile = { version = "3.6.0", optional = true }
-yup-oauth2 = { version = "=8.3.2", optional = true } # 8.3.3 makes the compiling of gcp-bigquery-client fail
+yup-oauth2 = { version = "9.0", optional = true }
 
 [features]
 # Use SQLite for tests and examples
 sqlite = ["dep:rusqlite"]
 mssql = ["dep:sqlx", "dep:tokio"]
-bigquery = ["dep:gcp-bigquery-client", "dep:wiremock", "dep:tempfile", "dep:yup-oauth2"]
+bigquery = ["dep:gcp-bigquery-client", "dep:wiremock", "dep:tempfile", "dep:yup-oauth2", "dep:tokio"]
 # Tests
 checked_injections = []
 # Multiplicity features are tested on large datasets (may take too much memory)

diff --git a/examples/ast.rs b/examples/ast.rs
@@ -62,6 +62,9 @@ fn build_ast() -> Result<(), &'static str> {
             having: None,
             qualify: None,
             named_window: vec![],
+            window_before_qualify: false,
+            value_table_mode: None,
+            connect_by: None,
         }))),
         order_by: vec![],
         limit: None,
@@ -80,6 +83,7 @@ fn build_ast() -> Result<(), &'static str> {
         },
         query: Box::new(query.clone()),
         from: None,
+        materialized: None
     };
     println!("{}", cte);
     let cte = Cte {
@@ -89,6 +93,7 @@ fn build_ast() -> Result<(), &'static str> {
         },
         query: Box::new(query),
         from: Some(Ident::new("fro")),
+        materialized: None
     };
     println!("{}", cte);
 

diff --git a/src/dialect_translation/bigquery.rs b/src/dialect_translation/bigquery.rs
@@ -24,6 +24,7 @@ impl RelationToQueryTranslator for BigQueryTranslator {
             },
             query: Box::new(query),
             from: None,
+            materialized: None
         }
     }
     fn first(&self, expr: &expr::Expr) -> ast::Expr {
@@ -55,6 +56,7 @@ impl RelationToQueryTranslator for BigQueryTranslator {
             expr: Box::new(ast_expr),
             data_type: ast::DataType::String(None),
             format: None,
+            kind: ast::CastKind::Cast,
         }
     }
     fn substr(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {

diff --git a/src/dialect_translation/mod.rs b/src/dialect_translation/mod.rs
@@ -152,6 +152,9 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                         having: None,
                         qualify: None,
                         named_window: vec![],
+                        window_before_qualify: false,
+                        value_table_mode: None,
+                        connect_by: None
                     }))),
                     order_by,
                     limit,
@@ -216,7 +219,7 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
             }
 
             fn insert(&self, prefix: &str, table: &Table) -> ast::Statement {
-                ast::Statement::Insert {
+                ast::Statement::Insert(ast::Insert {
                     or: None,
                     into: true,
                     table_name: ast::ObjectName(self.identifier( &(table.path().clone().into()) )),
@@ -251,7 +254,8 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                     ignore: false,
                     replace_into: false,
                     priority: None,
-                }
+                    insert_alias: None,
+                })
             }
 
             fn cte(
@@ -264,6 +268,7 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                     alias: ast::TableAlias {name, columns},
                     query: Box::new(query),
                     from: None,
+                    materialized: None
                 }
             }
             fn join_projection(&self, join: &Join) -> Vec<ast::SelectItem> {
@@ -773,17 +778,21 @@ pub trait QueryToRelationTranslator {
 
     fn try_function_args(
         &self,
-        args: Vec<ast::FunctionArg>,
+        args: ast::FunctionArguments,
         context: &Hierarchy<Identifier>,
     ) -> Result<Vec<expr::Expr>> {
-        args.iter()
-            .map(|func_arg| match func_arg {
-                ast::FunctionArg::Named { name: _, arg } => {
-                    self.try_function_arg_expr(arg, context)
-                }
-                ast::FunctionArg::Unnamed(arg) => self.try_function_arg_expr(arg, context),
-            })
-            .collect()
+        match args {
+            ast::FunctionArguments::None
+            | ast::FunctionArguments::Subquery(_) => Ok(vec![]),
+            ast::FunctionArguments::List(arg_list) => arg_list.args
+                .iter()
+                .map(|func_arg| match func_arg {
+                    ast::FunctionArg::Named {arg, .. } | ast::FunctionArg::Unnamed(arg) => {
+                        self.try_function_arg_expr(arg, context)
+                    }
+                })
+                .collect(),
+        }
     }
 
     fn try_function_arg_expr(
@@ -816,17 +825,21 @@ fn function_builder(name: &str, exprs: Vec<ast::Expr>, distinct: bool) -> ast::E
         .collect();
     let function_name = name.to_uppercase();
     let name = ast::ObjectName(vec![ast::Ident::from(&function_name[..])]);
-    let funtion = ast::Function {
-        name,
+    let ast_distinct = if distinct {Some(ast::DuplicateTreatment::Distinct)} else {None};
+    let func_args_list = ast::FunctionArgumentList {
+        duplicate_treatment: ast_distinct,
         args: function_args,
+        clauses: vec![],
+    };
+    let function = ast::Function {
+        name,
+        args: ast::FunctionArguments::List(func_args_list),
         over: None,
-        distinct: distinct,
-        special: false,
-        order_by: vec![],
         filter: None,
         null_treatment: None,
+        within_group: vec![]
     };
-    ast::Expr::Function(funtion)
+    ast::Expr::Function(function)
 }
 
 // AST CAST expression builder
@@ -835,6 +848,7 @@ fn cast_builder(expr: ast::Expr, as_type: ast::DataType) -> ast::Expr {
         expr: Box::new(expr),
         data_type: as_type,
         format: None,
+        kind: ast::CastKind::Cast,
     }
 }
 

diff --git a/src/dialect_translation/mssql.rs b/src/dialect_translation/mssql.rs
@@ -59,69 +59,37 @@ impl RelationToQueryTranslator for MsSqlTranslator {
 
     /// Converting MD5(X) to CONVERT(VARCHAR(MAX), HASHBYTES('MD5', X), 2)
     fn md5(&self, expr: &expr::Expr) -> ast::Expr {
-        // In sql parser 0.4 it has been introduced CONVERT as an expression
-        // but if doesn't allow for a style argument (see the doc here:
-        // https://learn.microsoft.com/fr-fr/sql/t-sql/functions/cast-and-convert-transact-sql?view=sql-server-ver16)
-        // which is needed for the convertion.
-        // So we can't parse the fllowing:
-        // let input_sql = r#"
-        // SELECT CONVERT(X, VARCHAR(MAX)) FROM table_x
-        // "#;
-        // TODO: If we need to parse it, maybe we can use something like CONVERT_SARUS such that it can be
-        // seen as a function from the parser. But maybe we don't need to parse it.
         let ast_expr = self.expr(expr);
-        let ast_expr_as_function_arg =
-            ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(ast_expr));
+
+        // Construct HASHBYTES('MD5', X)
         let md5_literal = ast::Expr::Value(ast::Value::SingleQuotedString("MD5".to_string()));
         let md5_literal_as_function_arg =
-            ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(md5_literal));
-        let hash_byte = ast::Expr::Function(ast::Function {
-            name: ast::ObjectName(vec![ast::Ident::from("HASHBYTES")]),
-            args: vec![md5_literal_as_function_arg, ast_expr_as_function_arg],
-            over: None,
-            distinct: false,
-            special: false,
-            order_by: vec![],
-            filter: None,
-            null_treatment: None,
-        });
-
-        let hash_byte_as_function_arg =
-            ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(hash_byte));
+        ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(md5_literal));
+        let ast_expr_as_function_arg =
+            ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(ast_expr));
 
-        // VARCHAR(MAX) is treated as a function with MAX argument as identifier.
-        let varchartype_expr = ast::Expr::Function(ast::Function {
-            name: ast::ObjectName(vec![ast::Ident::from("VARCHAR")]),
-            args: vec![ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(
-                ast::Expr::Identifier(ast::Ident::from("MAX")),
-            ))],
+        let func_args_list = ast::FunctionArgumentList {
+            duplicate_treatment: None,
+            args: vec![md5_literal_as_function_arg, ast_expr_as_function_arg],
+            clauses: vec![],
+        };
+        let hash_byte_expr = ast::Expr::Function(ast::Function {
+            name: ast::ObjectName(vec![ast::Ident::from("HASHBYTES")]),
+            args: ast::FunctionArguments::List(func_args_list),
             over: None,
-            distinct: false,
-            special: false,
-            order_by: vec![],
             filter: None,
             null_treatment: None,
+            within_group: vec![],
         });
 
-        let varchartype_as_function_arg =
-            ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(varchartype_expr));
-
-        ast::Expr::Function(ast::Function {
-            name: ast::ObjectName(vec![ast::Ident::from("CONVERT")]),
-            args: vec![
-                varchartype_as_function_arg,
-                hash_byte_as_function_arg,
-                ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(ast::Expr::Value(
-                    ast::Value::Number("2".to_string(), false),
-                ))),
-            ],
-            over: None,
-            distinct: false,
-            special: false,
-            order_by: vec![],
-            filter: None,
-            null_treatment: None,
-        })
+        // Construct the CONVERT expr
+        ast::Expr::Convert {
+            expr: Box::new(hash_byte_expr),
+            data_type: Some(ast::DataType::Varchar(Some(ast::CharacterLength::Max))),
+            charset: None,
+            target_before_value: true,
+            styles: vec![ast::Expr::Value(ast::Value::Number("2".to_string(), false))]
+        }
     }
 
     fn cast_as_boolean(&self, expr: &expr::Expr) -> ast::Expr {
@@ -135,8 +103,9 @@ impl RelationToQueryTranslator for MsSqlTranslator {
         let ast_expr = self.expr(expr);
         ast::Expr::Cast {
             expr: Box::new(ast_expr),
-            data_type: ast::DataType::Nvarchar(Some(255)),
+            data_type: ast::DataType::Nvarchar(Some(ast::CharacterLength::IntegerLength {length: 255, unit:None })),
             format: None,
+            kind: ast::CastKind::Cast,
         }
     }
     fn substr(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
@@ -219,6 +188,9 @@ impl RelationToQueryTranslator for MsSqlTranslator {
                 having: None,
                 qualify: None,
                 named_window: vec![],
+                window_before_qualify: false,
+                value_table_mode: None,
+                connect_by: None
             }))),
             order_by,
             limit: None,
@@ -317,19 +289,27 @@ impl QueryToRelationTranslator for MsSqlTranslator {
         context: &Hierarchy<expr::Identifier>,
     ) -> Result<expr::Expr> {
         // need to check func.args:
-        let args = &func.args;
+        let args = match &func.args {
+            ast::FunctionArguments::None
+            | ast::FunctionArguments::Subquery(_) => vec![],
+            ast::FunctionArguments::List(l) => l.args.iter().collect(),
+        };
         // We expect 2 args
         if args.len() != 3 {
             let expr = ast::Expr::Function(func.clone());
             expr::Expr::try_from(expr.with(context))
         } else {
             let is_first_arg_valid = is_varchar_valid(&args[0]);
             let is_last_arg_valid = is_literal_two_arg(&args[2]);
-            let extract_x_arg = extract_hashbyte_expression_if_valid(&args[1]);
+            let extract_x_arg = extract_hashbyte_expression_if_valid(&args[1]); 
             if is_first_arg_valid && is_last_arg_valid && extract_x_arg.is_some() {
-                // Code to execute when both booleans are true and the option is Some
+                let function_args = ast::FunctionArgumentList {
+                    duplicate_treatment: None,
+                    args: vec![extract_x_arg.unwrap()],
+                    clauses: vec![],
+                };
                 let converted_x_arg =
-                    self.try_function_args(vec![extract_x_arg.unwrap()], context)?;
+                    self.try_function_args(ast::FunctionArguments::List(function_args), context)?;
                 Ok(expr::Expr::md5(converted_x_arg[0].clone()))
             } else {
                 let expr = ast::Expr::Function(func.clone());
@@ -378,8 +358,13 @@ fn extract_hashbyte_expression_if_valid(func_arg: &ast::FunctionArg) -> Option<a
         ast::FunctionArg::Unnamed(fargexpr) => match fargexpr {
             ast::FunctionArgExpr::Expr(e) => match e {
                 ast::Expr::Function(f) => {
-                    if (f.name == expected_f_name) && (f.args[0] == expected_first_arg) {
-                        Some(f.args[1].clone())
+                    let arg_vec = match &f.args {
+                        ast::FunctionArguments::None
+                        | ast::FunctionArguments::Subquery(_) => vec![],
+                        ast::FunctionArguments::List(func_args) => func_args.args.iter().collect(),
+                    };
+                    if (f.name == expected_f_name) && (arg_vec[0] == &expected_first_arg) {
+                        Some(arg_vec[1].clone())
                     } else {
                         None
                     }
@@ -394,7 +379,7 @@ fn extract_hashbyte_expression_if_valid(func_arg: &ast::FunctionArg) -> Option<a
 // method to override DataType -> ast::DataType
 fn translate_data_type(dtype: DataType) -> ast::DataType {
     match dtype {
-        DataType::Text(_) => ast::DataType::Nvarchar(Some(255)),
+        DataType::Text(_) => ast::DataType::Nvarchar(Some(ast::CharacterLength::IntegerLength { length: 255, unit: None})),
         //DataType::Boolean(_) => Boolean should be displayed as BIT for MSSQL,
         // SQLParser doesn't support the BIT DataType (mssql equivalent of bool)
         DataType::Optional(o) => translate_data_type(o.data_type().clone()),