From ecaeef9e7eb020739b48faacba24a25d9c15e14f Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Tue, 9 Jan 2024 02:08:53 +0000 Subject: [PATCH 01/12] README: Update build status badge to reference correct repository (#38) In passing, sync up the src/lib.rs file with the README, as its intended. --- README.md | 4 ++-- src/lib.rs | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f62a158..d408e99 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ pg_query.rs   [![Build Status]][actions] [![Latest Version]][crates.io] [![Docs Badge]][docs] =========== -[Build Status]: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpganalyze%2Fpg_query%2Fbadge&label=build&logo=none -[actions]: https://actions-badge.atrox.dev/pganalyze/pg_query/goto +[Build Status]: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpganalyze%2Fpg_query.rs%2Fbadge%3Fref%3Dmain&style=flat&label=build&logo=none +[actions]: https://actions-badge.atrox.dev/pganalyze/pg_query.rs/goto?ref=main [Latest Version]: https://img.shields.io/crates/v/pg_query.svg [crates.io]: https://crates.io/crates/pg_query [Docs Badge]: https://docs.rs/pg_query/badge.svg diff --git a/src/lib.rs b/src/lib.rs index 01bb40e..1555f8d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,17 +1,20 @@ //! Rust pg_query   [![Build Status]][actions] [![Latest Version]][crates.io] [![Docs Badge]][docs] //! =========== //! -//! [Build Status]: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpaupino%2Fpg_query%2Fbadge&label=build&logo=none -//! [actions]: https://actions-badge.atrox.dev/paupino/pg_query/goto +//! [Build Status]: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpganalyze%2Fpg_query.rs%2Fbadge%3Fref%3Dmain&style=flat&label=build&logo=none +//! [actions]: https://actions-badge.atrox.dev/pganalyze/pg_query.rs/goto?ref=main //! [Latest Version]: https://img.shields.io/crates/v/pg_query.svg //! [crates.io]: https://crates.io/crates/pg_query //! [Docs Badge]: https://docs.rs/pg_query/badge.svg //! [docs]: https://docs.rs/pg_query //! -//! PostgreSQL parser that uses the [actual PostgreSQL server source]((https://github.com/pganalyze/libpg_query)) to parse -//! SQL queries and return the internal PostgreSQL parse tree. +//! This Rust library uses the actual PostgreSQL server source to parse SQL queries and return the internal PostgreSQL parse tree. //! -//! Warning! This library is in early stages of development so any APIs exposed are subject to change. +//! It also allows you to normalize queries (replacing constant values with $1, etc.) and parse these normalized queries into a parse tree again. +//! +//! When you build this library, it builds parts of the PostgreSQL server source (see [libpg_query](https://github.com/pganalyze/libpg_query)), and then statically links it into this library. +//! +//! You can find further examples and a longer rationale for the original Ruby implementation [here](https://pganalyze.com/blog/parse-postgresql-queries-in-ruby.html). The Rust version tries to have a very similar API. //! //! ## Getting started //! @@ -19,7 +22,7 @@ //! //! ```toml //! [dependencies] -//! pg_query = "0.7" +//! pg_query = "5.0" //! ``` //! //! # Example: Parsing a query From f18a47c430cbd6fa616f36f064675007dc76ad5f Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Mon, 1 Jan 2024 12:05:53 -0800 Subject: [PATCH 02/12] Add support for running on Windows --- .gitattributes | 1 + .github/workflows/main.yml | 7 +++++-- Cargo.lock | 2 ++ Cargo.toml | 2 ++ build.rs | 41 +++++++++++++++++++++++++------------- libpg_query | 2 +- 6 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b11351b --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/data/* binary diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37595ea..783c1ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,14 +11,17 @@ name: Continuous integration jobs: ci: env: - RUSTFLAGS: ${{ matrix.rust == 'nightly' && '-Z sanitizer=leak' || '' }} - runs-on: ubuntu-latest + RUSTFLAGS: ${{ matrix.rust == 'nightly' && matrix.os == 'ubuntu-latest' && '-Z sanitizer=leak' || '' }} strategy: fail-fast: false matrix: rust: - stable - nightly + os: + - ubuntu-latest + - windows-latest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/Cargo.lock b/Cargo.lock index fde4156..e4e5e80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,9 +393,11 @@ name = "pg_query" version = "5.0.0" dependencies = [ "bindgen", + "cc", "clippy", "easy-parallel", "fs_extra", + "glob", "itertools", "pretty_assertions", "prost", diff --git a/Cargo.toml b/Cargo.toml index c73e21a..f8399ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,8 @@ bindgen = "0.66.1" clippy = { version = "0.0.302", optional = true } prost-build = "0.10.4" fs_extra = "1.2.0" +cc = "1.0.83" +glob = "0.3.1" [dev-dependencies] easy-parallel = "3.2.0" diff --git a/build.rs b/build.rs index 65765d0..9a68551 100644 --- a/build.rs +++ b/build.rs @@ -2,6 +2,7 @@ #![cfg_attr(feature = "clippy", plugin(clippy))] use fs_extra::dir::CopyOptions; +use glob::glob; use std::env; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -12,12 +13,11 @@ static LIBRARY_NAME: &str = "pg_query"; fn main() -> Result<(), Box> { let out_dir = PathBuf::from(env::var("OUT_DIR")?); let build_path = Path::new(".").join(SOURCE_DIRECTORY); - let makefile_path = build_path.join("Makefile"); let out_header_path = out_dir.join(LIBRARY_NAME).with_extension("h"); let out_protobuf_path = out_dir.join("protobuf"); + let target = env::var("TARGET").unwrap(); // Configure cargo through stdout - println!("cargo:rerun-if-changed={}", makefile_path.display()); // Includes version number println!("cargo:rustc-link-search=native={}", out_dir.display()); println!("cargo:rustc-link-lib=static={LIBRARY_NAME}"); @@ -35,18 +35,31 @@ fn main() -> Result<(), Box> { fs_extra::copy_items(&source_paths, &out_dir, ©_options)?; // Compile the C library. - let mut make = Command::new("make"); - - make.env_remove("PROFILE").arg("-C").arg(&out_dir).arg("build"); - - if env::var("PROFILE").unwrap() == "debug" { - make.arg("DEBUG=1"); - } - - let status = make.stdin(Stdio::null()).stdout(Stdio::inherit()).stderr(Stdio::inherit()).status()?; - - if !status.success() { - return Err("libpg_query compilation failed".into()); + if target.contains("msvc") { + // Rust on Windows may not have "make" or "nmake" installed + cc::Build::new() + .files(glob(out_dir.join("src/*.c").to_str().unwrap()).unwrap().map(|p| p.unwrap())) + .files(glob(out_dir.join("src/postgres/*.c").to_str().unwrap()).unwrap().map(|p| p.unwrap())) + .file(out_dir.join("vendor/protobuf-c/protobuf-c.c")) + .file(out_dir.join("vendor/xxhash/xxhash.c")) + .file(out_dir.join("protobuf/pg_query.pb-c.c")) + .include(out_dir.join(".")) + .include(out_dir.join("./vendor")) + .include(out_dir.join("./src/postgres/include")) + .include(out_dir.join("./src/include")) + .include(out_dir.join("./src/postgres/include/port/win32")) + .include(out_dir.join("./src/postgres/include/port/win32_msvc")) + .compile(LIBRARY_NAME); + } else { + let mut make = Command::new("make"); + make.env_remove("PROFILE").arg("-C").arg(&out_dir).arg("build"); + if env::var("PROFILE").unwrap() == "debug" { + make.arg("DEBUG=1"); + } + let status = make.stdin(Stdio::null()).stdout(Stdio::inherit()).stderr(Stdio::inherit()).status()?; + if !status.success() { + return Err("libpg_query compilation failed".into()); + } } // Generate bindings for Rust diff --git a/libpg_query b/libpg_query index 2a00188..1ec3894 160000 --- a/libpg_query +++ b/libpg_query @@ -1 +1 @@ -Subproject commit 2a0018867c20011fc7166767083c05965241140b +Subproject commit 1ec38940e5c6f09a4c1d17a46d839a881c4f2db7 From 0a2995acf2e456ff8a98564890dff085351786f0 Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Sat, 6 Jan 2024 21:42:23 -0800 Subject: [PATCH 03/12] Always build C library using "cc" crate This avoids unnecessarily going through the Makefile logic, which isn't necessary when we have build.rs. --- build.rs | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/build.rs b/build.rs index 9a68551..355f525 100644 --- a/build.rs +++ b/build.rs @@ -35,32 +35,28 @@ fn main() -> Result<(), Box> { fs_extra::copy_items(&source_paths, &out_dir, ©_options)?; // Compile the C library. - if target.contains("msvc") { - // Rust on Windows may not have "make" or "nmake" installed - cc::Build::new() - .files(glob(out_dir.join("src/*.c").to_str().unwrap()).unwrap().map(|p| p.unwrap())) - .files(glob(out_dir.join("src/postgres/*.c").to_str().unwrap()).unwrap().map(|p| p.unwrap())) - .file(out_dir.join("vendor/protobuf-c/protobuf-c.c")) - .file(out_dir.join("vendor/xxhash/xxhash.c")) - .file(out_dir.join("protobuf/pg_query.pb-c.c")) - .include(out_dir.join(".")) - .include(out_dir.join("./vendor")) - .include(out_dir.join("./src/postgres/include")) - .include(out_dir.join("./src/include")) - .include(out_dir.join("./src/postgres/include/port/win32")) - .include(out_dir.join("./src/postgres/include/port/win32_msvc")) - .compile(LIBRARY_NAME); - } else { - let mut make = Command::new("make"); - make.env_remove("PROFILE").arg("-C").arg(&out_dir).arg("build"); - if env::var("PROFILE").unwrap() == "debug" { - make.arg("DEBUG=1"); - } - let status = make.stdin(Stdio::null()).stdout(Stdio::inherit()).stderr(Stdio::inherit()).status()?; - if !status.success() { - return Err("libpg_query compilation failed".into()); + let mut build = cc::Build::new(); + build + .files(glob(out_dir.join("src/*.c").to_str().unwrap()).unwrap().map(|p| p.unwrap())) + .files(glob(out_dir.join("src/postgres/*.c").to_str().unwrap()).unwrap().map(|p| p.unwrap())) + .file(out_dir.join("vendor/protobuf-c/protobuf-c.c")) + .file(out_dir.join("vendor/xxhash/xxhash.c")) + .file(out_dir.join("protobuf/pg_query.pb-c.c")) + .include(out_dir.join(".")) + .include(out_dir.join("./vendor")) + .include(out_dir.join("./src/postgres/include")) + .include(out_dir.join("./src/include")) + .warnings(false); // Avoid unnecessary warnings, as they are already considered as part of libpg_query development + if env::var("PROFILE").unwrap() == "debug" || env::var("DEBUG").unwrap() == "1" { + build.define("USE_ASSERT_CHECKING", None); + } + if target.contains("windows") { + build.include(out_dir.join("./src/postgres/include/port/win32")); + if target.contains("msvc") { + build.include(out_dir.join("./src/postgres/include/port/win32_msvc")); } } + build.compile(LIBRARY_NAME); // Generate bindings for Rust bindgen::Builder::default() From f4ea6d15a9e5fa917f7f4bd23c1d0449740256ca Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Sun, 7 Jan 2024 10:08:46 -0800 Subject: [PATCH 04/12] Don't use actions-rs/toolchain@v1, it's unmaintained See https://github.com/actions-rs/toolchain/issues/216 --- .github/workflows/main.yml | 61 +++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 783c1ef..6979aeb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive @@ -34,39 +34,51 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - key: ${{ runner.os }}-${{ matrix.backend }}-cargo-${{ hashFiles('**/Cargo.toml') }} + key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Install rustup if needed + run: | + if ! command -v rustup &>/dev/null; then + curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused --location --silent --show-error --fail "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y + echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH + fi + if: runner.os != 'Windows' + shell: bash - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.rust }} - override: true + run: rustup toolchain install ${{ matrix.rust }} --profile minimal --no-self-update + shell: bash + + - name: Default to nightly if requested + run: rustup default nightly + if: matrix.rust == 'nightly' - name: Build pg_query - uses: actions-rs/cargo@v1 - with: - command: build + run: cargo build - name: Run tests - uses: actions-rs/cargo@v1 - with: - command: test + run: cargo test check_style: name: Check file formatting and style runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: recursive - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - components: clippy, rustfmt - override: true + - name: Install rustup if needed + run: | + if ! command -v rustup &>/dev/null; then + curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused --location --silent --show-error --fail "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y + echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH + fi + if: runner.os != 'Windows' + shell: bash + + - name: Install toolchain + run: rustup toolchain install stable --component clippy --component rustfmt --profile minimal --no-self-update + shell: bash - name: Cache cargo registry uses: actions/cache@v2 @@ -77,12 +89,7 @@ jobs: key: clippy-cargo-${{ hashFiles('**/Cargo.toml') }} - name: Check file formatting - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check + run: cargo fmt --all -- --check - name: Run clippy - uses: actions-rs/cargo@v1 - with: - command: clippy + run: cargo clippy From 3febe5b9fc5d79549f95606e830a92a8fa54fbc7 Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Sun, 7 Jan 2024 10:09:19 -0800 Subject: [PATCH 05/12] On Windows, test 64-bit GNU and 32-bit MSVC as well --- .github/workflows/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6979aeb..b8a55e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,6 +21,11 @@ jobs: os: - ubuntu-latest - windows-latest + include: + - rust: stable-x86_64-pc-windows-gnu + os: windows-latest + - rust: stable-i686-pc-windows-msvc + os: windows-latest runs-on: ${{ matrix.os }} steps: From f2ea0d7b82cf2e87166a4c13fb3666ddea9751b3 Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Wed, 10 Jan 2024 00:29:04 +0000 Subject: [PATCH 06/12] Add support for getting filter columns for a parsed query (#21) This works the same as the "#filter_columns" method in the pg_query Ruby gem, and returns the table name (if present) and column name for every column that's referenced in a JOIN or WHERE clause. To support this functionality, the nodes() and nodes_mut() functions gain a 4th value for each result item, indicating whether the current node can be considered a filter column. This is intended to be used when matching a ColumnRef node, to determine whether it is part of a WHERE/JOIN clause, or part of something else (e.g. target list, or ORDER BY, etc). --- src/node_enum.rs | 168 +++++++++++++++++++++-------------- src/parse_result.rs | 24 ++++- tests/filter_column_tests.rs | 105 ++++++++++++++++++++++ 3 files changed, 226 insertions(+), 71 deletions(-) create mode 100644 tests/filter_column_tests.rs diff --git a/src/node_enum.rs b/src/node_enum.rs index ddcb71e..665308c 100644 --- a/src/node_enum.rs +++ b/src/node_enum.rs @@ -19,11 +19,11 @@ impl NodeEnum { }) } - pub fn nodes(&self) -> Vec<(NodeRef, i32, Context)> { - let mut iter = vec![(self.to_ref(), 0, Context::None)]; + pub fn nodes(&self) -> Vec<(NodeRef, i32, Context, bool)> { + let mut iter = vec![(self.to_ref(), 0, Context::None, false)]; let mut nodes = Vec::new(); while !iter.is_empty() { - let (node, depth, context) = iter.remove(0); + let (node, depth, context, has_filter_columns) = iter.remove(0); let depth = depth + 1; match node { // @@ -32,33 +32,33 @@ impl NodeEnum { NodeRef::SelectStmt(s) => { s.target_list.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } }); if let Some(n) = &s.where_clause { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, true)); } } s.sort_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } }); s.group_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } }); if let Some(n) = &s.having_clause { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } }); } @@ -66,32 +66,32 @@ impl NodeEnum { Some(protobuf::SetOperation::SetopNone) => { s.from_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } }); } Some(protobuf::SetOperation::SetopUnion) => { if let Some(left) = s.larg.as_ref() { - iter.push((left.to_ref(), depth, Context::Select)); + iter.push((left.to_ref(), depth, Context::Select, false)); } if let Some(right) = s.rarg.as_ref() { - iter.push((right.to_ref(), depth, Context::Select)); + iter.push((right.to_ref(), depth, Context::Select, false)); } } Some(protobuf::SetOperation::SetopExcept) => { if let Some(left) = s.larg.as_ref() { - iter.push((left.to_ref(), depth, Context::Select)); + iter.push((left.to_ref(), depth, Context::Select, false)); } if let Some(right) = s.rarg.as_ref() { - iter.push((right.to_ref(), depth, Context::Select)); + iter.push((right.to_ref(), depth, Context::Select, false)); } } Some(protobuf::SetOperation::SetopIntersect) => { if let Some(left) = s.larg.as_ref() { - iter.push((left.to_ref(), depth, Context::Select)); + iter.push((left.to_ref(), depth, Context::Select, false)); } if let Some(right) = s.rarg.as_ref() { - iter.push((right.to_ref(), depth, Context::Select)); + iter.push((right.to_ref(), depth, Context::Select, false)); } } Some(protobuf::SetOperation::Undefined) | None => (), @@ -100,46 +100,46 @@ impl NodeEnum { NodeRef::InsertStmt(s) => { if let Some(n) = &s.select_stmt { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, false)); } } if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML)); + iter.push((rel.to_ref(), depth, Context::DML, false)); } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, false)); } }); } if let Some(n) = &s.on_conflict_clause { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, false)); } } NodeRef::UpdateStmt(s) => { s.target_list.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, false)); } }); s.where_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, true)); } }); s.from_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } }); if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML)); + iter.push((rel.to_ref(), depth, Context::DML, false)); } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, false)); } }); } @@ -147,40 +147,40 @@ impl NodeEnum { NodeRef::DeleteStmt(s) => { if let Some(n) = &s.where_clause { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, true)); } } if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML)); + iter.push((rel.to_ref(), depth, Context::DML, false)); } if let Some(clause) = &s.with_clause { clause.ctes.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, false)); } }); } s.using_clause.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::Select)); + iter.push((n.to_ref(), depth, Context::Select, false)); } }); } NodeRef::CommonTableExpr(s) => { if let Some(n) = &s.ctequery { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, false)); } } } NodeRef::CopyStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DML)); + iter.push((n.to_ref(), depth, Context::DML, false)); } } if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DML)); + iter.push((rel.to_ref(), depth, Context::DML, false)); } } // @@ -188,89 +188,89 @@ impl NodeEnum { // NodeRef::AlterTableStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::CreateStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::CreateTableAsStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL)); + iter.push((n.to_ref(), depth, Context::DDL, false)); } } if let Some(n) = &s.into { if let Some(rel) = n.rel.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } } NodeRef::TruncateStmt(s) => { s.relations.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL)); + iter.push((n.to_ref(), depth, Context::DDL, false)); } }); } NodeRef::ViewStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL)); + iter.push((n.to_ref(), depth, Context::DDL, false)); } } if let Some(rel) = s.view.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::IndexStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } s.index_params.iter().for_each(|n| { if let Some(NodeEnum::IndexElem(n)) = n.node.as_ref() { if let Some(n) = n.expr.as_ref().and_then(|n| n.node.as_ref()) { - iter.push((n.to_ref(), depth, Context::DDL)); + iter.push((n.to_ref(), depth, Context::DDL, false)); } } }); if let Some(n) = s.where_clause.as_ref() { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL)); + iter.push((n.to_ref(), depth, Context::DDL, true)); } } } NodeRef::CreateTrigStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::RuleStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::VacuumStmt(s) => { for node in &s.rels { if let Some(NodeEnum::VacuumRelation(r)) = &node.node { if let Some(rel) = r.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } } } NodeRef::RefreshMatViewStmt(s) => { if let Some(rel) = s.relation.as_ref() { - iter.push((rel.to_ref(), depth, Context::DDL)); + iter.push((rel.to_ref(), depth, Context::DDL, false)); } } NodeRef::GrantStmt(s) => { if let Some(protobuf::ObjectType::ObjectTable) = protobuf::ObjectType::from_i32(s.objtype) { s.objects.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL)); + iter.push((n.to_ref(), depth, Context::DDL, false)); } }); } @@ -278,14 +278,14 @@ impl NodeEnum { NodeRef::LockStmt(s) => { s.relations.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, Context::DDL)); + iter.push((n.to_ref(), depth, Context::DDL, false)); } }); } NodeRef::ExplainStmt(s) => { if let Some(n) = &s.query { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, false)); } } } @@ -295,92 +295,106 @@ impl NodeEnum { NodeRef::AExpr(e) => { if let Some(n) = &e.lexpr { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } if let Some(n) = &e.rexpr { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::BoolExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); } + NodeRef::BooleanTest(e) => { + if let Some(n) = &e.arg { + if let Some(n) = n.node.as_ref() { + iter.push((n.to_ref(), depth, context, has_filter_columns)); + } + } + } NodeRef::CoalesceExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::MinMaxExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); } + NodeRef::NullTest(e) => { + if let Some(n) = &e.arg { + if let Some(n) = n.node.as_ref() { + iter.push((n.to_ref(), depth, context, has_filter_columns)); + } + } + } NodeRef::ResTarget(t) => { if let Some(n) = &t.val { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::SubLink(l) => { if let Some(n) = &l.subselect { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::FuncCall(c) => { c.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::CaseExpr(c) => { c.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); if let Some(n) = &c.defresult { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::CaseWhen(w) => { if let Some(n) = &w.expr { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } if let Some(n) = &w.result { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::SortBy(n) => { if let Some(n) = &n.node { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::TypeCast(n) => { if let Some(n) = &n.arg { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } @@ -390,7 +404,7 @@ impl NodeEnum { NodeRef::List(l) => { l.items.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); } @@ -398,7 +412,7 @@ impl NodeEnum { [&e.larg, &e.rarg, &e.quals].iter().for_each(|n| { if let Some(n) = n { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } }); @@ -406,27 +420,27 @@ impl NodeEnum { NodeRef::RowExpr(e) => { e.args.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); } NodeRef::RangeSubselect(s) => { if let Some(n) = &s.subquery { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } } } NodeRef::RangeFunction(f) => { f.functions.iter().for_each(|n| { if let Some(n) = n.node.as_ref() { - iter.push((n.to_ref(), depth, context)); + iter.push((n.to_ref(), depth, context, has_filter_columns)); } }); } _ => (), } - nodes.push((node, depth, context)); + nodes.push((node, depth, context, has_filter_columns)); } nodes } @@ -761,6 +775,14 @@ impl NodeEnum { } }); } + NodeMut::NullTest(e) => { + let e = e.as_mut().unwrap(); + if let Some(n) = e.arg.as_mut() { + if let Some(n) = n.node.as_mut() { + iter.push((n.to_mut(), depth, context)); + } + } + } NodeMut::ResTarget(t) => { let t = t.as_mut().unwrap(); if let Some(n) = t.val.as_mut() { @@ -819,6 +841,14 @@ impl NodeEnum { } } } + NodeMut::TypeCast(t) => { + let t = t.as_mut().unwrap(); + if let Some(n) = t.arg.as_mut() { + if let Some(n) = n.node.as_mut() { + iter.push((n.to_mut(), depth, context)); + } + } + } // // from-clause items // diff --git a/src/parse_result.rs b/src/parse_result.rs index 19f1064..4063d80 100644 --- a/src/parse_result.rs +++ b/src/parse_result.rs @@ -24,7 +24,7 @@ impl protobuf::ParseResult { } // Note: this doesn't iterate over every possible node type, since we only care about a subset of nodes. - pub fn nodes(&self) -> Vec<(NodeRef, i32, Context)> { + pub fn nodes(&self) -> Vec<(NodeRef, i32, Context, bool)> { self.stmts .iter() .filter_map(|s| @@ -58,6 +58,7 @@ pub struct ParseResult { pub aliases: HashMap, pub cte_names: Vec, functions: Vec<(String, Context)>, + pub filter_columns: Vec<(Option, String)>, } impl ParseResult { @@ -67,8 +68,9 @@ impl ParseResult { let mut aliases: HashMap = HashMap::new(); let mut cte_names: HashSet = HashSet::new(); let mut functions: HashSet<(String, Context)> = HashSet::new(); + let mut filter_columns: HashSet<(Option, String)> = HashSet::new(); - for (node, _depth, context) in protobuf.nodes().into_iter() { + for (node, _depth, context, has_filter_columns) in protobuf.nodes().into_iter() { match node { NodeRef::CommonTableExpr(s) => { cte_names.insert(s.ctename.to_owned()); @@ -139,6 +141,23 @@ impl ParseResult { } } } + NodeRef::ColumnRef(c) => { + if !has_filter_columns { + continue; + } + let f: Vec = c + .fields + .iter() + .filter_map(|n| match n.node.as_ref() { + Some(NodeEnum::String(s)) => Some(s.sval.to_string()), + _ => None, + }) + .rev() + .collect(); + if f.len() > 0 { + filter_columns.insert((f.get(1).cloned(), f[0].to_string())); + } + } _ => (), } } @@ -150,6 +169,7 @@ impl ParseResult { aliases, cte_names: Vec::from_iter(cte_names), functions: Vec::from_iter(functions), + filter_columns: Vec::from_iter(filter_columns), } } diff --git a/tests/filter_column_tests.rs b/tests/filter_column_tests.rs new file mode 100644 index 0000000..f6c59ba --- /dev/null +++ b/tests/filter_column_tests.rs @@ -0,0 +1,105 @@ +#![allow(non_snake_case)] +#![cfg(test)] + +#[cfg(test)] +use itertools::sorted; + +use pg_query::parse; + +#[test] +fn it_finds_unqualified_names() { + let result = parse("SELECT * FROM x WHERE y = $1 AND z = 1").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "y".into()), (None, "z".into())]); +} + +#[test] +fn it_finds_qualified_names() { + let result = parse("SELECT * FROM x WHERE x.y = $1 AND x.z = 1").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(Some("x".into()), "y".into()), (Some("x".into()), "z".into())]); +} + +#[test] +fn it_traverses_into_ctes() { + let result = parse("WITH a AS (SELECT * FROM x WHERE x.y = $1 AND x.z = 1) SELECT * FROM a WHERE b = 5").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "b".into()), (Some("x".into()), "y".into()), (Some("x".into()), "z".into())]); +} + +#[test] +fn it_recognizes_boolean_tests() { + let result = parse("SELECT * FROM x WHERE x.y IS TRUE AND x.z IS NOT FALSE").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(Some("x".into()), "y".into()), (Some("x".into()), "z".into())]); +} + +#[test] +fn it_recognizes_null_tests() { + let result = parse("SELECT * FROM x WHERE x.y IS NULL AND x.z IS NOT NULL").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(Some("x".into()), "y".into()), (Some("x".into()), "z".into())]); +} + +#[test] +fn it_finds_coalesce_argument_names() { + let result = parse("SELECT * FROM x WHERE x.y = COALESCE(z.a, z.b)").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(Some("x".into()), "y".into()), (Some("z".into()), "a".into()), (Some("z".into()), "b".into())]); +} + +#[test] +fn it_finds_unqualified_names_in_union_query() { + let result = parse("SELECT * FROM x where y = $1 UNION SELECT * FROM x where z = $2").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "y".into()), (None, "z".into())]); +} + +#[test] +fn it_finds_unqualified_names_in_union_all_query() { + let result = parse("SELECT * FROM x where y = $1 UNION ALL SELECT * FROM x where z = $2").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "y".into()), (None, "z".into())]); +} + +#[test] +fn it_finds_unqualified_names_in_except_query() { + let result = parse("SELECT * FROM x where y = $1 EXCEPT SELECT * FROM x where z = $2").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "y".into()), (None, "z".into())]); +} + +#[test] +fn it_finds_unqualified_names_in_except_all_query() { + let result = parse("SELECT * FROM x where y = $1 EXCEPT ALL SELECT * FROM x where z = $2").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "y".into()), (None, "z".into())]); +} + +#[test] +fn it_finds_unqualified_names_in_intersect_query() { + let result = parse("SELECT * FROM x where y = $1 INTERSECT SELECT * FROM x where z = $2").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "y".into()), (None, "z".into())]); +} + +#[test] +fn it_finds_unqualified_names_in_intersect_all_query() { + let result = parse("SELECT * FROM x where y = $1 INTERSECT ALL SELECT * FROM x where z = $2").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(None, "y".into()), (None, "z".into())]); +} + +#[test] +fn it_ignores_target_list_columns() { + let result = parse("SELECT a, y, z FROM x WHERE x.y = $1 AND x.z = 1").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(Some("x".into()), "y".into()), (Some("x".into()), "z".into())]); +} + +#[test] +fn it_ignores_order_by_columns() { + let result = parse("SELECT * FROM x WHERE x.y = $1 AND x.z = 1 ORDER BY a, b").unwrap(); + let filter_columns: Vec<(Option, String)> = sorted(result.filter_columns).collect(); + assert_eq!(filter_columns, [(Some("x".into()), "y".into()), (Some("x".into()), "z".into())]); +} From fdfeda387751766708d8db4e74217ef105425f02 Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Wed, 10 Jan 2024 06:26:46 +0000 Subject: [PATCH 07/12] Release 5.1.0 (#40) --- CHANGELOG.md | 11 +++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b1545b..f6d09af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 5.1.0 2024-01-09 + +* Update to libpg_query 16-5.1.0 + - Add support for running on Windows + - Add support for compiling on 32-bit systems +* Always build C library using "cc" crate +* Add `filter_columns` for getting columns that a query filters by + - This returns the table name (if present) and column name for every + column that's referenced in a JOIN or WHERE clause. + + ## 5.0.0 2023-12-22 * Align versioning scheme with that of other pg_query libraries diff --git a/Cargo.lock b/Cargo.lock index e4e5e80..0a3d1ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "pg_query" -version = "5.0.0" +version = "5.1.0" dependencies = [ "bindgen", "cc", diff --git a/Cargo.toml b/Cargo.toml index f8399ca..23ae4d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pg_query" description = "PostgreSQL parser that uses the actual PostgreSQL server source to parse SQL queries and return the internal PostgreSQL parse tree." -version = "5.0.0" +version = "5.1.0" edition = "2021" documentation = "https://docs.rs/pg_query/" build = "build.rs" From 0bcda8ca73a3db9ce50076c022c2d3aaa97ec7ba Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Wed, 10 Jan 2024 06:42:17 +0000 Subject: [PATCH 08/12] Remove unused imports (#41) We were receiving warnings about these, and it doesn't appear they are needed. --- build.rs | 1 - tests/parse_plpgsql_tests.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/build.rs b/build.rs index 355f525..57d3664 100644 --- a/build.rs +++ b/build.rs @@ -5,7 +5,6 @@ use fs_extra::dir::CopyOptions; use glob::glob; use std::env; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; static SOURCE_DIRECTORY: &str = "libpg_query"; static LIBRARY_NAME: &str = "pg_query"; diff --git a/tests/parse_plpgsql_tests.rs b/tests/parse_plpgsql_tests.rs index 208ffeb..216e7d4 100644 --- a/tests/parse_plpgsql_tests.rs +++ b/tests/parse_plpgsql_tests.rs @@ -1,6 +1,5 @@ #[macro_use] mod support; -use support::*; #[test] fn it_can_parse_a_simple_function() { From 5562e4aeea885ef514134dcb084d98d6993c8c3a Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Wed, 10 Jan 2024 06:42:27 +0000 Subject: [PATCH 09/12] README and lib docs: Update reference to crate version (#42) --- README.md | 2 +- src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d408e99..24bf41c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Add the following to your `Cargo.toml` ```toml [dependencies] -pg_query = "5.0" +pg_query = "5.1" ``` ## Examples diff --git a/src/lib.rs b/src/lib.rs index 1555f8d..f26d11d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,7 @@ //! //! ```toml //! [dependencies] -//! pg_query = "5.0" +//! pg_query = "5.1" //! ``` //! //! # Example: Parsing a query From 37e66905431081af7cafc92878ce32ed5f386b66 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 2 Aug 2024 06:57:55 -0700 Subject: [PATCH 10/12] Make `crate::parse_result::ParseResult` public (#43) --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index f26d11d..1e3b3e1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -52,7 +52,7 @@ pub use error::*; pub use node_enum::*; pub use node_mut::*; pub use node_ref::*; -pub use node_structs::*; +pub use parse_result::*; pub use query::*; pub use truncate::*; From 4bbe866559d628fbbdff248a5d710f5ef892f8b4 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Wed, 30 Oct 2024 09:42:51 -0500 Subject: [PATCH 11/12] Release 5.1.1 (#46) --- .gitignore | 1 + CHANGELOG.md | 4 ++++ Cargo.lock | 2 +- Cargo.toml | 2 +- src/parse_result.rs | 6 +++++- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index c403c34..807922d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target .idea/ +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d09af..2cd2caa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 5.1.1 2024-10-30 + +* Make `ParseResult` struct public and implement `Debug` + ## 5.1.0 2024-01-09 * Update to libpg_query 16-5.1.0 diff --git a/Cargo.lock b/Cargo.lock index 0a3d1ca..9865652 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "pg_query" -version = "5.1.0" +version = "5.1.1" dependencies = [ "bindgen", "cc", diff --git a/Cargo.toml b/Cargo.toml index 23ae4d9..7d1e606 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pg_query" description = "PostgreSQL parser that uses the actual PostgreSQL server source to parse SQL queries and return the internal PostgreSQL parse tree." -version = "5.1.0" +version = "5.1.1" edition = "2021" documentation = "https://docs.rs/pg_query/" build = "build.rs" diff --git a/src/parse_result.rs b/src/parse_result.rs index 4063d80..cd908b5 100644 --- a/src/parse_result.rs +++ b/src/parse_result.rs @@ -51,6 +51,8 @@ impl protobuf::ParseResult { } } +/// Result from calling [parse] +#[derive(Debug)] pub struct ParseResult { pub protobuf: protobuf::ParseResult, pub warnings: Vec, @@ -215,7 +217,7 @@ impl ParseResult { .collect() } - /// Returns any references to tables in the query + /// Returns all function references pub fn functions(&self) -> Vec { let mut functions = HashSet::new(); self.functions.iter().for_each(|(f, _c)| { @@ -246,6 +248,7 @@ impl ParseResult { .collect() } + /// Converts the parsed query back into a SQL string pub fn deparse(&self) -> Result { crate::deparse(&self.protobuf) } @@ -263,6 +266,7 @@ impl ParseResult { crate::truncate(&self.protobuf, max_length) } + /// Returns all statement types in the query pub fn statement_types(&self) -> Vec<&str> { self.protobuf .stmts From 23ab4ae1c989fe0dea919d4fee043993e97f6223 Mon Sep 17 00:00:00 2001 From: msepga <111702535+msepga@users.noreply.github.com> Date: Tue, 26 Nov 2024 18:41:02 -0500 Subject: [PATCH 12/12] Upgrade to Postgres 17 and libpg_query 6.0.0 (#47) * Upgrade to libpg_query 17 * Update tests * Update crate and CHANGELOG --- CHANGELOG.md | 16 ++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- libpg_query | 2 +- src/node_enum.rs | 34 +++++++++++++++++++++++++++++++++- src/node_mut.rs | 34 +++++++++++++++++++++++++++++++++- src/node_ref.rs | 34 +++++++++++++++++++++++++++++++++- tests/data/plpgsql_query.json | 4 ++-- tests/data/plpgsql_simple.json | 6 +++--- tests/fingerprint_tests.rs | 2 +- tests/parse_tests.rs | 7 ++++--- 11 files changed, 128 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cd2caa..d899286 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 6.0.0 2024-11-26 + +* Upgrade to libpg_query 17-6.0.0 + - Updates to the Postgres 17 parser + - Deparser improvements: + - Add support for deparsing `JSON_TABLE`, `JSON_QUERY`, `JSON_EXISTS`, `JSON_VALUE` + - Add support for deparsing `JSON`, `JSON_SCALAR`, `JSON_SERIALIZE` + - Add support for deparsing `COPY ... FORCE_NULL(*)` + - Add support for deparsing `ALTER COLUMN ... SET EXPRESSION AS` + - Add support for deparsing `SET STATISTICS DEFAULT` + - Add support for deparsing `SET ACCESS METHOD DEFAULT` + - Add support for deparsing `... AT LOCAL` + - Add support for deparsing `merge_action()` + - Add support for deparsing `MERGE ... RETURNING` + - Add support for deparsing `NOT MATCHED [ BY TARGET ]` + ## 5.1.1 2024-10-30 * Make `ParseResult` struct public and implement `Debug` diff --git a/Cargo.lock b/Cargo.lock index 9865652..9efd998 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -390,7 +390,7 @@ dependencies = [ [[package]] name = "pg_query" -version = "5.1.1" +version = "6.0.0" dependencies = [ "bindgen", "cc", diff --git a/Cargo.toml b/Cargo.toml index 7d1e606..ab8cdcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pg_query" description = "PostgreSQL parser that uses the actual PostgreSQL server source to parse SQL queries and return the internal PostgreSQL parse tree." -version = "5.1.1" +version = "6.0.0" edition = "2021" documentation = "https://docs.rs/pg_query/" build = "build.rs" diff --git a/libpg_query b/libpg_query index 1ec3894..e1a98c3 160000 --- a/libpg_query +++ b/libpg_query @@ -1 +1 @@ -Subproject commit 1ec38940e5c6f09a4c1d17a46d839a881c4f2db7 +Subproject commit e1a98c31d90981cdfdba3c734a60e5abf9c522c2 diff --git a/src/node_enum.rs b/src/node_enum.rs index 665308c..4b4b371 100644 --- a/src/node_enum.rs +++ b/src/node_enum.rs @@ -1163,6 +1163,22 @@ impl NodeEnum { NodeEnum::JsonObjectAgg(n) => NodeRef::JsonObjectAgg(n), NodeEnum::JsonArrayAgg(n) => NodeRef::JsonArrayAgg(n), NodeEnum::RtepermissionInfo(n) => NodeRef::RtepermissionInfo(n), + NodeEnum::WindowFuncRunCondition(n) => NodeRef::WindowFuncRunCondition(n), + NodeEnum::MergeSupportFunc(n) => NodeRef::MergeSupportFunc(n), + NodeEnum::JsonBehavior(n) => NodeRef::JsonBehavior(n), + NodeEnum::JsonExpr(n) => NodeRef::JsonExpr(n), + NodeEnum::JsonTablePath(n) => NodeRef::JsonTablePath(n), + NodeEnum::JsonTablePathScan(n) => NodeRef::JsonTablePathScan(n), + NodeEnum::JsonTableSiblingJoin(n) => NodeRef::JsonTableSiblingJoin(n), + NodeEnum::SinglePartitionSpec(n) => NodeRef::SinglePartitionSpec(n), + NodeEnum::JsonArgument(n) => NodeRef::JsonArgument(n), + NodeEnum::JsonFuncExpr(n) => NodeRef::JsonFuncExpr(n), + NodeEnum::JsonTablePathSpec(n) => NodeRef::JsonTablePathSpec(n), + NodeEnum::JsonTable(n) => NodeRef::JsonTable(n), + NodeEnum::JsonTableColumn(n) => NodeRef::JsonTableColumn(n), + NodeEnum::JsonParseExpr(n) => NodeRef::JsonParseExpr(n), + NodeEnum::JsonScalarExpr(n) => NodeRef::JsonScalarExpr(n), + NodeEnum::JsonSerializeExpr(n) => NodeRef::JsonSerializeExpr(n), } } @@ -1332,7 +1348,7 @@ impl NodeEnum { NodeEnum::CreateStatsStmt(n) => NodeMut::CreateStatsStmt(n as *mut _), NodeEnum::AlterCollationStmt(n) => NodeMut::AlterCollationStmt(n as *mut _), NodeEnum::CallStmt(n) => NodeMut::CallStmt(&mut **n as *mut _), - NodeEnum::AlterStatsStmt(n) => NodeMut::AlterStatsStmt(n as *mut _), + NodeEnum::AlterStatsStmt(n) => NodeMut::AlterStatsStmt(&mut **n as *mut _), NodeEnum::AExpr(n) => NodeMut::AExpr(&mut **n as *mut _), NodeEnum::ColumnRef(n) => NodeMut::ColumnRef(n as *mut _), NodeEnum::ParamRef(n) => NodeMut::ParamRef(n as *mut _), @@ -1420,6 +1436,22 @@ impl NodeEnum { NodeEnum::JsonObjectAgg(n) => NodeMut::JsonObjectAgg(&mut **n as *mut _), NodeEnum::JsonArrayAgg(n) => NodeMut::JsonArrayAgg(&mut **n as *mut _), NodeEnum::RtepermissionInfo(n) => NodeMut::RtepermissionInfo(&mut *n as *mut _), + NodeEnum::WindowFuncRunCondition(n) => NodeMut::WindowFuncRunCondition(&mut **n as *mut _), + NodeEnum::MergeSupportFunc(n) => NodeMut::MergeSupportFunc(&mut **n as *mut _), + NodeEnum::JsonBehavior(n) => NodeMut::JsonBehavior(&mut **n as *mut _), + NodeEnum::JsonExpr(n) => NodeMut::JsonExpr(&mut **n as *mut _), + NodeEnum::JsonTablePath(n) => NodeMut::JsonTablePath(&mut *n as *mut _), + NodeEnum::JsonTablePathScan(n) => NodeMut::JsonTablePathScan(&mut **n as *mut _), + NodeEnum::JsonTableSiblingJoin(n) => NodeMut::JsonTableSiblingJoin(&mut **n as *mut _), + NodeEnum::SinglePartitionSpec(n) => NodeMut::SinglePartitionSpec(&mut *n as *mut _), + NodeEnum::JsonArgument(n) => NodeMut::JsonArgument(&mut **n as *mut _), + NodeEnum::JsonFuncExpr(n) => NodeMut::JsonFuncExpr(&mut **n as *mut _), + NodeEnum::JsonTablePathSpec(n) => NodeMut::JsonTablePathSpec(&mut **n as *mut _), + NodeEnum::JsonTable(n) => NodeMut::JsonTable(&mut **n as *mut _), + NodeEnum::JsonTableColumn(n) => NodeMut::JsonTableColumn(&mut **n as *mut _), + NodeEnum::JsonParseExpr(n) => NodeMut::JsonParseExpr(&mut **n as *mut _), + NodeEnum::JsonScalarExpr(n) => NodeMut::JsonScalarExpr(&mut **n as *mut _), + NodeEnum::JsonSerializeExpr(n) => NodeMut::JsonSerializeExpr(&mut **n as *mut _), } } } diff --git a/src/node_mut.rs b/src/node_mut.rs index 8006bd5..615cebb 100644 --- a/src/node_mut.rs +++ b/src/node_mut.rs @@ -254,6 +254,22 @@ pub enum NodeMut { JsonObjectAgg(*mut protobuf::JsonObjectAgg), JsonArrayAgg(*mut protobuf::JsonArrayAgg), RtepermissionInfo(*mut protobuf::RtePermissionInfo), + WindowFuncRunCondition(*mut protobuf::WindowFuncRunCondition), + MergeSupportFunc(*mut protobuf::MergeSupportFunc), + JsonBehavior(*mut protobuf::JsonBehavior), + JsonExpr(*mut protobuf::JsonExpr), + JsonTablePath(*mut protobuf::JsonTablePath), + JsonTablePathScan(*mut protobuf::JsonTablePathScan), + JsonTableSiblingJoin(*mut protobuf::JsonTableSiblingJoin), + SinglePartitionSpec(*mut protobuf::SinglePartitionSpec), + JsonArgument(*mut protobuf::JsonArgument), + JsonFuncExpr(*mut protobuf::JsonFuncExpr), + JsonTablePathSpec(*mut protobuf::JsonTablePathSpec), + JsonTable(*mut protobuf::JsonTable), + JsonTableColumn(*mut protobuf::JsonTableColumn), + JsonParseExpr(*mut protobuf::JsonParseExpr), + JsonScalarExpr(*mut protobuf::JsonScalarExpr), + JsonSerializeExpr(*mut protobuf::JsonSerializeExpr), } impl NodeMut { @@ -432,7 +448,7 @@ impl NodeMut { NodeMut::CreateStatsStmt(n) => Ok(NodeEnum::CreateStatsStmt(n.as_ref().ok_or(err)?.clone())), NodeMut::AlterCollationStmt(n) => Ok(NodeEnum::AlterCollationStmt(n.as_ref().ok_or(err)?.clone())), NodeMut::CallStmt(n) => Ok(NodeEnum::CallStmt(Box::new(n.as_ref().ok_or(err)?.clone()))), - NodeMut::AlterStatsStmt(n) => Ok(NodeEnum::AlterStatsStmt(n.as_ref().ok_or(err)?.clone())), + NodeMut::AlterStatsStmt(n) => Ok(NodeEnum::AlterStatsStmt(Box::new(n.as_ref().ok_or(err)?.clone()))), NodeMut::AExpr(n) => Ok(NodeEnum::AExpr(Box::new(n.as_ref().ok_or(err)?.clone()))), NodeMut::ColumnRef(n) => Ok(NodeEnum::ColumnRef(n.as_ref().ok_or(err)?.clone())), NodeMut::ParamRef(n) => Ok(NodeEnum::ParamRef(n.as_ref().ok_or(err)?.clone())), @@ -520,6 +536,22 @@ impl NodeMut { NodeMut::JsonObjectAgg(n) => Ok(NodeEnum::JsonObjectAgg(Box::new(n.as_ref().ok_or(err)?.clone()))), NodeMut::JsonArrayAgg(n) => Ok(NodeEnum::JsonArrayAgg(Box::new(n.as_ref().ok_or(err)?.clone()))), NodeMut::RtepermissionInfo(n) => Ok(NodeEnum::RtepermissionInfo(n.as_ref().ok_or(err)?.clone())), + NodeMut::WindowFuncRunCondition(n) => Ok(NodeEnum::WindowFuncRunCondition(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::MergeSupportFunc(n) => Ok(NodeEnum::MergeSupportFunc(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonBehavior(n) => Ok(NodeEnum::JsonBehavior(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonExpr(n) => Ok(NodeEnum::JsonExpr(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonTablePath(n) => Ok(NodeEnum::JsonTablePath(n.as_ref().ok_or(err)?.clone())), + NodeMut::JsonTablePathScan(n) => Ok(NodeEnum::JsonTablePathScan(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonTableSiblingJoin(n) => Ok(NodeEnum::JsonTableSiblingJoin(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::SinglePartitionSpec(n) => Ok(NodeEnum::SinglePartitionSpec(n.as_ref().ok_or(err)?.clone())), + NodeMut::JsonArgument(n) => Ok(NodeEnum::JsonArgument(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonFuncExpr(n) => Ok(NodeEnum::JsonFuncExpr(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonTablePathSpec(n) => Ok(NodeEnum::JsonTablePathSpec(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonTable(n) => Ok(NodeEnum::JsonTable(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonTableColumn(n) => Ok(NodeEnum::JsonTableColumn(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonParseExpr(n) => Ok(NodeEnum::JsonParseExpr(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonScalarExpr(n) => Ok(NodeEnum::JsonScalarExpr(Box::new(n.as_ref().ok_or(err)?.clone()))), + NodeMut::JsonSerializeExpr(n) => Ok(NodeEnum::JsonSerializeExpr(Box::new(n.as_ref().ok_or(err)?.clone()))), } } } diff --git a/src/node_ref.rs b/src/node_ref.rs index 41a97d6..6209544 100644 --- a/src/node_ref.rs +++ b/src/node_ref.rs @@ -254,6 +254,22 @@ pub enum NodeRef<'a> { JsonObjectAgg(&'a protobuf::JsonObjectAgg), JsonArrayAgg(&'a protobuf::JsonArrayAgg), RtepermissionInfo(&'a protobuf::RtePermissionInfo), + WindowFuncRunCondition(&'a protobuf::WindowFuncRunCondition), + MergeSupportFunc(&'a protobuf::MergeSupportFunc), + JsonBehavior(&'a protobuf::JsonBehavior), + JsonExpr(&'a protobuf::JsonExpr), + JsonTablePath(&'a protobuf::JsonTablePath), + JsonTablePathScan(&'a protobuf::JsonTablePathScan), + JsonTableSiblingJoin(&'a protobuf::JsonTableSiblingJoin), + SinglePartitionSpec(&'a protobuf::SinglePartitionSpec), + JsonArgument(&'a protobuf::JsonArgument), + JsonFuncExpr(&'a protobuf::JsonFuncExpr), + JsonTablePathSpec(&'a protobuf::JsonTablePathSpec), + JsonTable(&'a protobuf::JsonTable), + JsonTableColumn(&'a protobuf::JsonTableColumn), + JsonParseExpr(&'a protobuf::JsonParseExpr), + JsonScalarExpr(&'a protobuf::JsonScalarExpr), + JsonSerializeExpr(&'a protobuf::JsonSerializeExpr), } impl<'a> NodeRef<'a> { @@ -431,7 +447,7 @@ impl<'a> NodeRef<'a> { NodeRef::CreateStatsStmt(n) => NodeEnum::CreateStatsStmt((*n).clone()), NodeRef::AlterCollationStmt(n) => NodeEnum::AlterCollationStmt((*n).clone()), NodeRef::CallStmt(n) => NodeEnum::CallStmt(Box::new((*n).clone())), - NodeRef::AlterStatsStmt(n) => NodeEnum::AlterStatsStmt((*n).clone()), + NodeRef::AlterStatsStmt(n) => NodeEnum::AlterStatsStmt(Box::new((*n).clone())), NodeRef::AExpr(n) => NodeEnum::AExpr(Box::new((*n).clone())), NodeRef::ColumnRef(n) => NodeEnum::ColumnRef((*n).clone()), NodeRef::ParamRef(n) => NodeEnum::ParamRef((*n).clone()), @@ -519,6 +535,22 @@ impl<'a> NodeRef<'a> { NodeRef::JsonObjectAgg(n) => NodeEnum::JsonObjectAgg(Box::new((*n).clone())), NodeRef::JsonArrayAgg(n) => NodeEnum::JsonArrayAgg(Box::new((*n).clone())), NodeRef::RtepermissionInfo(n) => NodeEnum::RtepermissionInfo((*n).clone()), + NodeRef::WindowFuncRunCondition(n) => NodeEnum::WindowFuncRunCondition(Box::new((*n).clone())), + NodeRef::MergeSupportFunc(n) => NodeEnum::MergeSupportFunc(Box::new((*n).clone())), + NodeRef::JsonBehavior(n) => NodeEnum::JsonBehavior(Box::new((*n).clone())), + NodeRef::JsonExpr(n) => NodeEnum::JsonExpr(Box::new((*n).clone())), + NodeRef::JsonTablePath(n) => NodeEnum::JsonTablePath((*n).clone()), + NodeRef::JsonTablePathScan(n) => NodeEnum::JsonTablePathScan(Box::new((*n).clone())), + NodeRef::JsonTableSiblingJoin(n) => NodeEnum::JsonTableSiblingJoin(Box::new((*n).clone())), + NodeRef::SinglePartitionSpec(n) => NodeEnum::SinglePartitionSpec((*n).clone()), + NodeRef::JsonArgument(n) => NodeEnum::JsonArgument(Box::new((*n).clone())), + NodeRef::JsonFuncExpr(n) => NodeEnum::JsonFuncExpr(Box::new((*n).clone())), + NodeRef::JsonTablePathSpec(n) => NodeEnum::JsonTablePathSpec(Box::new((*n).clone())), + NodeRef::JsonTable(n) => NodeEnum::JsonTable(Box::new((*n).clone())), + NodeRef::JsonTableColumn(n) => NodeEnum::JsonTableColumn(Box::new((*n).clone())), + NodeRef::JsonParseExpr(n) => NodeEnum::JsonParseExpr(Box::new((*n).clone())), + NodeRef::JsonScalarExpr(n) => NodeEnum::JsonScalarExpr(Box::new((*n).clone())), + NodeRef::JsonSerializeExpr(n) => NodeEnum::JsonSerializeExpr(Box::new((*n).clone())), } } } diff --git a/tests/data/plpgsql_query.json b/tests/data/plpgsql_query.json index cd9dcdd..41b7110 100644 --- a/tests/data/plpgsql_query.json +++ b/tests/data/plpgsql_query.json @@ -48,7 +48,7 @@ "PLpgSQL_var": { "datatype": { "PLpgSQL_type": { - "typname": "UNKNOWN" + "typname": "pg_catalog.int4" } }, "refname": "input" @@ -58,7 +58,7 @@ "PLpgSQL_var": { "datatype": { "PLpgSQL_type": { - "typname": "UNKNOWN" + "typname": "pg_catalog.\"boolean\"" } }, "refname": "found" diff --git a/tests/data/plpgsql_simple.json b/tests/data/plpgsql_simple.json index b70cc76..0f58cc9 100644 --- a/tests/data/plpgsql_simple.json +++ b/tests/data/plpgsql_simple.json @@ -48,7 +48,7 @@ "PLpgSQL_var": { "datatype": { "PLpgSQL_type": { - "typname": "UNKNOWN" + "typname": "pg_catalog.\"varchar\"" } }, "refname": "v_name" @@ -58,7 +58,7 @@ "PLpgSQL_var": { "datatype": { "PLpgSQL_type": { - "typname": "UNKNOWN" + "typname": "pg_catalog.\"varchar\"" } }, "refname": "v_version" @@ -68,7 +68,7 @@ "PLpgSQL_var": { "datatype": { "PLpgSQL_type": { - "typname": "UNKNOWN" + "typname": "pg_catalog.\"boolean\"" } }, "refname": "found" diff --git a/tests/fingerprint_tests.rs b/tests/fingerprint_tests.rs index d552ee2..e395c58 100644 --- a/tests/fingerprint_tests.rs +++ b/tests/fingerprint_tests.rs @@ -178,7 +178,7 @@ fn it_works() { assert_eq!(result.hex, "d8a65a814fbc5f95"); let result = fingerprint("DEALLOCATE ALL").unwrap(); - assert_eq!(result.hex, "d8a65a814fbc5f95"); + assert_eq!(result.hex, "2debfb8745df64a7"); let result = fingerprint("EXPLAIN ANALYZE SELECT a").unwrap(); assert_eq!(result.hex, "82845c1b5c6102e5"); diff --git a/tests/parse_tests.rs b/tests/parse_tests.rs index 57e5284..a14c1ff 100644 --- a/tests/parse_tests.rs +++ b/tests/parse_tests.rs @@ -125,11 +125,13 @@ fn it_parses_ALTER_TABLE() { conname: "", deferrable: false, initdeferred: false, - location: 21, + skip_validation: false, + initially_valid: false, is_no_inherit: false, raw_expr: None, cooked_expr: "", generated_when: "", + inhcount: 0, nulls_not_distinct: false, keys: [ Node { @@ -159,8 +161,7 @@ fn it_parses_ALTER_TABLE() { fk_del_set_cols: [], old_conpfeqop: [], old_pktable_oid: 0, - skip_validation: false, - initially_valid: false, + location: 21, }, ), ),