diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 3d9cddc03..4da2e5f44 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -386,6 +386,7 @@ CREATE TABLE public.locations ( CREATE TABLE image (title text, raster lo); +CREATE TABLE employees (id INT PRIMARY KEY, salary INT); -- create table with multirange data types -- Create tables with primary keys directly diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql index 157d311ce..1fc6a82cb 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql @@ -34,6 +34,11 @@ CREATE VIEW public.orders_view AS orders.xmin AS transaction_id FROM public.orders; +CREATE VIEW top_employees_view AS SELECT * FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees; CREATE VIEW public.my_films_view AS SELECT jt.* FROM my_films, @@ -43,4 +48,4 @@ SELECT jt.* FROM kind text PATH '$.kind', NESTED PATH '$.films[*]' COLUMNS ( title text FORMAT JSON PATH '$.title' OMIT QUOTES, - director text PATH '$.director' KEEP QUOTES))) AS jt; \ No newline at end of file + director text PATH '$.director' KEEP QUOTES))) AS jt; diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index a6a63a62b..0bfd66b0e 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -1931,6 +1931,16 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318", "MinimumVersionsFixedIn": null }, + { + "IssueType": "unsupported_features", + "ObjectType": "VIEW", + "ObjectName": "top_employees_view", + "Reason": "FETCH .. WITH TIES", + "SqlStatement": "CREATE VIEW top_employees_view AS SELECT * FROM (\n\t\t\tSELECT * FROM employees\n\t\t\tORDER BY salary DESC\n\t\t\tFETCH FIRST 2 ROWS WITH TIES\n\t\t) AS top_employees;", + "Suggestion": "No workaround available right now", + "GH": "", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_datatypes", "ObjectType": "TABLE", diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index bdc832263..5e9353c7a 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,10 @@ }, { "ObjectType": "TABLE", - "TotalCount": 57, + "TotalCount": 58, "InvalidCount": 49, - "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, + { "ObjectType": "INDEX", "TotalCount": 43, @@ -50,9 +51,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 6, - "InvalidCount": 6, - "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name" + "TotalCount": 7, + "InvalidCount": 7, + "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name, top_employees_view" }, { "ObjectType": "TRIGGER", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index ec45f22bf..76e5ffb54 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,15 +38,15 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 41, + "TotalCount": 43, "InvalidCount": 0, - "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 80, + "TotalCount": 82, "InvalidCount": 35, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -73,9 +73,9 @@ }, { "ObjectType": "VIEW", - "TotalCount": 8, - "InvalidCount": 4, - "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker" + "TotalCount": 10, + "InvalidCount": 6, + "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker, schema2.top_employees_view, public.top_employees_view" }, { "ObjectType": "TRIGGER", @@ -162,6 +162,7 @@ "schema2.products", "schema2.foo", "schema2.Case_Sensitive_Columns", + "schema2.employeesforview", "schema2.with_example1", "test_views.xyz_mview", "test_views.view_table2", @@ -183,9 +184,10 @@ "schema2.int_multirange_table", "schema2.numeric_multirange_table", "schema2.timestamp_multirange_table", - "schema2.timestamptz_multirange_table" + "schema2.timestamptz_multirange_table", + "public.employeesforview" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 86 objects (78 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -642,6 +644,20 @@ ], "MinimumVersionsFixedIn": null }, + { + "FeatureName": "FETCH .. WITH TIES Clause", + "Objects": [ + { + "ObjectName": "public.top_employees_view", + "SqlStatement": "CREATE VIEW public.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM public.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;" + }, + { + "ObjectName": "schema2.top_employees_view", + "SqlStatement": "CREATE VIEW schema2.top_employees_view AS\n SELECT id,\n first_name,\n last_name,\n salary\n FROM ( SELECT employeesforview.id,\n employeesforview.first_name,\n employeesforview.last_name,\n employeesforview.salary\n FROM schema2.employeesforview\n ORDER BY employeesforview.salary DESC\n FETCH FIRST 2 ROWS WITH TIES) top_employees;" + } + ], + "MinimumVersionsFixedIn": null + }, { "FeatureName": "Security Invoker Views", "Objects": [ @@ -938,7 +954,7 @@ { "SchemaName": "public", "ObjectName": "employees", - "RowCount": 5, + "RowCount": 10, "ColumnCount": 4, "Reads": 0, "Writes": 10, @@ -2069,6 +2085,20 @@ "ParentTableName": "schema2.mixed_data_types_table1", "SizeInBytes": 8192 }, + { + "SchemaName": "public", + "ObjectName": "employeesforview", + "RowCount": 0, + "ColumnCount": 4, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "public", "ObjectName": "employees3", @@ -2167,6 +2197,20 @@ "ParentTableName": null, "SizeInBytes": 0 }, + { + "SchemaName": "schema2", + "ObjectName": "employeesforview", + "RowCount": 0, + "ColumnCount": 4, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "schema2", "ObjectName": "bigint_multirange_table", diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index e1411914d..7e3c798ea 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -393,6 +393,20 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- for FETCH .. WITH TIES +CREATE TABLE employeesForView ( + id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + salary NUMERIC(10, 2) NOT NULL +); + +CREATE VIEW top_employees_view AS SELECT * FROM ( + SELECT * FROM employeesForView + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees; + -- SECURITY INVOKER VIEW CREATE TABLE public.employees ( employee_id SERIAL PRIMARY KEY, diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 12cae929b..c54e998f5 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1023,6 +1023,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go index e825bc997..a56ec1097 100644 --- a/yb-voyager/cmd/constants.go +++ b/yb-voyager/cmd/constants.go @@ -217,6 +217,7 @@ const ( BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables" PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE = "Primary / Unique key constraints on complex datatypes" REGEX_FUNCTIONS_FEATURE = "Regex Functions" + FETCH_WITH_TIES_FEATURE = "FETCH .. WITH TIES Clause" // Migration caveats diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod index 8cf7a703f..7df52aa40 100644 --- a/yb-voyager/go.mod +++ b/yb-voyager/go.mod @@ -15,7 +15,6 @@ require ( github.com/docker/go-connections v0.5.0 github.com/dustin/go-humanize v1.0.1 github.com/fatih/color v1.13.0 - github.com/fergusstrange/embedded-postgres v1.29.0 github.com/go-sql-driver/mysql v1.7.0 github.com/godror/godror v0.30.2 github.com/google/uuid v1.6.0 @@ -25,7 +24,6 @@ require ( github.com/jackc/pgconn v1.14.3 github.com/jackc/pgx/v4 v4.18.3 github.com/jackc/pgx/v5 v5.0.3 - github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 github.com/mitchellh/go-ps v1.0.0 @@ -83,7 +81,6 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect diff --git a/yb-voyager/go.sum b/yb-voyager/go.sum index e68839fdc..1c06c76f9 100644 --- a/yb-voyager/go.sum +++ b/yb-voyager/go.sum @@ -919,8 +919,6 @@ github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fergusstrange/embedded-postgres v1.29.0 h1:Uv8hdhoiaNMuH0w8UuGXDHr60VoAQPFdgx7Qf3bzXJM= -github.com/fergusstrange/embedded-postgres v1.29.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -1971,8 +1969,6 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -2103,7 +2099,6 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index 329c25658..586957bf9 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -68,8 +68,8 @@ const ( ADVISORY_LOCKS_NAME = "Advisory Locks" SYSTEM_COLUMNS_NAME = "System Columns" XML_FUNCTIONS_NAME = "XML Functions" - - REGEX_FUNCTIONS = "REGEX_FUNCTIONS" + FETCH_WITH_TIES = "FETCH_WITH_TIES" + REGEX_FUNCTIONS = "REGEX_FUNCTIONS" MULTI_RANGE_DATATYPE = "MULTI_RANGE_DATATYPE" COPY_FROM_WHERE = "COPY FROM ... WHERE" diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index e6ec4c4cc..c834fa6db 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -206,6 +206,40 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } +type SelectStmtDetector struct { + query string + limitOptionWithTiesDetected bool +} + +func NewSelectStmtDetector(query string) *SelectStmtDetector { + return &SelectStmtDetector{ + query: query, + } +} + +func (d *SelectStmtDetector) Detect(msg protoreflect.Message) error { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_SELECTSTMT_NODE { + selectStmtNode, err := queryparser.ProtoAsSelectStmt(msg) + if err != nil { + return err + } + // checks if a SelectStmt node uses a FETCH clause with TIES + // https://www.postgresql.org/docs/13/sql-select.html#SQL-LIMIT + if selectStmtNode.LimitOption == queryparser.LIMIT_OPTION_WITH_TIES { + d.limitOptionWithTiesDetected = true + } + } + return nil +} + +func (d *SelectStmtDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.limitOptionWithTiesDetected { + issues = append(issues, NewFetchWithTiesIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + type CopyCommandUnsupportedConstructsDetector struct { query string copyFromWhereConstructDetected bool diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index e6915177c..7e6f8aed8 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -167,3 +167,16 @@ var copyOnErrorIssue = issue.Issue{ func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { return newQueryIssue(copyOnErrorIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) } + +var fetchWithTiesIssue = issue.Issue{ + Type: FETCH_WITH_TIES, + TypeName: "FETCH .. WITH TIES", + TypeDescription: "FETCH .. WITH TIES is not supported in YugabyteDB", + Suggestion: "No workaround available right now", + GH: "", + DocsLink: "", //TODO +} + +func NewFetchWithTiesIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(fetchWithTiesIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 8bbda2d34..cff5910c2 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -61,16 +61,35 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testFetchWithTiesIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + + stmts := []string{ + `SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES;`, + } + + for _, stmt := range stmts { + _, err = conn.Exec(ctx, stmt) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `syntax error at or near "WITH"`, regexFunctionsIssue) + } +} + func testCopyOnErrorIssue(t *testing.T) { ctx := context.Background() conn, err := getConn() assert.NoError(t, err) defer conn.Close(context.Background()) + // In case the COPY ... ON_ERROR construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`) assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: option \"on_error\" not recognized (SQLSTATE 42601)", copyOnErrorIssue) - } func testCopyFromWhereIssue(t *testing.T) { @@ -171,6 +190,9 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "fetch with ties", ybVersion), testFetchWithTiesIssue) + assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "copy on error", ybVersion), testCopyOnErrorIssue) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 9c32fca53..c7faf1209 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewSelectStmtDetector(query), NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e4c6f09d1..e5b6315b0 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -653,6 +653,68 @@ func TestRegexFunctionsIssue(t *testing.T) { } +func TestFetchWithTiesInSelect(t *testing.T) { + + stmt1 := ` + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES;` + + // subquery + stmt2 := `SELECT * + FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees;` + + stmt3 := `CREATE VIEW top_employees_view AS + SELECT * + FROM ( + SELECT * FROM employees + ORDER BY salary DESC + FETCH FIRST 2 ROWS WITH TIES + ) AS top_employees;` + + expectedIssues := map[string][]QueryIssue{ + stmt1: []QueryIssue{NewFetchWithTiesIssue("DML_QUERY", "", stmt1)}, + stmt2: []QueryIssue{NewFetchWithTiesIssue("DML_QUERY", "", stmt2)}, + } + expectedDDLIssues := map[string][]QueryIssue{ + stmt3: []QueryIssue{NewFetchWithTiesIssue("VIEW", "top_employees_view", stmt3)}, + } + + parserIssueDetector := NewParserIssueDetector() + + for stmt, expectedIssues := range expectedIssues { + issues, err := parserIssueDetector.GetDMLIssues(stmt, ybversion.LatestStable) + + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } + + for stmt, expectedIssues := range expectedDDLIssues { + issues, err := parserIssueDetector.GetDDLIssues(stmt, ybversion.LatestStable) + + assert.NoError(t, err, "Error detecting issues for statement: %s", stmt) + + assert.Equal(t, len(expectedIssues), len(issues), "Mismatch in issue count for statement: %s", stmt) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + } + } +} + func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { expectedIssues := map[string][]QueryIssue{ `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`)}, @@ -683,6 +745,7 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { issues, err := parserIssueDetector.getDMLIssues(stmt) testutils.FatalIfError(t, err) assert.Equal(t, len(expectedIssues), len(issues)) + for _, expectedIssue := range expectedIssues { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { return cmp.Equal(expectedIssue, queryIssue) diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 8fe62bc82..de3204f3c 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -21,6 +21,7 @@ import ( pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -385,7 +386,7 @@ func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List // GetEnumNumField retrieves a enum field from a message // FieldDescriptor{Syntax: proto3, FullName: pg_query.JsonFuncExpr.op, Number: 1, Cardinality: optional, Kind: enum, HasJSONName: true, JSONName: "op", Enum: pg_query.JsonExprOp} -//val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT +// val:{json_func_expr:{op:JSON_QUERY_OP context_item:{raw_expr:{column_ref:{fields:{string:{sval:"details"}} location:2626}} format:{format_type:JS_FORMAT_DEFAULT encoding:JS_ENC_DEFAULT func GetEnumNumField(msg protoreflect.Message, fieldName string) protoreflect.EnumNumber { field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) if field != nil && msg.Has(field) { @@ -407,6 +408,18 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { return schemaName, objectName } +func ProtoAsSelectStmt(msg protoreflect.Message) (*pg_query.SelectStmt, error) { + protoMsg, ok := msg.Interface().(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to cast msg to proto.Message") + } + selectStmtNode, ok := protoMsg.(*pg_query.SelectStmt) + if !ok { + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_SELECTSTMT_NODE) + } + return selectStmtNode, nil +} + /* Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} diff --git a/yb-voyager/src/query/queryparser/helpers_struct.go b/yb-voyager/src/query/queryparser/helpers_struct.go index eafc90791..37d421bdf 100644 --- a/yb-voyager/src/query/queryparser/helpers_struct.go +++ b/yb-voyager/src/query/queryparser/helpers_struct.go @@ -23,6 +23,10 @@ import ( "github.com/samber/lo" ) +const ( + LIMIT_OPTION_WITH_TIES = pg_query.LimitOption_LIMIT_OPTION_WITH_TIES +) + func IsPLPGSQLObject(parseTree *pg_query.ParseResult) bool { // CREATE FUNCTION is same parser NODE for FUNCTION/PROCEDURE _, isPlPgSQLObject := getCreateFuncStmtNode(parseTree) diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index e742a8172..9ca3ae3ad 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -41,6 +41,7 @@ const ( PG_QUERY_INSERTSTMT_NODE = "pg_query.InsertStmt" PG_QUERY_UPDATESTMT_NODE = "pg_query.UpdateStmt" PG_QUERY_DELETESTMT_NODE = "pg_query.DeleteStmt" + PG_QUERY_SELECTSTMT_NODE = "pg_query.SelectStmt" PG_QUERY_JSON_OBJECT_AGG_NODE = "pg_query.JsonObjectAgg" PG_QUERY_JSON_ARRAY_AGG_NODE = "pg_query.JsonArrayAgg"