FETCH .. WITH TIES issue detection (#2111)

* Added new detector SelectStmtDetector
yugabyte · Dec 30, 2024 · fa542b7 · fa542b7
1 parent a85c6fc
commit fa542b7
Show file tree

Hide file tree

Showing 19 changed files with 248 additions and 28 deletions.
diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql
@@ -386,6 +386,7 @@ CREATE TABLE public.locations (
 
 CREATE TABLE image (title text, raster lo);
 
+CREATE TABLE employees (id INT PRIMARY KEY, salary INT);
 -- create table with multirange data types
 
 -- Create tables with primary keys directly

diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/views/view.sql
@@ -34,6 +34,11 @@ CREATE VIEW public.orders_view AS
     orders.xmin AS transaction_id
    FROM public.orders;
 
+CREATE VIEW top_employees_view AS SELECT * FROM (
+			SELECT * FROM employees
+			ORDER BY salary DESC
+			FETCH FIRST 2 ROWS WITH TIES
+		) AS top_employees;
 CREATE VIEW public.my_films_view AS 
 SELECT jt.* FROM
  my_films,
@@ -43,4 +48,4 @@ SELECT jt.* FROM
     kind text PATH '$.kind',
     NESTED PATH '$.films[*]' COLUMNS (
       title text FORMAT JSON PATH '$.title' OMIT QUOTES,
-      director text PATH '$.director' KEEP QUOTES))) AS jt;
+      director text PATH '$.director' KEEP QUOTES))) AS jt;
diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json
@@ -1931,6 +1931,16 @@
             "GH": "https://github.com/yugabyte/yugabyte-db/issues/25318",
             "MinimumVersionsFixedIn": null
         },
+        {
+            "IssueType": "unsupported_features",
+            "ObjectType": "VIEW",
+            "ObjectName": "top_employees_view",
+            "Reason": "FETCH .. WITH TIES",
+            "SqlStatement": "CREATE VIEW top_employees_view AS SELECT * FROM (\n\t\t\tSELECT * FROM employees\n\t\t\tORDER BY salary DESC\n\t\t\tFETCH FIRST 2 ROWS WITH TIES\n\t\t) AS top_employees;",
+            "Suggestion": "No workaround available right now",
+            "GH": "",
+            "MinimumVersionsFixedIn": null
+        },
         {
             "IssueType": "unsupported_datatypes",
             "ObjectType": "TABLE",

diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json
@@ -26,9 +26,10 @@
             },
             {
                 "ObjectType": "TABLE",
-                "TotalCount": 57,
+                "TotalCount": 58,
                 "InvalidCount": 49,
-                "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table"            },
+                "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table"            },
+
             {
                 "ObjectType": "INDEX",
                 "TotalCount": 43,
@@ -50,9 +51,9 @@
             },
             {
                 "ObjectType": "VIEW",
-                "TotalCount": 6,
-                "InvalidCount": 6,
-                "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name"
+                "TotalCount": 7,
+                "InvalidCount": 7,
+                "ObjectNames": "public.my_films_view, v1, v2, test, public.orders_view, view_name, top_employees_view"
             },
             {
                 "ObjectType": "TRIGGER",

diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json
@@ -38,15 +38,15 @@
 			},
 			{
 				"ObjectType": "SEQUENCE",
-				"TotalCount": 41,
+				"TotalCount": 43,
 				"InvalidCount": 0,
-				"ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
+				"ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
 			},
 			{
 				"ObjectType": "TABLE",
-				"TotalCount": 80,
+				"TotalCount": 82,
 				"InvalidCount": 35,
-				"ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
+				"ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
 			},
 			{
 				"ObjectType": "INDEX",
@@ -73,9 +73,9 @@
 			},
 			{
 				"ObjectType": "VIEW",
-				"TotalCount": 8,
-				"InvalidCount": 4,
-				"ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker"
+				"TotalCount": 10,
+				"InvalidCount": 6,
+				"ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker, schema2.top_employees_view, public.top_employees_view"
 			},
 			{
 				"ObjectType": "TRIGGER",
@@ -162,6 +162,7 @@
 				"schema2.products",
 				"schema2.foo",
 				"schema2.Case_Sensitive_Columns",
+				"schema2.employeesforview",
 				"schema2.with_example1",
 				"test_views.xyz_mview",
 				"test_views.view_table2",
@@ -183,9 +184,10 @@
 				"schema2.int_multirange_table",
 				"schema2.numeric_multirange_table",
 				"schema2.timestamp_multirange_table",
-				"schema2.timestamptz_multirange_table"
+				"schema2.timestamptz_multirange_table",
+				"public.employeesforview"
 			],
-			"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 86 objects (78 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
+			"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
 			"ShardedTables": [
 				"public.combined_tbl",
 				"public.citext_type",
@@ -642,6 +644,20 @@
 			],
 			"MinimumVersionsFixedIn": null
 		},
+		{
+			"FeatureName": "FETCH .. WITH TIES Clause",
+			"Objects": [
+				{
+					"ObjectName": "public.top_employees_view",
+					"SqlStatement": "CREATE VIEW public.top_employees_view AS\n SELECT id,\n    first_name,\n    last_name,\n    salary\n   FROM ( SELECT employeesforview.id,\n            employeesforview.first_name,\n            employeesforview.last_name,\n            employeesforview.salary\n           FROM public.employeesforview\n          ORDER BY employeesforview.salary DESC\n         FETCH FIRST 2 ROWS WITH TIES) top_employees;"
+				},
+				{
+					"ObjectName": "schema2.top_employees_view",
+					"SqlStatement": "CREATE VIEW schema2.top_employees_view AS\n SELECT id,\n    first_name,\n    last_name,\n    salary\n   FROM ( SELECT employeesforview.id,\n            employeesforview.first_name,\n            employeesforview.last_name,\n            employeesforview.salary\n           FROM schema2.employeesforview\n          ORDER BY employeesforview.salary DESC\n         FETCH FIRST 2 ROWS WITH TIES) top_employees;"
+				}
+			],
+			"MinimumVersionsFixedIn": null
+		},
 		{
 			"FeatureName": "Security Invoker Views",
 			"Objects": [
@@ -938,7 +954,7 @@
 		{
 			"SchemaName": "public",
 			"ObjectName": "employees",
-			"RowCount": 5,
+			"RowCount": 10,
 			"ColumnCount": 4,
 			"Reads": 0,
 			"Writes": 10,
@@ -2069,6 +2085,20 @@
 			"ParentTableName": "schema2.mixed_data_types_table1",
 			"SizeInBytes": 8192
 		},
+		{
+			"SchemaName": "public",
+			"ObjectName": "employeesforview",
+			"RowCount": 0,
+			"ColumnCount": 4,
+			"Reads": 0,
+			"Writes": 0,
+			"ReadsPerSecond": 0,
+			"WritesPerSecond": 0,
+			"IsIndex": false,
+			"ObjectType": "",
+			"ParentTableName": null,
+			"SizeInBytes": 0
+		},
 		{
 			"SchemaName": "public",
 			"ObjectName": "employees3",
@@ -2167,6 +2197,20 @@
 			"ParentTableName": null,
 			"SizeInBytes": 0
 		},
+		{
+			"SchemaName": "schema2",
+			"ObjectName": "employeesforview",
+			"RowCount": 0,
+			"ColumnCount": 4,
+			"Reads": 0,
+			"Writes": 0,
+			"ReadsPerSecond": 0,
+			"WritesPerSecond": 0,
+			"IsIndex": false,
+			"ObjectType": "",
+			"ParentTableName": null,
+			"SizeInBytes": 0
+		},
 		{
 			"SchemaName": "schema2",
 			"ObjectName": "bigint_multirange_table",

diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql
@@ -393,6 +393,20 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
+-- for FETCH .. WITH TIES
+CREATE TABLE employeesForView (
+    id SERIAL PRIMARY KEY,
+    first_name VARCHAR(50) NOT NULL,
+    last_name VARCHAR(50) NOT NULL,
+    salary NUMERIC(10, 2) NOT NULL
+);
+
+CREATE VIEW top_employees_view AS SELECT * FROM (
+			SELECT * FROM employeesForView
+			ORDER BY salary DESC
+			FETCH FIRST 2 ROWS WITH TIES
+		) AS top_employees;
+
 -- SECURITY INVOKER VIEW
 CREATE TABLE public.employees (
     employee_id SERIAL PRIMARY KEY,

diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go
@@ -1023,6 +1023,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SYSTEM_COLUMNS_NAME, queryissue.SYSTEM_COLUMNS_NAME, "", schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.LARGE_OBJECT_FUNCTIONS_NAME, queryissue.LARGE_OBJECT_FUNCTIONS_NAME, "", schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(REGEX_FUNCTIONS_FEATURE, "", queryissue.REGEX_FUNCTIONS, schemaAnalysisReport, false, ""))
+	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(FETCH_WITH_TIES_FEATURE, "", queryissue.FETCH_WITH_TIES, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_QUERY_FUNCTIONS_NAME, "", queryissue.JSON_QUERY_FUNCTION, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, ""))
 	unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, ""))

diff --git a/yb-voyager/cmd/constants.go b/yb-voyager/cmd/constants.go
@@ -217,6 +217,7 @@ const (
 	BEFORE_FOR_EACH_ROW_TRIGGERS_ON_PARTITIONED_TABLE_FEATURE = "BEFORE ROW triggers on Partitioned tables"
 	PK_UK_CONSTRAINT_ON_COMPLEX_DATATYPES_FEATURE             = "Primary / Unique key constraints on complex datatypes"
 	REGEX_FUNCTIONS_FEATURE                                   = "Regex Functions"
+	FETCH_WITH_TIES_FEATURE                                   = "FETCH .. WITH TIES Clause"
 
 	// Migration caveats
 

diff --git a/yb-voyager/go.mod b/yb-voyager/go.mod
@@ -15,7 +15,6 @@ require (
 	github.com/docker/go-connections v0.5.0
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fatih/color v1.13.0
-	github.com/fergusstrange/embedded-postgres v1.29.0
 	github.com/go-sql-driver/mysql v1.7.0
 	github.com/godror/godror v0.30.2
 	github.com/google/uuid v1.6.0
@@ -25,7 +24,6 @@ require (
 	github.com/jackc/pgconn v1.14.3
 	github.com/jackc/pgx/v4 v4.18.3
 	github.com/jackc/pgx/v5 v5.0.3
-	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2
 	github.com/mitchellh/go-ps v1.0.0
@@ -83,7 +81,6 @@ require (
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
-	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect